diff --git a/.github/workflows/kotlin.yml b/.github/workflows/kotlin.yml index 84429df339..b9bdb988ac 100644 --- a/.github/workflows/kotlin.yml +++ b/.github/workflows/kotlin.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest env: - LDK_NODE_EXTRA_FEATURES: chain-bitcoind + LDK_NODE_EXTRA_FEATURES: chain-bitcoind,storage-tier LDK_NODE_JVM_DIR: bindings/kotlin/ldk-node-jvm LDK_NODE_ANDROID_DIR: bindings/kotlin/ldk-node-android diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 3639a1a1c5..21227fc8cd 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -13,7 +13,7 @@ jobs: env: CARGO_TARGET_DIR: /tmp/cargo-target-ldk-node-python-ci - LDK_NODE_EXTRA_FEATURES: chain-bitcoind + LDK_NODE_EXTRA_FEATURES: chain-bitcoind,storage-tier LDK_NODE_PYTHON_DIR: bindings/python steps: diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ea3df44b28..79d30998cf 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -86,6 +86,14 @@ jobs: if: "matrix.platform != 'windows-latest'" run: | RUSTFLAGS="--cfg no_download --cfg cycle_tests --cfg tokio_unstable" cargo test + - name: Test tiered storage + if: matrix.check-fmt + run: | + cargo test --lib --features storage-tier io::tier_store + RUSTFLAGS="--cfg no_download" cargo test \ + --features storage-tier \ + --test integration_tests_rust \ + builder_configures_sqlite_backup_store - name: Test with UniFFI support on Rust ${{ matrix.toolchain }} if: "matrix.platform != 'windows-latest' && matrix.build-uniffi" run: | diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index af3928c4a8..cd8841f7d4 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -12,7 +12,7 @@ jobs: runs-on: macos-latest env: - LDK_NODE_EXTRA_FEATURES: chain-bitcoind + LDK_NODE_EXTRA_FEATURES: chain-bitcoind,storage-tier steps: - name: Checkout repository diff --git a/Cargo.toml b/Cargo.toml index 0110c027b0..11bec90d1c 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ keywords = ["bitcoin", "lightning", "ldk", "bdk"] categories = ["cryptography::cryptocurrencies"] [package.metadata.docs.rs] -features = ["storage-postgres-vendored-tls"] +features = ["storage-postgres-vendored-tls", "storage-tier"] rustdoc-args = ["--cfg", "docsrs"] [lib] @@ -53,6 +53,7 @@ chain-electrum = [ ] chain-bitcoind = ["dep:lightning-block-sync"] storage-sqlite = ["dep:rusqlite"] +storage-tier = ["storage-sqlite"] storage-filesystem = ["dep:lightning-persister"] storage-vss = ["dep:vss-client", "dep:prost"] storage-postgres = ["dep:tokio-postgres", "dep:native-tls", "dep:postgres-native-tls"] diff --git a/README.md b/README.md index ec9de516ff..f42a8f3ff2 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,7 @@ LDK Node's optional dependencies are grouped by the functionality they provide: | `chain-electrum` | Electrum chain source | | `chain-bitcoind` | Bitcoin Core RPC and REST chain source | | `storage-sqlite` | SQLite storage | +| `storage-tier` | Tiered storage with optional ephemeral and backup SQLite stores | | `storage-filesystem` | Filesystem storage | | `storage-vss` | Versioned Storage Service storage | | `storage-postgres` | PostgreSQL storage | @@ -88,7 +89,8 @@ LDK Node's optional dependencies are grouped by the functionality they provide: The `default` feature set preserves the native Rust API's previous behavior. It enables all three chain sources, SQLite, filesystem and VSS storage, and unified payments. PostgreSQL and UniFFI -remain opt-in. Every build must enable at least one chain source feature. +remain opt-in. Tiered storage is also opt-in; enabling `storage-tier` automatically enables +`storage-sqlite`. Every build must enable at least one chain source feature. On Linux, `storage-postgres` uses the system OpenSSL installation and requires the OpenSSL development headers and `pkg-config`. Enable `storage-postgres-vendored-tls` instead to build diff --git a/bindings/ldk_node.udl b/bindings/ldk_node.udl index 4c4c1a438a..e54b8b02df 100644 --- a/bindings/ldk_node.udl +++ b/bindings/ldk_node.udl @@ -31,6 +31,28 @@ interface LogWriter { void log(LogRecord record); }; +[Trait, WithForeign] +interface DynStoreTrait { + [Throws=IOError, Async] + bytes read(string primary_namespace, string secondary_namespace, string key); + [Throws=IOError, Async] + void write(string primary_namespace, string secondary_namespace, string key, bytes buf); + [Throws=IOError, Async] + void remove(string primary_namespace, string secondary_namespace, string key, boolean lazy); + [Throws=IOError, Async] + sequence list(string primary_namespace, string secondary_namespace); + [Throws=IOError, Async] + PaginatedListResponse list_paginated(string primary_namespace, string secondary_namespace, PageToken? page_token); + [Throws=IOError, Async] + sequence list_all_keys(); +}; + +typedef dictionary KVStoreKey; + +typedef dictionary PaginatedListResponse; + +typedef enum IOError; + interface ProbingConfigBuilder { [Name=high_degree] constructor(u64 top_node_count); diff --git a/bindings/python/src/ldk_node/test_ldk_node.py b/bindings/python/src/ldk_node/test_ldk_node.py index 08a866c3f0..399cd71e77 100644 --- a/bindings/python/src/ldk_node/test_ldk_node.py +++ b/bindings/python/src/ldk_node/test_ldk_node.py @@ -1,3 +1,6 @@ +import asyncio +import sqlite3 +import threading import unittest import tempfile import time @@ -8,11 +11,79 @@ import socket from ldk_node import * +from ldk_node.ldk_node import uniffi_set_event_loop DEFAULT_ESPLORA_SERVER_URL = "http://127.0.0.1:3002" DEFAULT_TEST_NETWORK = Network.REGTEST DEFAULT_BITCOIN_CLI_BIN = "bitcoin-cli" +class TestKvStore: + """Thread-safe in-memory store implementing the foreign DynStoreTrait.""" + + def __init__(self): + self._storage = {} + self._lock = threading.Lock() + + def put(self, primary_namespace, secondary_namespace, key, value): + with self._lock: + namespace = self._storage.setdefault( + (primary_namespace, secondary_namespace), {} + ) + namespace[key] = bytes(value) + + def get(self, primary_namespace, secondary_namespace, key): + with self._lock: + namespace = self._storage.get((primary_namespace, secondary_namespace), {}) + return namespace.get(key) + + def keys(self, primary_namespace, secondary_namespace): + with self._lock: + namespace = self._storage.get((primary_namespace, secondary_namespace), {}) + return list(namespace) + + async def read(self, primary_namespace, secondary_namespace, key): + value = self.get(primary_namespace, secondary_namespace, key) + if value is None: + raise IoError.NotFound() + return value + + async def write(self, primary_namespace, secondary_namespace, key, buf): + self.put(primary_namespace, secondary_namespace, key, buf) + + async def remove(self, primary_namespace, secondary_namespace, key, lazy): + with self._lock: + namespace_key = (primary_namespace, secondary_namespace) + namespace = self._storage.get(namespace_key) + if namespace is None: + return + namespace.pop(key, None) + if not namespace: + del self._storage[namespace_key] + + async def list(self, primary_namespace, secondary_namespace): + return self.keys(primary_namespace, secondary_namespace) + + async def list_paginated( + self, primary_namespace, secondary_namespace, page_token + ): + if page_token is not None: + return PaginatedListResponse(keys=[], next_page_token=None) + + keys = list(reversed(self.keys(primary_namespace, secondary_namespace))) + return PaginatedListResponse(keys=keys, next_page_token=None) + + async def list_all_keys(self): + with self._lock: + return [ + KvStoreKey( + primary_namespace=primary_namespace, + secondary_namespace=secondary_namespace, + key=key, + ) + for (primary_namespace, secondary_namespace), namespace in self._storage.items() + for key in namespace + ] + def bitcoin_cli(cmd): args = [] @@ -108,6 +179,46 @@ def setup_node(tmp_dir, esplora_endpoint, listening_addresses): builder.set_listening_addresses(listening_addresses) return builder.build(node_entropy) +def setup_tiered_node( + tmp_dir, + backup_dir, + ephemeral_dir, + esplora_endpoint, + listening_addresses, + primary_store, +): + mnemonic = Mnemonic.generate(24) + node_entropy = NodeEntropy.from_bip39_mnemonic(mnemonic, None) + builder = Builder.from_config(default_config()) + builder.set_storage_dir_path(tmp_dir) + builder.set_chain_source_esplora(esplora_endpoint, None) + builder.set_network(DEFAULT_TEST_NETWORK) + builder.set_listening_addresses(listening_addresses) + builder.set_backup_storage_dir_path(backup_dir) + builder.set_ephemeral_storage_dir_path(ephemeral_dir) + return builder.build_with_store(node_entropy, primary_store) + +def read_sqlite_value(store_dir, database_name, primary_namespace, secondary_namespace, key): + database_path = os.path.join(store_dir, database_name) + with sqlite3.connect(database_path) as connection: + row = connection.execute( + """SELECT value FROM ldk_node_data + WHERE primary_namespace = ? AND secondary_namespace = ? AND key = ?""", + (primary_namespace, secondary_namespace, key), + ).fetchone() + return None if row is None else row[0] + +def list_sqlite_keys(store_dir, database_name, primary_namespace, secondary_namespace): + database_path = os.path.join(store_dir, database_name) + with sqlite3.connect(database_path) as connection: + rows = connection.execute( + """SELECT key FROM ldk_node_data + WHERE primary_namespace = ? AND secondary_namespace = ? + ORDER BY key""", + (primary_namespace, secondary_namespace), + ).fetchall() + return [row[0] for row in rows] + def get_esplora_endpoint(): if os.environ.get('ESPLORA_ENDPOINT'): return str(os.environ['ESPLORA_ENDPOINT']) @@ -365,5 +476,174 @@ def test_channel_full_cycle(self): # Stop nodes stop_and_cleanup(node_1, node_2, tmp_dir_1, tmp_dir_2) + def test_tier_store_with_python_primary(self): + loop = asyncio.new_event_loop() + + def run_loop(): + asyncio.set_event_loop(loop) + loop.run_forever() + + loop_thread = threading.Thread(target=run_loop, daemon=True) + loop_thread.start() + uniffi_set_event_loop(loop) + + node_1 = None + node_2 = None + tmp_dir_1 = tempfile.TemporaryDirectory("_ldk_node_1") + tmp_dir_2 = tempfile.TemporaryDirectory("_ldk_node_2") + backup_dir = tempfile.TemporaryDirectory("_ldk_node_backup") + ephemeral_dir = tempfile.TemporaryDirectory("_ldk_node_ephemeral") + + try: + primary_store = TestKvStore() + preexisting_value = b"preexisting durable value" + primary_store.put("test", "", "preexisting", preexisting_value) + + port_1, port_2 = find_two_free_ports() + listening_addresses_1 = [f"127.0.0.1:{port_1}"] + listening_addresses_2 = [f"127.0.0.1:{port_2}"] + esplora_endpoint = get_esplora_endpoint() + + node_1 = setup_tiered_node( + tmp_dir_1.name, + backup_dir.name, + ephemeral_dir.name, + esplora_endpoint, + listening_addresses_1, + primary_store, + ) + node_2 = setup_node(tmp_dir_2.name, esplora_endpoint, listening_addresses_2) + node_1.start() + node_2.start() + + fund_nodes(node_1, node_2, esplora_endpoint) + _, channel_ready_event_2, _ = open_channel_and_wait_ready( + node_1, + node_2, + node_2.node_id(), + listening_addresses_2[0], + esplora_endpoint, + ) + + invoice = node_2.bolt11_payment().receive( + 2_500_000, Bolt11InvoiceDescription.DIRECT("tiered storage"), 9217 + ) + node_1.bolt11_payment().send(invoice, None) + expect_event(node_1, Event.PAYMENT_SUCCESSFUL) + expect_event(node_2, Event.PAYMENT_RECEIVED) + + node_2.close_channel(channel_ready_event_2.user_channel_id, node_1.node_id()) + expect_event(node_1, Event.CHANNEL_CLOSED) + expect_event(node_2, Event.CHANNEL_CLOSED) + + node_1.stop() + node_1 = None + node_2.stop() + node_2 = None + + backup_database = "ldk_node_data_backup.sqlite" + ephemeral_database = "ldk_node_data_ephemeral.sqlite" + + self.assertEqual( + read_sqlite_value( + backup_dir.name, + backup_database, + "test", + "", + "preexisting", + ), + preexisting_value, + ) + + channel_manager = primary_store.get("", "", "manager") + self.assertIsNotNone(channel_manager) + self.assertEqual( + read_sqlite_value( + backup_dir.name, + backup_database, + "", + "", + "manager", + ), + channel_manager, + ) + self.assertIsNone( + read_sqlite_value( + ephemeral_dir.name, + ephemeral_database, + "", + "", + "manager", + ) + ) + + wallet_descriptor = primary_store.get("bdk_wallet", "", "descriptor") + self.assertIsNotNone(wallet_descriptor) + self.assertEqual( + read_sqlite_value( + backup_dir.name, + backup_database, + "bdk_wallet", + "", + "descriptor", + ), + wallet_descriptor, + ) + self.assertIsNone( + read_sqlite_value( + ephemeral_dir.name, + ephemeral_database, + "bdk_wallet", + "", + "descriptor", + ) + ) + + primary_payments = sorted(primary_store.keys("payments", "")) + self.assertGreater(len(primary_payments), 0) + self.assertEqual( + list_sqlite_keys(backup_dir.name, backup_database, "payments", ""), + primary_payments, + ) + self.assertEqual( + list_sqlite_keys(ephemeral_dir.name, ephemeral_database, "payments", ""), + [], + ) + + self.assertIsNone(primary_store.get("", "", "network_graph")) + self.assertIsNone( + read_sqlite_value( + backup_dir.name, + backup_database, + "", + "", + "network_graph", + ) + ) + self.assertIsNotNone( + read_sqlite_value( + ephemeral_dir.name, + ephemeral_database, + "", + "", + "network_graph", + ) + ) + finally: + for node in (node_1, node_2): + if node is not None: + try: + node.stop() + except NodeError: + pass + tmp_dir_1.cleanup() + tmp_dir_2.cleanup() + backup_dir.cleanup() + ephemeral_dir.cleanup() + loop.call_soon_threadsafe(loop.stop) + loop_thread.join(timeout=5) + uniffi_set_event_loop(None) + loop.close() + if __name__ == '__main__': unittest.main() diff --git a/src/builder.rs b/src/builder.rs index ab641cf86b..0543500c7a 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -11,7 +11,7 @@ use std::convert::TryInto; use std::default::Default; #[cfg(feature = "unified-payments")] use std::net::ToSocketAddrs; -#[cfg(feature = "storage-filesystem")] +#[cfg(any(feature = "storage-filesystem", feature = "storage-tier"))] use std::path::PathBuf; use std::sync::{Arc, Mutex, Once, RwLock}; use std::time::SystemTime; @@ -41,6 +41,8 @@ use lightning::routing::scoring::{ }; use lightning::sign::{EntropySource, NodeSigner}; use lightning::util::config::HTLCInterceptionFlags; +#[cfg(feature = "storage-tier")] +use lightning::util::persist::MigratableKVStore; use lightning::util::persist::{ KVStore, PaginatedKVStore, CHANNEL_MANAGER_PERSISTENCE_KEY, CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, @@ -67,11 +69,15 @@ use crate::data_store::{KeepAllEntries, KeepLeastRecentlyUsed}; use crate::entropy::NodeEntropy; use crate::event::EventQueue; use crate::fee_estimator::OnchainFeeEstimator; +#[cfg(all(feature = "uniffi", feature = "storage-tier"))] +use crate::ffi::DynStoreTrait; use crate::gossip::GossipSource; #[cfg(feature = "storage-filesystem")] use crate::io::fs_store::open_or_migrate_fs_store; #[cfg(feature = "storage-sqlite")] use crate::io::sqlite_store::SqliteStore; +#[cfg(feature = "storage-tier")] +use crate::io::tier_store::{setup_index_store, BackupSyncStatus, TierStore}; use crate::io::utils::{ read_all_objects, read_event_queue, read_external_pathfinding_scores_from_cache, read_n_objects, read_network_graph, read_node_metrics, read_output_sweeper, read_peer_info, @@ -173,6 +179,13 @@ impl std::fmt::Debug for LogWriterConfig { } } +#[cfg(feature = "storage-tier")] +#[derive(Default, Debug)] +struct TierStoreConfig { + ephemeral_storage_dir_path: Option, + backup_storage_dir_path: Option, +} + /// An error encountered during building a [`Node`]. /// /// [`Node`]: crate::Node @@ -326,6 +339,8 @@ pub struct NodeBuilder { liquidity_source_config: Option, log_writer_config: Option, async_payments_role: Option, + #[cfg(feature = "storage-tier")] + tier_store_config: Option, runtime_handle: Option, pathfinding_scores_sync_config: Option, probing_config: Option, @@ -347,6 +362,8 @@ impl NodeBuilder { let gossip_source_config = None; let liquidity_source_config = None; let log_writer_config = None; + #[cfg(feature = "storage-tier")] + let tier_store_config = None; let runtime_handle = None; let pathfinding_scores_sync_config = None; let probing_config = None; @@ -356,6 +373,8 @@ impl NodeBuilder { gossip_source_config, liquidity_source_config, log_writer_config, + #[cfg(feature = "storage-tier")] + tier_store_config, runtime_handle, async_payments_role: None, pathfinding_scores_sync_config, @@ -686,6 +705,41 @@ impl NodeBuilder { self } + /// Configures a local SQLite backup store for disaster recovery. + /// + /// When building with tiered storage, a SQLite store will be created at the + /// given directory path using [`SQLITE_BACKUP_DB_FILE_NAME`] as its database + /// file name. It receives a second durable copy of data written to the + /// primary store. + /// + /// Writes and removals for primary-backed data only succeed once both the + /// primary and backup SQLite stores complete successfully. + /// + /// If not set, durable data will be stored only in the primary store. + /// + /// [`SQLITE_BACKUP_DB_FILE_NAME`]: crate::io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME + #[cfg(feature = "storage-tier")] + pub fn set_backup_storage_dir_path(&mut self, backup_storage_dir_path: String) -> &mut Self { + let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default()); + tier_store_config.backup_storage_dir_path = Some(backup_storage_dir_path.into()); + self + } + + /// Configures the ephemeral storage directory path for non-critical, frequently-accessed data. + /// + /// When set, a local SQLite store is created at this path for ephemeral data like + /// the network graph and scorer. Data stored here can be rebuilt if lost. + /// + /// If not set, non-critical data will be stored in the primary store. + #[cfg(feature = "storage-tier")] + pub fn set_ephemeral_storage_dir_path( + &mut self, ephemeral_storage_dir_path: String, + ) -> &mut Self { + let tier_store_config = self.tier_store_config.get_or_insert(TierStoreConfig::default()); + tier_store_config.ephemeral_storage_dir_path = Some(ephemeral_storage_dir_path.into()); + self + } + /// Builds a [`Node`] instance with a [`SqliteStore`] backend and according to the options /// previously configured. #[cfg(feature = "storage-sqlite")] @@ -901,11 +955,41 @@ impl NodeBuilder { } /// Builds a [`Node`] instance according to the options previously configured. + /// + /// The provided `kv_store` will be used as the primary storage backend. Optionally, + /// an ephemeral store for frequently-accessed non-critical data (e.g., network graph, scorer) + /// and a local SQLite backup store for disaster recovery can be configured via + /// [`set_ephemeral_storage_dir_path`] and [`set_backup_storage_dir_path`]. + /// + /// [`set_ephemeral_storage_dir_path`]: Self::set_ephemeral_storage_dir_path + /// [`set_backup_storage_dir_path`]: Self::set_backup_storage_dir_path + #[cfg(not(feature = "storage-tier"))] + #[cfg_attr(feature = "uniffi", allow(dead_code))] pub fn build_with_store( &self, node_entropy: NodeEntropy, kv_store: S, ) -> Result { let logger = setup_logger(&self.log_writer_config, &self.config)?; + self.build_with_store_and_logger(node_entropy, kv_store, logger) + } + /// Builds a [`Node`] instance according to the options previously configured. + /// + /// The provided `kv_store` will be used as the primary storage backend. Optionally, + /// an ephemeral store for frequently-accessed non-critical data (e.g., network graph, scorer) + /// and a local SQLite backup store for disaster recovery can be configured via + /// [`set_ephemeral_storage_dir_path`] and [`set_backup_storage_dir_path`]. + /// + /// The store must implement [`MigratableKVStore`] so a configured backup can be backfilled or + /// resilvered from the complete set of primary-store keys before the node starts. + /// + /// [`set_ephemeral_storage_dir_path`]: Self::set_ephemeral_storage_dir_path + /// [`set_backup_storage_dir_path`]: Self::set_backup_storage_dir_path + #[cfg(feature = "storage-tier")] + #[cfg_attr(feature = "uniffi", allow(dead_code))] + pub fn build_with_store( + &self, node_entropy: NodeEntropy, kv_store: S, + ) -> Result { + let logger = setup_logger(&self.log_writer_config, &self.config)?; self.build_with_store_and_logger(node_entropy, kv_store, logger) } @@ -920,6 +1004,7 @@ impl NodeBuilder { } } + #[cfg(not(feature = "storage-tier"))] fn build_with_store_and_logger( &self, node_entropy: NodeEntropy, kv_store: S, logger: Arc, ) -> Result { @@ -927,8 +1012,121 @@ impl NodeBuilder { self.build_with_store_runtime_and_logger(node_entropy, kv_store, runtime, logger) } + #[cfg(feature = "storage-tier")] + fn build_with_store_and_logger< + S: PaginatedKVStore + MigratableKVStore + Send + Sync + 'static, + >( + &self, node_entropy: NodeEntropy, kv_store: S, logger: Arc, + ) -> Result { + let runtime = self.setup_runtime(&logger)?; + self.build_with_store_runtime_and_logger(node_entropy, kv_store, runtime, logger) + } + + #[cfg(not(feature = "storage-tier"))] fn build_with_store_runtime_and_logger( &self, node_entropy: NodeEntropy, kv_store: S, runtime: Arc, logger: Arc, + ) -> Result { + let store: Arc = Arc::new(DynStoreWrapper(kv_store)); + self.build_with_dyn_store(node_entropy, store, runtime, logger) + } + + #[cfg(feature = "storage-tier")] + fn build_with_store_runtime_and_logger< + S: PaginatedKVStore + MigratableKVStore + Send + Sync + 'static, + >( + &self, node_entropy: NodeEntropy, kv_store: S, runtime: Arc, logger: Arc, + ) -> Result { + let primary_store: Arc = Arc::new(DynStoreWrapper(kv_store)); + let store = self.setup_tier_store(primary_store, &runtime, &logger)?; + self.build_with_dyn_store(node_entropy, store, runtime, logger) + } + + #[cfg(all(feature = "uniffi", feature = "storage-tier"))] + fn build_with_ffi_store( + &self, node_entropy: NodeEntropy, kv_store: Arc, + ) -> Result { + let logger = setup_logger(&self.log_writer_config, &self.config)?; + let runtime = self.setup_runtime(&logger)?; + let primary_store: Arc = Arc::new(crate::ffi::DynStore::new(kv_store)); + let store = self.setup_tier_store(primary_store, &runtime, &logger)?; + self.build_with_dyn_store(node_entropy, store, runtime, logger) + } + + #[cfg(feature = "storage-tier")] + fn setup_tier_store( + &self, primary_store: Arc, runtime: &Arc, logger: &Arc, + ) -> Result, BuildError> { + let store: Arc = { + let ts_config = self.tier_store_config.as_ref(); + let mut tier_store = TierStore::new(primary_store, Arc::clone(&logger)); + let tier_index_exists = PathBuf::from(&self.config.storage_dir_path) + .join(io::sqlite_store::SQLITE_TIER_INDEX_DB_FILE_NAME) + .exists(); + let tier_index_required = ts_config + .map(|config| { + config.ephemeral_storage_dir_path.is_some() + || config.backup_storage_dir_path.is_some() + }) + .unwrap_or(false); + if tier_index_required || tier_index_exists { + let index_store = runtime + .block_on(setup_index_store(self.config.storage_dir_path.clone().into())) + .map_err(|e| { + log_error!(logger, "Failed to setup tier-store index: {}", e); + BuildError::KVStoreSetupFailed + })?; + tier_store.set_index_store(index_store); + } + if let Some(config) = ts_config { + if let Some(ephemeral_storage_dir_path) = config.ephemeral_storage_dir_path.as_ref() + { + let ephemeral_store = SqliteStore::new( + ephemeral_storage_dir_path.clone(), + Some(io::sqlite_store::SQLITE_EPHEMERAL_DB_FILE_NAME.to_string()), + Some(io::sqlite_store::KV_TABLE_NAME.to_string()), + ) + .map_err(|e| { + log_error!(logger, "Failed to setup ephemeral SQLite store: {}", e); + BuildError::KVStoreSetupFailed + })?; + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(ephemeral_store)); + tier_store.set_ephemeral_store(ephemeral_store); + } + + if let Some(backup_storage_dir_path) = config.backup_storage_dir_path.as_ref() { + let backup_store = SqliteStore::new( + backup_storage_dir_path.clone(), + Some(io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME.to_string()), + Some(io::sqlite_store::KV_TABLE_NAME.to_string()), + ) + .map_err(|e| { + log_error!(logger, "Failed to setup backup SQLite store: {}", e); + BuildError::KVStoreSetupFailed + })?; + let backup_store: Arc = Arc::new(DynStoreWrapper(backup_store)); + tier_store.set_backup_store(backup_store); + } + } + runtime + .block_on(async { + let status = tier_store.initialize_backup_synchronization().await?; + if status == BackupSyncStatus::Required { + tier_store.synchronize_backup().await?; + } + Ok::<(), bitcoin::io::Error>(()) + }) + .map_err(|e| { + log_error!(logger, "Failed to prepare or synchronize tier-store backup: {}", e); + BuildError::KVStoreSetupFailed + })?; + Arc::new(DynStoreWrapper(tier_store)) + }; + Ok(store) + } + + fn build_with_dyn_store( + &self, node_entropy: NodeEntropy, store: Arc, runtime: Arc, + logger: Arc, ) -> Result { let seed_bytes = node_entropy.to_seed_bytes(); let config = Arc::new(self.config.clone()); @@ -944,7 +1142,7 @@ impl NodeBuilder { seed_bytes, runtime, logger, - Arc::new(DynStoreWrapper(kv_store)), + store, ) } } @@ -1470,11 +1668,11 @@ impl Builder { } } -#[cfg(feature = "uniffi")] +#[cfg(all(feature = "uniffi", not(feature = "storage-tier")))] impl ArcedNodeBuilder { /// Builds a [`Node`] instance according to the options previously configured. - // Note that the generics here don't actually work for Uniffi, but we don't currently expose - // this so its not needed. + // Note that the generics here don't actually work for UniFFI, but we don't currently expose + // this so it is only used by Rust tests compiled with the `uniffi` feature. pub fn build_with_store( &self, node_entropy: Arc, kv_store: S, ) -> Result, BuildError> { @@ -1482,6 +1680,36 @@ impl ArcedNodeBuilder { } } +#[cfg(all(feature = "uniffi", feature = "storage-tier"))] +#[uniffi::export] +impl Builder { + /// Configures a local SQLite backup store for durable data. + /// + /// The backup is brought up to date before node construction completes. While configured, + /// durable writes and removals only succeed when both primary and backup storage succeed. + pub fn set_backup_storage_dir_path(&self, backup_storage_dir_path: String) { + self.inner.write().expect("lock").set_backup_storage_dir_path(backup_storage_dir_path); + } + + /// Configures a local SQLite store for rebuildable cache data. + pub fn set_ephemeral_storage_dir_path(&self, ephemeral_storage_dir_path: String) { + self.inner + .write() + .expect("lock") + .set_ephemeral_storage_dir_path(ephemeral_storage_dir_path); + } + + /// Builds a [`Node`] instance according to the options previously configured. + /// + /// The provided store is used as authoritative primary storage. It must support paginated + /// namespace listing and exhaustive key enumeration for backup synchronization. + pub fn build_with_store( + &self, node_entropy: Arc, kv_store: Arc, + ) -> Result, BuildError> { + self.inner.read().expect("lock").build_with_ffi_store(*node_entropy, kv_store).map(Arc::new) + } +} + /// Builds a [`Node`] instance according to the options previously configured. fn build_with_store_internal( config: Arc, chain_data_source_config: Option<&ChainDataSourceConfig>, @@ -2579,11 +2807,13 @@ pub(crate) fn sanitize_alias(alias_str: &str) -> Result { #[cfg(test)] mod tests { use std::future::Future; + #[cfg(all(feature = "storage-tier", not(feature = "uniffi")))] + use std::path::PathBuf; use std::sync::Arc; use lightning::io; use lightning::util::persist::{ - KVStore, PageToken, PaginatedKVStore, PaginatedListResponse, + KVStore, MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse, CHANNEL_MANAGER_PERSISTENCE_KEY, CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, }; @@ -2591,18 +2821,52 @@ mod tests { use super::{sanitize_alias, BuildError, NodeAlias, NodeBuilder}; use crate::entropy::NodeEntropy; use crate::io::test_utils::InMemoryStore; + #[cfg(all(feature = "storage-tier", not(feature = "uniffi")))] + use crate::io::{ + sqlite_store::{SqliteStore, KV_TABLE_NAME, SQLITE_BACKUP_DB_FILE_NAME}, + test_utils::random_storage_path, + }; use crate::logger::Logger; - struct ChannelManagerReadFailingStore(InMemoryStore); + #[cfg(all(feature = "storage-tier", not(feature = "uniffi")))] + struct CleanupDir(PathBuf); + + #[cfg(all(feature = "storage-tier", not(feature = "uniffi")))] + impl Drop for CleanupDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + #[derive(Clone, Copy)] + enum BuilderStoreBehavior { + #[cfg(all(feature = "storage-tier", not(feature = "uniffi")))] + Normal, + FailChannelManagerRead, + #[cfg(all(feature = "storage-tier", not(feature = "uniffi")))] + FailListAllKeys, + } + + struct BuilderTestStore { + inner: InMemoryStore, + behavior: BuilderStoreBehavior, + } + + impl BuilderTestStore { + fn new(behavior: BuilderStoreBehavior) -> Self { + Self { inner: InMemoryStore::new(), behavior } + } + } - impl KVStore for ChannelManagerReadFailingStore { + impl KVStore for BuilderTestStore { fn read( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, ) -> impl Future, io::Error>> + 'static + Send { - let fail_read = primary_namespace == CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE + let fail_read = matches!(self.behavior, BuilderStoreBehavior::FailChannelManagerRead) + && primary_namespace == CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE && secondary_namespace == CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE && key == CHANNEL_MANAGER_PERSISTENCE_KEY; - let read = KVStore::read(&self.0, primary_namespace, secondary_namespace, key); + let read = KVStore::read(&self.inner, primary_namespace, secondary_namespace, key); async move { if fail_read { Err(io::Error::new(io::ErrorKind::Other, "channel manager read failed")) @@ -2615,29 +2879,29 @@ mod tests { fn write( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, ) -> impl Future> + 'static + Send { - KVStore::write(&self.0, primary_namespace, secondary_namespace, key, buf) + KVStore::write(&self.inner, primary_namespace, secondary_namespace, key, buf) } fn remove( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, ) -> impl Future> + 'static + Send { - KVStore::remove(&self.0, primary_namespace, secondary_namespace, key, lazy) + KVStore::remove(&self.inner, primary_namespace, secondary_namespace, key, lazy) } fn list( &self, primary_namespace: &str, secondary_namespace: &str, ) -> impl Future, io::Error>> + 'static + Send { - KVStore::list(&self.0, primary_namespace, secondary_namespace) + KVStore::list(&self.inner, primary_namespace, secondary_namespace) } } - impl PaginatedKVStore for ChannelManagerReadFailingStore { + impl PaginatedKVStore for BuilderTestStore { fn list_paginated( &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, ) -> impl Future> + 'static + Send { PaginatedKVStore::list_paginated( - &self.0, + &self.inner, primary_namespace, secondary_namespace, page_token, @@ -2645,6 +2909,26 @@ mod tests { } } + impl MigratableKVStore for BuilderTestStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send + { + #[cfg(all(feature = "storage-tier", not(feature = "uniffi")))] + let fail = matches!(self.behavior, BuilderStoreBehavior::FailListAllKeys); + #[cfg(any(not(feature = "storage-tier"), feature = "uniffi"))] + let fail = false; + let list = MigratableKVStore::list_all_keys(&self.inner); + async move { + if fail { + Err(io::Error::new(io::ErrorKind::Other, "list_all_keys failed")) + } else { + list.await + } + } + } + } + #[test] fn channel_manager_read_failure_fails_build() { let builder = NodeBuilder::new(); @@ -2656,13 +2940,80 @@ mod tests { let result = builder.build_with_store_and_logger( node_entropy, - ChannelManagerReadFailingStore(InMemoryStore::new()), + BuilderTestStore::new(BuilderStoreBehavior::FailChannelManagerRead), logger, ); assert!(matches!(result, Err(BuildError::ReadFailed))); } + #[cfg(all(feature = "storage-tier", not(feature = "uniffi")))] + #[test] + fn builder_synchronizes_a_configured_backup_before_returning() { + let base_dir = random_storage_path(); + let _cleanup = CleanupDir(base_dir.clone()); + let storage_dir = base_dir.join("node"); + let backup_dir = base_dir.join("backup"); + let runtime = tokio::runtime::Runtime::new().unwrap(); + let primary_store = BuilderTestStore::new(BuilderStoreBehavior::Normal); + runtime.block_on(primary_store.write("namespace", "", "current", vec![1])).unwrap(); + let backup_store = SqliteStore::new( + backup_dir.clone(), + Some(SQLITE_BACKUP_DB_FILE_NAME.to_string()), + Some(KV_TABLE_NAME.to_string()), + ) + .unwrap(); + runtime + .block_on(async { backup_store.write("namespace", "", "stale", vec![2]).await }) + .unwrap(); + drop(backup_store); + drop(runtime); + + let mut builder = NodeBuilder::new(); + builder + .set_storage_dir_path(storage_dir.to_string_lossy().into_owned()) + .set_backup_storage_dir_path(backup_dir.to_string_lossy().into_owned()); + let node = builder + .build_with_store(NodeEntropy::from_seed_bytes([42; 64]), primary_store) + .unwrap(); + + let backup_store = SqliteStore::new( + backup_dir, + Some(SQLITE_BACKUP_DB_FILE_NAME.to_string()), + Some(KV_TABLE_NAME.to_string()), + ) + .unwrap(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + assert_eq!( + runtime + .block_on(async { backup_store.read("namespace", "", "current").await }) + .unwrap(), + vec![1] + ); + assert!(runtime + .block_on(async { backup_store.read("namespace", "", "stale").await }) + .is_err()); + drop(node); + } + + #[cfg(all(feature = "storage-tier", not(feature = "uniffi")))] + #[test] + fn builder_fails_when_backup_synchronization_cannot_list_primary_keys() { + let base_dir = random_storage_path(); + let _cleanup = CleanupDir(base_dir.clone()); + let mut builder = NodeBuilder::new(); + builder + .set_storage_dir_path(base_dir.join("node").to_string_lossy().into_owned()) + .set_backup_storage_dir_path(base_dir.join("backup").to_string_lossy().into_owned()); + + let result = builder.build_with_store( + NodeEntropy::from_seed_bytes([42; 64]), + BuilderTestStore::new(BuilderStoreBehavior::FailListAllKeys), + ); + + assert!(matches!(result, Err(BuildError::KVStoreSetupFailed))); + } + #[test] fn sanitize_empty_node_alias() { // Empty node alias diff --git a/src/data_store.rs b/src/data_store.rs index a9fe0d0f59..b424615d5e 100644 --- a/src/data_store.rs +++ b/src/data_store.rs @@ -750,7 +750,9 @@ mod tests { use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::Duration; - use lightning::util::persist::{PageToken, PaginatedKVStore, PaginatedListResponse}; + use lightning::util::persist::{ + MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse, + }; use lightning::util::test_utils::TestLogger; use lightning::{impl_writeable_tlv_based, io}; use tokio::sync::Notify; @@ -902,6 +904,16 @@ mod tests { } } + impl MigratableKVStore for FailingStore { + fn list_all_keys( + &self, + ) -> impl std::future::Future, io::Error>> + + 'static + + Send { + async { Err(io::Error::new(io::ErrorKind::Other, "list_all_keys failed")) } + } + } + fn new_failing_data_store(objects: Vec) -> DataStore> { let store: Arc = Arc::new(DynStoreWrapper(FailingStore)); let logger = Arc::new(TestLogger::new()); @@ -968,6 +980,16 @@ mod tests { } } + impl MigratableKVStore for GatedStore { + fn list_all_keys( + &self, + ) -> impl std::future::Future, io::Error>> + + 'static + + Send { + MigratableKVStore::list_all_keys(&self.inner) + } + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn readers_wait_for_in_flight_writes() { let write_parked = Arc::new(Notify::new()); @@ -1345,6 +1367,17 @@ mod tests { } } + impl MigratableKVStore for CountingStore { + fn list_all_keys( + &self, + ) -> impl std::future::Future, io::Error>> + + 'static + + Send { + self.lists.fetch_add(1, Ordering::Relaxed); + MigratableKVStore::list_all_keys(&self.inner) + } + } + /// A store whose writes and removals can be made to fail on demand, while reads keep working. /// /// Note a store that fails *reads* would be useless for testing the write paths of a bounded @@ -1404,6 +1437,16 @@ mod tests { } } + impl MigratableKVStore for WriteFailingStore { + fn list_all_keys( + &self, + ) -> impl std::future::Future, io::Error>> + + 'static + + Send { + MigratableKVStore::list_all_keys(&self.inner) + } + } + /// Returns a bounded store of the given capacity, together with a handle on the underlying /// `KVStore`, and the ids of `num_objects` objects inserted through it. /// @@ -1802,6 +1845,16 @@ mod tests { } } + impl MigratableKVStore for PhantomKeyStore { + fn list_all_keys( + &self, + ) -> impl std::future::Future, io::Error>> + + 'static + + Send { + MigratableKVStore::list_all_keys(&self.inner) + } + } + /// Sweeps every page and returns the objects in the order they were listed. async fn list_all_pages( data_store: &DataStore, P>, diff --git a/src/ffi/io.rs b/src/ffi/io.rs new file mode 100644 index 0000000000..31b573cca1 --- /dev/null +++ b/src/ffi/io.rs @@ -0,0 +1,381 @@ +// This file is Copyright its original authors, visible in version control history. +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license , at your option. You may not use this file except in +// accordance with one or both of these licenses. + +#[cfg(feature = "storage-tier")] +use std::future::Future; +#[cfg(feature = "storage-tier")] +use std::pin::Pin; +use std::sync::Arc; + +use lightning::util::persist::{KVStore, MigratableKVStore, PaginatedKVStore}; +#[cfg(feature = "storage-tier")] +use lightning::util::persist::{ + PageToken as LdkPageToken, PaginatedListResponse as LdkPaginatedListResponse, +}; + +use crate::ffi::PageToken; +#[cfg(feature = "storage-tier")] +use crate::io::utils::check_namespace_key_validity; + +/// An I/O error returned by a foreign key-value store. +#[derive(Debug, uniffi::Error)] +pub enum IOError { + NotFound, + PermissionDenied, + ConnectionRefused, + ConnectionReset, + ConnectionAborted, + NotConnected, + AddrInUse, + AddrNotAvailable, + BrokenPipe, + AlreadyExists, + WouldBlock, + InvalidInput, + InvalidData, + TimedOut, + WriteZero, + Interrupted, + UnexpectedEof, + Other, +} + +impl From for IOError { + fn from(error: bitcoin::io::Error) -> Self { + match error.kind() { + bitcoin::io::ErrorKind::NotFound => Self::NotFound, + bitcoin::io::ErrorKind::PermissionDenied => Self::PermissionDenied, + bitcoin::io::ErrorKind::ConnectionRefused => Self::ConnectionRefused, + bitcoin::io::ErrorKind::ConnectionReset => Self::ConnectionReset, + bitcoin::io::ErrorKind::ConnectionAborted => Self::ConnectionAborted, + bitcoin::io::ErrorKind::NotConnected => Self::NotConnected, + bitcoin::io::ErrorKind::AddrInUse => Self::AddrInUse, + bitcoin::io::ErrorKind::AddrNotAvailable => Self::AddrNotAvailable, + bitcoin::io::ErrorKind::BrokenPipe => Self::BrokenPipe, + bitcoin::io::ErrorKind::AlreadyExists => Self::AlreadyExists, + bitcoin::io::ErrorKind::WouldBlock => Self::WouldBlock, + bitcoin::io::ErrorKind::InvalidInput => Self::InvalidInput, + bitcoin::io::ErrorKind::InvalidData => Self::InvalidData, + bitcoin::io::ErrorKind::TimedOut => Self::TimedOut, + bitcoin::io::ErrorKind::WriteZero => Self::WriteZero, + bitcoin::io::ErrorKind::Interrupted => Self::Interrupted, + bitcoin::io::ErrorKind::UnexpectedEof => Self::UnexpectedEof, + bitcoin::io::ErrorKind::Other => Self::Other, + } + } +} + +impl From for bitcoin::io::Error { + fn from(error: IOError) -> Self { + match error { + IOError::NotFound => bitcoin::io::ErrorKind::NotFound.into(), + IOError::PermissionDenied => bitcoin::io::ErrorKind::PermissionDenied.into(), + IOError::ConnectionRefused => bitcoin::io::ErrorKind::ConnectionRefused.into(), + IOError::ConnectionReset => bitcoin::io::ErrorKind::ConnectionReset.into(), + IOError::ConnectionAborted => bitcoin::io::ErrorKind::ConnectionAborted.into(), + IOError::NotConnected => bitcoin::io::ErrorKind::NotConnected.into(), + IOError::AddrInUse => bitcoin::io::ErrorKind::AddrInUse.into(), + IOError::AddrNotAvailable => bitcoin::io::ErrorKind::AddrNotAvailable.into(), + IOError::BrokenPipe => bitcoin::io::ErrorKind::BrokenPipe.into(), + IOError::AlreadyExists => bitcoin::io::ErrorKind::AlreadyExists.into(), + IOError::WouldBlock => bitcoin::io::ErrorKind::WouldBlock.into(), + IOError::InvalidInput => bitcoin::io::ErrorKind::InvalidInput.into(), + IOError::InvalidData => bitcoin::io::ErrorKind::InvalidData.into(), + IOError::TimedOut => bitcoin::io::ErrorKind::TimedOut.into(), + IOError::WriteZero => bitcoin::io::ErrorKind::WriteZero.into(), + IOError::Interrupted => bitcoin::io::ErrorKind::Interrupted.into(), + IOError::UnexpectedEof => bitcoin::io::ErrorKind::UnexpectedEof.into(), + IOError::Other => bitcoin::io::ErrorKind::Other.into(), + } + } +} + +impl std::fmt::Display for IOError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +/// One fully-qualified key returned by [`DynStoreTrait::list_all_keys`]. +#[derive(Clone, Debug, PartialEq, Eq, uniffi::Record)] +pub struct KVStoreKey { + pub primary_namespace: String, + pub secondary_namespace: String, + pub key: String, +} + +/// A page of keys returned by [`DynStoreTrait::list_paginated`]. +#[derive(Clone, Debug, PartialEq, Eq, uniffi::Record)] +pub struct PaginatedListResponse { + pub keys: Vec, + pub next_page_token: Option>, +} + +/// An asynchronous key-value store implemented by a foreign-language caller. +/// +/// Implementations must support namespace listing, paginated listing, and exhaustive key +/// enumeration in addition to ordinary key-value operations. +#[async_trait::async_trait] +pub trait DynStoreTrait: Send + Sync { + /// Returns the data stored for the given `primary_namespace`, `secondary_namespace`, and + /// `key`. + /// + /// Returns [`IOError::NotFound`] if the key does not exist in the given namespaces. + async fn read( + &self, primary_namespace: String, secondary_namespace: String, key: String, + ) -> Result, IOError>; + + /// Persists the given data under the given `key`. + /// + /// Creates the given `primary_namespace` and `secondary_namespace` if they do not already + /// exist. + async fn write( + &self, primary_namespace: String, secondary_namespace: String, key: String, buf: Vec, + ) -> Result<(), IOError>; + + /// Removes any data previously persisted under the given `key`. + /// + /// If `lazy` is `true`, the implementation may defer the removal, for example to batch several + /// removals. Lazy removals may be lost if the process crashes before they are persisted and + /// should therefore only be used when the removal can safely be replayed. + /// + /// Returns successfully if the key is absent, whether or not it existed before this call. + async fn remove( + &self, primary_namespace: String, secondary_namespace: String, key: String, lazy: bool, + ) -> Result<(), IOError>; + + /// Returns the keys stored under the given `secondary_namespace` in `primary_namespace`. + /// + /// Keys may be returned in any order. Returns an empty list if either namespace is unknown. + async fn list( + &self, primary_namespace: String, secondary_namespace: String, + ) -> Result, IOError>; + + /// Returns one page of keys from the requested namespace, ordered from most recently created + /// to least recently created. + /// + /// If `page_token` is provided, listing continues from where the previous page ended. If it is + /// absent, listing begins with the most recently created key. The `next_page_token` in the + /// response can be passed to a subsequent call to retrieve the next page. + /// + /// Page tokens are scoped to a particular `(primary_namespace, secondary_namespace)` pair. If + /// the key referenced by a token has been removed, listing should continue from the next valid + /// position rather than fail. + /// + /// Returns an empty page if either namespace is unknown or no more keys remain. + async fn list_paginated( + &self, primary_namespace: String, secondary_namespace: String, + page_token: Option>, + ) -> Result; + + /// Returns every key in the store together with its namespaces. + /// + /// Exhaustive key enumeration is required for storage migrations. This includes backfilling a + /// newly configured backup and resilvering a backup that became stale while the node operated + /// without it. + /// + /// The result must include every key known to the store so the destination is not left + /// incomplete, but keys may be returned in any order. + async fn list_all_keys(&self) -> Result, IOError>; +} + +/// Adapts a foreign [`DynStoreTrait`] implementation to ldk-node's internal store interface. +#[cfg(feature = "storage-tier")] +pub(crate) struct DynStore { + inner: Arc, +} + +#[cfg(feature = "storage-tier")] +impl DynStore { + pub(crate) fn new(inner: Arc) -> Self { + Self { inner } + } +} + +#[cfg(feature = "storage-tier")] +impl crate::types::DynStoreTrait for DynStore { + fn read_async( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> Pin, bitcoin::io::Error>> + Send + 'static>> { + let inner = Arc::clone(&self.inner); + let primary_namespace = primary_namespace.to_owned(); + let secondary_namespace = secondary_namespace.to_owned(); + let key = key.to_owned(); + Box::pin(async move { + check_namespace_key_validity( + &primary_namespace, + &secondary_namespace, + Some(&key), + "read", + )?; + inner.read(primary_namespace, secondary_namespace, key).await.map_err(Into::into) + }) + } + + fn write_async( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, + ) -> Pin> + Send + 'static>> { + let inner = Arc::clone(&self.inner); + let primary_namespace = primary_namespace.to_owned(); + let secondary_namespace = secondary_namespace.to_owned(); + let key = key.to_owned(); + Box::pin(async move { + check_namespace_key_validity( + &primary_namespace, + &secondary_namespace, + Some(&key), + "write", + )?; + inner.write(primary_namespace, secondary_namespace, key, buf).await.map_err(Into::into) + }) + } + + fn remove_async( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, + ) -> Pin> + Send + 'static>> { + let inner = Arc::clone(&self.inner); + let primary_namespace = primary_namespace.to_owned(); + let secondary_namespace = secondary_namespace.to_owned(); + let key = key.to_owned(); + Box::pin(async move { + check_namespace_key_validity( + &primary_namespace, + &secondary_namespace, + Some(&key), + "remove", + )?; + inner + .remove(primary_namespace, secondary_namespace, key, lazy) + .await + .map_err(Into::into) + }) + } + + fn list_async( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> Pin, bitcoin::io::Error>> + Send + 'static>> { + let inner = Arc::clone(&self.inner); + let primary_namespace = primary_namespace.to_owned(); + let secondary_namespace = secondary_namespace.to_owned(); + Box::pin(async move { + check_namespace_key_validity(&primary_namespace, &secondary_namespace, None, "list")?; + inner.list(primary_namespace, secondary_namespace).await.map_err(Into::into) + }) + } + + fn list_paginated_async( + &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, + ) -> Pin< + Box< + dyn Future> + + Send + + 'static, + >, + > { + let inner = Arc::clone(&self.inner); + let primary_namespace = primary_namespace.to_owned(); + let secondary_namespace = secondary_namespace.to_owned(); + let page_token = page_token.map(|token| Arc::new(token.into())); + Box::pin(async move { + check_namespace_key_validity(&primary_namespace, &secondary_namespace, None, "list")?; + let response = inner + .list_paginated(primary_namespace, secondary_namespace, page_token) + .await + .map_err(bitcoin::io::Error::from)?; + Ok(LdkPaginatedListResponse { + keys: response.keys, + next_page_token: response + .next_page_token + .map(|token| token.as_ref().as_ref().clone()), + }) + }) + } + + fn list_all_keys_async( + &self, + ) -> Pin< + Box< + dyn Future, bitcoin::io::Error>> + + Send + + 'static, + >, + > { + let inner = Arc::clone(&self.inner); + Box::pin(async move { + inner.list_all_keys().await.map_err(Into::into).map(|keys| { + keys.into_iter() + .map(|key| (key.primary_namespace, key.secondary_namespace, key.key)) + .collect() + }) + }) + } +} + +#[async_trait::async_trait] +impl DynStoreTrait for T +where + T: PaginatedKVStore + MigratableKVStore + Send + Sync + 'static, +{ + async fn read( + &self, primary_namespace: String, secondary_namespace: String, key: String, + ) -> Result, IOError> { + KVStore::read(self, &primary_namespace, &secondary_namespace, &key) + .await + .map_err(Into::into) + } + + async fn write( + &self, primary_namespace: String, secondary_namespace: String, key: String, buf: Vec, + ) -> Result<(), IOError> { + KVStore::write(self, &primary_namespace, &secondary_namespace, &key, buf) + .await + .map_err(Into::into) + } + + async fn remove( + &self, primary_namespace: String, secondary_namespace: String, key: String, lazy: bool, + ) -> Result<(), IOError> { + KVStore::remove(self, &primary_namespace, &secondary_namespace, &key, lazy) + .await + .map_err(Into::into) + } + + async fn list( + &self, primary_namespace: String, secondary_namespace: String, + ) -> Result, IOError> { + KVStore::list(self, &primary_namespace, &secondary_namespace).await.map_err(Into::into) + } + + async fn list_paginated( + &self, primary_namespace: String, secondary_namespace: String, + page_token: Option>, + ) -> Result { + let page_token = page_token.map(|token| token.as_ref().as_ref().clone()); + PaginatedKVStore::list_paginated(self, &primary_namespace, &secondary_namespace, page_token) + .await + .map(|response| PaginatedListResponse { + keys: response.keys, + next_page_token: response.next_page_token.map(|token| Arc::new(token.into())), + }) + .map_err(Into::into) + } + + async fn list_all_keys(&self) -> Result, IOError> { + MigratableKVStore::list_all_keys(self) + .await + .map(|keys| { + keys.into_iter() + .map(|(primary_namespace, secondary_namespace, key)| KVStoreKey { + primary_namespace, + secondary_namespace, + key, + }) + .collect() + }) + .map_err(Into::into) + } +} diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index 32464d0445..627667175e 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -4,6 +4,10 @@ // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license , at your option. You may not use this file except in +#[cfg(feature = "uniffi")] +mod io; +#[cfg(feature = "uniffi")] +pub use io::*; #[cfg(feature = "uniffi")] mod types; diff --git a/src/io/mod.rs b/src/io/mod.rs index c11475c431..58e2efc853 100644 --- a/src/io/mod.rs +++ b/src/io/mod.rs @@ -15,6 +15,8 @@ pub mod postgres_store; pub mod sqlite_store; #[cfg(test)] pub(crate) mod test_utils; +#[cfg(feature = "storage-tier")] +pub(crate) mod tier_store; pub(crate) mod utils; #[cfg(feature = "storage-vss")] pub mod vss_store; diff --git a/src/io/sqlite_store/mod.rs b/src/io/sqlite_store/mod.rs index 2587220598..f469f593a0 100644 --- a/src/io/sqlite_store/mod.rs +++ b/src/io/sqlite_store/mod.rs @@ -12,12 +12,14 @@ use std::future::Future; use std::path::PathBuf; use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; +use std::time::Duration; use lightning::io; use lightning::util::persist::{ KVStore, MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse, }; use lightning_types::string::PrintableString; +use rusqlite::ffi::ErrorCode; use rusqlite::{named_params, Connection}; use crate::io::utils::check_namespace_key_validity; @@ -26,6 +28,15 @@ mod migrations; /// LDK Node's database file name. pub const SQLITE_DB_FILE_NAME: &str = "ldk_node_data.sqlite"; +/// LDK Node's internal tier-store index database file name. +#[cfg(feature = "storage-tier")] +pub(crate) const SQLITE_TIER_INDEX_DB_FILE_NAME: &str = "ldk_node_tier_index.sqlite"; +/// LDK Node's backup database file name. +#[cfg(feature = "storage-tier")] +pub const SQLITE_BACKUP_DB_FILE_NAME: &str = "ldk_node_data_backup.sqlite"; +/// LDK Node's ephemeral database file name. +#[cfg(feature = "storage-tier")] +pub const SQLITE_EPHEMERAL_DB_FILE_NAME: &str = "ldk_node_data_ephemeral.sqlite"; /// LDK Node's table in which we store all data. pub const KV_TABLE_NAME: &str = "ldk_node_data"; @@ -41,6 +52,17 @@ const SCHEMA_USER_VERSION: u16 = 3; // The number of entries returned per page in paginated list operations. const PAGE_SIZE: usize = 50; +fn exclusive_lock_error_kind(error: &rusqlite::Error) -> io::ErrorKind { + match error { + rusqlite::Error::SqliteFailure(error, _) + if matches!(error.code, ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked) => + { + io::ErrorKind::AlreadyExists + }, + _ => io::ErrorKind::Other, + } +} + /// A [`KVStore`] implementation that writes to and reads from an [SQLite] database. /// /// [SQLite]: https://sqlite.org @@ -62,7 +84,23 @@ impl SqliteStore { pub fn new( data_dir: PathBuf, db_file_name: Option, kv_table_name: Option, ) -> io::Result { - let inner = Arc::new(SqliteStoreInner::new(data_dir, db_file_name, kv_table_name)?); + Self::new_internal(data_dir, db_file_name, kv_table_name, false) + } + + /// Constructs a new [`SqliteStore`] which exclusively owns its database for its lifetime. + #[cfg(feature = "storage-tier")] + pub(crate) fn new_exclusive( + data_dir: PathBuf, db_file_name: Option, kv_table_name: Option, + ) -> io::Result { + Self::new_internal(data_dir, db_file_name, kv_table_name, true) + } + + fn new_internal( + data_dir: PathBuf, db_file_name: Option, kv_table_name: Option, + exclusive: bool, + ) -> io::Result { + let inner = + Arc::new(SqliteStoreInner::new(data_dir, db_file_name, kv_table_name, exclusive)?); let next_write_version = AtomicU64::new(1); Ok(Self { inner, next_write_version }) @@ -230,6 +268,7 @@ struct SqliteStoreInner { impl SqliteStoreInner { fn new( data_dir: PathBuf, db_file_name: Option, kv_table_name: Option, + exclusive: bool, ) -> io::Result { let db_file_name = db_file_name.unwrap_or(DEFAULT_SQLITE_DB_FILE_NAME.to_string()); let kv_table_name = kv_table_name.unwrap_or(DEFAULT_KV_TABLE_NAME.to_string()); @@ -251,6 +290,33 @@ impl SqliteStoreInner { io::Error::new(io::ErrorKind::Other, msg) })?; + if exclusive { + connection.busy_timeout(Duration::ZERO).map_err(|e| { + let msg = format!( + "Failed to configure exclusive database lock timeout for {}: {}", + db_file_path.display(), + e + ); + io::Error::new(io::ErrorKind::Other, msg) + })?; + connection.pragma_update(None, "locking_mode", "EXCLUSIVE").map_err(|e| { + let msg = format!( + "Failed to exclusively lock database file {}: {}", + db_file_path.display(), + e + ); + io::Error::new(io::ErrorKind::Other, msg) + })?; + connection.execute_batch("BEGIN EXCLUSIVE; COMMIT;").map_err(|e| { + let msg = format!( + "Failed to exclusively lock database file {}: {}", + db_file_path.display(), + e + ); + io::Error::new(exclusive_lock_error_kind(&e), msg) + })?; + } + let sql = format!("SELECT user_version FROM pragma_user_version"); let version_res: u16 = connection.query_row(&sql, [], |row| row.get(0)).map_err(|e| { let msg = format!("Failed to read PRAGMA user_version: {}", e); @@ -700,6 +766,21 @@ mod tests { } } + #[test] + fn exclusive_lock_error_kind_distinguishes_contention_from_other_failures() { + for result_code in [rusqlite::ffi::SQLITE_BUSY, rusqlite::ffi::SQLITE_LOCKED] { + let error = + rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(result_code), None); + assert_eq!(exclusive_lock_error_kind(&error), io::ErrorKind::AlreadyExists); + } + + let io_error = rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_IOERR), + None, + ); + assert_eq!(exclusive_lock_error_kind(&io_error), io::ErrorKind::Other); + } + #[tokio::test] async fn read_write_remove_list_persist() { let mut temp_path = random_storage_path(); diff --git a/src/io/tier_store.rs b/src/io/tier_store.rs new file mode 100644 index 0000000000..22bdea94bb --- /dev/null +++ b/src/io/tier_store.rs @@ -0,0 +1,4503 @@ +// This file is Copyright its original authors, visible in version control history. +// +// This file is licensed under the Apache License, Version 2.0 or the MIT license , at your option. You may not use this file except in +// accordance with one or both of these licenses. + +use std::collections::{HashMap, HashSet}; +use std::future::Future; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +use bitcoin::hashes::{sha256, Hash, HashEngine}; +use bitcoin::hex::{DisplayHex, FromHex}; +use lightning::util::persist::{ + KVStore, MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse, + NETWORK_GRAPH_PERSISTENCE_KEY, NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_KEY, SCORER_PERSISTENCE_PRIMARY_NAMESPACE, +}; +use lightning::util::ser::{Readable, Writeable}; +use lightning::{impl_writeable_tlv_based, impl_writeable_tlv_based_enum, io, log_error}; +use tokio::sync::Mutex as TokioMutex; + +use crate::io::sqlite_store::{SqliteStore, KV_TABLE_NAME, SQLITE_TIER_INDEX_DB_FILE_NAME}; +use crate::io::utils::{check_namespace_key_validity, EXTERNAL_PATHFINDING_SCORES_CACHE_KEY}; +use crate::logger::{LdkLogger, Logger}; +use crate::types::{DynStore, DynStoreWrapper}; + +const INDEX_DATABASE_ID_LEN: usize = 16; +const INDEX_ENTRIES_PRIMARY_NAMESPACE: &str = "_tier_store_entries"; +const INDEX_JOURNAL_PRIMARY_NAMESPACE: &str = "_tier_store_journal"; +const INDEX_METADATA_PRIMARY_NAMESPACE: &str = "_tier_store_metadata"; +const INDEX_DATABASE_ID_KEY: &str = "index_database_id"; +const INDEX_NAMESPACE_READY_KEY_PREFIX: &str = "ready_"; +const INDEX_CACHE_READY_KEY_PREFIX: &str = "cache_ready_"; +const INDEX_ENTRY_VALUE: &[u8] = &[1]; + +const PAGE_TOKEN_FORMAT_VERSION: u8 = 1; + +const GENERATION_ID_LEN: usize = 16; + +const PRIMARY_SYNC_GENERATION_KEY: &str = "primary_generation"; + +const BACKUP_SYNC_PRIMARY_NAMESPACE: &str = "_tier_store_backup_sync"; +const BACKUP_SYNC_COMPLETED_GENERATION_KEY: &str = "completed_generation"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct BackupSyncCompletion { + /// The primary generation whose durable values were copied. + primary_generation_id: [u8; GENERATION_ID_LEN], + /// The index database whose journal had been fully recovered before copying. + index_database_id: [u8; INDEX_DATABASE_ID_LEN], +} + +impl BackupSyncCompletion { + /// Constructs the completion expected for the currently configured primary and index stores. + fn new( + primary_generation_id: [u8; GENERATION_ID_LEN], + index_database_id: [u8; INDEX_DATABASE_ID_LEN], + ) -> Self { + Self { primary_generation_id, index_database_id } + } + + /// Encodes both fixed-width identities into the value persisted in the backup store. + fn encode(self) -> Vec { + let mut encoded = Vec::with_capacity(GENERATION_ID_LEN + INDEX_DATABASE_ID_LEN); + encoded.extend_from_slice(&self.primary_generation_id); + encoded.extend_from_slice(&self.index_database_id); + encoded + } + + /// Decodes and validates the completion value read from the backup store. + fn decode(encoded: Vec) -> io::Result { + if encoded.len() != GENERATION_ID_LEN + INDEX_DATABASE_ID_LEN { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Invalid tier-store backup synchronization completion", + )); + } + let mut primary_generation_id = [0; GENERATION_ID_LEN]; + primary_generation_id.copy_from_slice(&encoded[..GENERATION_ID_LEN]); + let mut index_database_id = [0; INDEX_DATABASE_ID_LEN]; + index_database_id.copy_from_slice(&encoded[GENERATION_ID_LEN..]); + Ok(Self { primary_generation_id, index_database_id }) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum BackupSyncStatus { + NotConfigured, + Synchronized, + Required, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ValueTier { + Primary, + Ephemeral, +} + +impl_writeable_tlv_based_enum!(ValueTier, + (0, Primary) => {}, + (2, Ephemeral) => {}, +); + +#[derive(Debug, PartialEq, Eq)] +enum JournalOperation { + Create { + /// The complete intended value, retained locally so recovery does not depend on which remote + /// or local value-store write completed before interruption. + value: Vec, + }, + Update { + /// The complete intended value, retained locally so an interrupted dual-store update can be + /// retried against the configured stores or completed against primary-only storage. + value: Vec, + }, + Remove { + lazy: bool, + }, +} + +impl_writeable_tlv_based_enum!(JournalOperation, + (0, Create) => { + (0, value, required), + }, + (2, Remove) => { + (0, lazy, required), + }, + (4, Update) => { + (0, value, required), + }, +); + +#[derive(Debug, PartialEq, Eq)] +struct JournalEntry { + primary_namespace: String, + secondary_namespace: String, + key: String, + tier: ValueTier, + requires_backup: bool, + operation: JournalOperation, +} + +impl_writeable_tlv_based!(JournalEntry, { + (0, primary_namespace, required), + (2, secondary_namespace, required), + (4, key, required), + (6, tier, required), + (8, requires_backup, required), + (10, operation, required), +}); + +impl JournalEntry { + /// Encodes a pending operation for persistence in the local index database. + fn serialize(&self) -> Vec { + Writeable::encode(self) + } + + /// Decodes and validates a pending operation from the local index database. + fn deserialize(encoded: &[u8]) -> io::Result { + Readable::read(&mut &*encoded) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "Invalid journal entry")) + } +} + +struct TierStorePageToken { + format_version: u8, + index_database_id: Vec, + namespace_id: String, + index_page_token: String, +} + +impl_writeable_tlv_based!(TierStorePageToken, { + (0, format_version, required), + (2, index_database_id, required), + (4, namespace_id, required), + (6, index_page_token, required), +}); + +impl TierStorePageToken { + /// Wraps an index-store page token with the context required to validate its later use. + fn encode( + index_database_id: &[u8; INDEX_DATABASE_ID_LEN], namespace_id: String, + index_page_token: PageToken, + ) -> PageToken { + let token = Self { + format_version: PAGE_TOKEN_FORMAT_VERSION, + index_database_id: index_database_id.to_vec(), + namespace_id, + index_page_token: index_page_token.to_string(), + }; + PageToken::new(Writeable::encode(&token).to_lower_hex_string()) + } + + /// Decodes a TierStore token and rejects tokens issued for another index or namespace. + fn decode( + token: PageToken, expected_database_id: &[u8; INDEX_DATABASE_ID_LEN], + expected_namespace_id: &str, + ) -> io::Result { + let encoded = Vec::from_hex(token.as_str()).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "Invalid TierStore page token") + })?; + let mut reader = &*encoded; + let token: Self = Readable::read(&mut reader).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "Invalid TierStore page token") + })?; + if !reader.is_empty() + || token.format_version != PAGE_TOKEN_FORMAT_VERSION + || token.index_database_id != expected_database_id + || token.namespace_id != expected_namespace_id + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "TierStore page token does not belong to this index namespace", + )); + } + Ok(PageToken::new(token.index_page_token)) + } +} + +pub(crate) struct TierStoreIndex { + // Holding the store keeps its exclusive SQLite lock for the lifetime of the tier store. + store: Arc, + database_id: [u8; INDEX_DATABASE_ID_LEN], +} + +impl TierStoreIndex { + /// Opens the internal SQLite index and ensures that it has a persistent database identity. + async fn new(data_dir: PathBuf) -> io::Result { + let store = SqliteStore::new_exclusive( + data_dir, + Some(SQLITE_TIER_INDEX_DB_FILE_NAME.to_string()), + Some(KV_TABLE_NAME.to_string()), + )?; + let store: Arc = Arc::new(DynStoreWrapper(store)); + let database_id = Self::read_or_create_database_id(store.as_ref()).await?; + Ok(Self { store, database_id }) + } + + /// Constructs an index over the supplied store for tests that do not need SQLite persistence. + #[cfg(test)] + fn from_store(store: Arc) -> Self { + Self { store, database_id: [1; INDEX_DATABASE_ID_LEN] } + } + + /// Constructs a test index with a specified database identity. + #[cfg(test)] + fn from_store_with_database_id( + store: Arc, database_id: [u8; INDEX_DATABASE_ID_LEN], + ) -> Self { + Self { store, database_id } + } + + /// Returns the persistent identity of this index database. + fn database_id(&self) -> [u8; INDEX_DATABASE_ID_LEN] { + self.database_id + } + + /// Derives the internal secondary namespace for a logical namespace pair. + /// + /// Length-prefixing keeps distinct pairs from producing the same hash input. The original pair + /// is also stored as namespace metadata so that a hash collision can be detected. + fn namespace_id(primary_namespace: &str, secondary_namespace: &str) -> String { + let mut engine = sha256::Hash::engine(); + engine.input(&(primary_namespace.len() as u64).to_be_bytes()); + engine.input(primary_namespace.as_bytes()); + engine.input(&(secondary_namespace.len() as u64).to_be_bytes()); + engine.input(secondary_namespace.as_bytes()); + sha256::Hash::from_engine(engine).to_string() + } + + /// Derives the metadata key that records whether a logical namespace is index-backed. + fn namespace_ready_key(primary_namespace: &str, secondary_namespace: &str) -> String { + format!( + "{}{}", + INDEX_NAMESPACE_READY_KEY_PREFIX, + Self::namespace_id(primary_namespace, secondary_namespace) + ) + } + + /// Derives the internal identity for one logical cache key. + fn cache_id(primary_namespace: &str, secondary_namespace: &str, key: &str) -> String { + let mut engine = sha256::Hash::engine(); + engine.input(&(primary_namespace.len() as u64).to_be_bytes()); + engine.input(primary_namespace.as_bytes()); + engine.input(&(secondary_namespace.len() as u64).to_be_bytes()); + engine.input(secondary_namespace.as_bytes()); + engine.input(&(key.len() as u64).to_be_bytes()); + engine.input(key.as_bytes()); + sha256::Hash::from_engine(engine).to_string() + } + + /// Derives the metadata key recording that one indexed cache value occupies the ephemeral tier. + fn cache_ready_key(primary_namespace: &str, secondary_namespace: &str, key: &str) -> String { + format!( + "{}{}", + INDEX_CACHE_READY_KEY_PREFIX, + Self::cache_id(primary_namespace, secondary_namespace, key) + ) + } + + /// Encodes the original logical namespace pair for collision detection. + fn namespace_metadata(primary_namespace: &str, secondary_namespace: &str) -> Vec { + // The fixed five-byte overhead is one format-version byte plus two big-endian u16 + // namespace-length prefixes: 1 + 2 + 2 = 5. + let mut metadata = + Vec::with_capacity(5 + primary_namespace.len() + secondary_namespace.len()); + metadata.push(1); + metadata.extend_from_slice(&(primary_namespace.len() as u16).to_be_bytes()); + metadata.extend_from_slice(primary_namespace.as_bytes()); + metadata.extend_from_slice(&(secondary_namespace.len() as u16).to_be_bytes()); + metadata.extend_from_slice(secondary_namespace.as_bytes()); + metadata + } + + /// Encodes the original logical cache-key identity for collision detection. + fn cache_metadata(primary_namespace: &str, secondary_namespace: &str, key: &str) -> Vec { + // The fixed seven-byte overhead is one format-version byte plus three big-endian u16 + // component-length prefixes: 1 + 2 + 2 + 2 = 7. + let mut metadata = + Vec::with_capacity(7 + primary_namespace.len() + secondary_namespace.len() + key.len()); + metadata.push(1); + metadata.extend_from_slice(&(primary_namespace.len() as u16).to_be_bytes()); + metadata.extend_from_slice(primary_namespace.as_bytes()); + metadata.extend_from_slice(&(secondary_namespace.len() as u16).to_be_bytes()); + metadata.extend_from_slice(secondary_namespace.as_bytes()); + metadata.extend_from_slice(&(key.len() as u16).to_be_bytes()); + metadata.extend_from_slice(key.as_bytes()); + metadata + } + + /// Returns whether the namespace's index is authoritative for listing. + /// + /// Returns an error if the stored namespace metadata does not match the requested namespace, + /// which indicates either corrupt metadata or a namespace-ID collision. + async fn is_namespace_ready( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result { + self.is_marker_set( + &Self::namespace_ready_key(primary_namespace, secondary_namespace), + Self::namespace_metadata(primary_namespace, secondary_namespace), + ) + .await + } + + /// Returns whether one indexed cache value has been reconciled into ephemeral storage. + async fn is_cache_ready( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result { + self.is_marker_set( + &Self::cache_ready_key(primary_namespace, secondary_namespace, key), + Self::cache_metadata(primary_namespace, secondary_namespace, key), + ) + .await + } + + async fn is_marker_set( + &self, marker_key: &str, expected_metadata: Vec, + ) -> io::Result { + match KVStore::read(self.store.as_ref(), INDEX_METADATA_PRIMARY_NAMESPACE, "", marker_key) + .await + { + Ok(metadata) if metadata == expected_metadata => Ok(true), + Ok(_) => { + Err(io::Error::new(io::ErrorKind::InvalidData, "Tier-store index marker collision")) + }, + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(e), + } + } + + /// Marks the namespace's index as authoritative by persisting its original namespace pair. + async fn mark_namespace_ready( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result<()> { + self.set_marker( + &Self::namespace_ready_key(primary_namespace, secondary_namespace), + Self::namespace_metadata(primary_namespace, secondary_namespace), + ) + .await + } + + /// Marks one indexed cache value as reconciled into ephemeral storage. + async fn mark_cache_ready( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result<()> { + self.set_marker( + &Self::cache_ready_key(primary_namespace, secondary_namespace, key), + Self::cache_metadata(primary_namespace, secondary_namespace, key), + ) + .await + } + + async fn set_marker(&self, marker_key: &str, metadata: Vec) -> io::Result<()> { + KVStore::write( + self.store.as_ref(), + INDEX_METADATA_PRIMARY_NAMESPACE, + "", + marker_key, + metadata, + ) + .await + } + + /// Adds a logical key to the namespace's ordered listing index. + /// + /// Rewriting an existing key preserves its original index-store creation order. + async fn write_entry( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result<()> { + KVStore::write( + self.store.as_ref(), + INDEX_ENTRIES_PRIMARY_NAMESPACE, + &Self::namespace_id(primary_namespace, secondary_namespace), + key, + INDEX_ENTRY_VALUE.to_vec(), + ) + .await + } + + /// Returns whether the namespace's listing index contains the logical key. + async fn contains_entry( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result { + match KVStore::read( + self.store.as_ref(), + INDEX_ENTRIES_PRIMARY_NAMESPACE, + &Self::namespace_id(primary_namespace, secondary_namespace), + key, + ) + .await + { + Ok(value) if value == INDEX_ENTRY_VALUE => Ok(true), + Ok(_) => Err(io::Error::new(io::ErrorKind::InvalidData, "Invalid index entry")), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(e), + } + } + + /// Removes a logical key from the namespace's listing index. + async fn remove_entry( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, + ) -> io::Result<()> { + KVStore::remove( + self.store.as_ref(), + INDEX_ENTRIES_PRIMARY_NAMESPACE, + &Self::namespace_id(primary_namespace, secondary_namespace), + key, + lazy, + ) + .await + } + + /// Lists all logical keys recorded in the namespace's index. + async fn list( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result> { + KVStore::list( + self.store.as_ref(), + INDEX_ENTRIES_PRIMARY_NAMESPACE, + &Self::namespace_id(primary_namespace, secondary_namespace), + ) + .await + } + + /// Lists logical keys in the index store's creation order using a namespace-bound token. + async fn list_paginated( + &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, + ) -> io::Result { + let namespace_id = Self::namespace_id(primary_namespace, secondary_namespace); + let index_page_token = page_token + .map(|token| TierStorePageToken::decode(token, &self.database_id, &namespace_id)) + .transpose()?; + let response = PaginatedKVStore::list_paginated( + self.store.as_ref(), + INDEX_ENTRIES_PRIMARY_NAMESPACE, + &namespace_id, + index_page_token, + ) + .await?; + let next_page_token = response + .next_page_token + .map(|token| TierStorePageToken::encode(&self.database_id, namespace_id, token)); + Ok(PaginatedListResponse { keys: response.keys, next_page_token }) + } + + /// Persists a pending operation before its value-store effects begin. + async fn write_journal_entry(&self, entry: &JournalEntry) -> io::Result<()> { + KVStore::write( + self.store.as_ref(), + INDEX_JOURNAL_PRIMARY_NAMESPACE, + &Self::namespace_id(&entry.primary_namespace, &entry.secondary_namespace), + &entry.key, + entry.serialize(), + ) + .await + } + + /// Reads a pending operation for a logical key. + async fn read_journal_entry( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result { + let encoded = KVStore::read( + self.store.as_ref(), + INDEX_JOURNAL_PRIMARY_NAMESPACE, + &Self::namespace_id(primary_namespace, secondary_namespace), + key, + ) + .await?; + let entry = JournalEntry::deserialize(&encoded)?; + if entry.primary_namespace != primary_namespace + || entry.secondary_namespace != secondary_namespace + || entry.key != key + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Journal entry identity does not match its location", + )); + } + Ok(entry) + } + + /// Lists pending-operation keys from oldest to newest journal insertion. + async fn list_journal_entries( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result> { + let secondary_namespace = Self::namespace_id(primary_namespace, secondary_namespace); + let mut keys = Vec::new(); + let mut page_token = None; + loop { + let page = PaginatedKVStore::list_paginated( + self.store.as_ref(), + INDEX_JOURNAL_PRIMARY_NAMESPACE, + &secondary_namespace, + page_token, + ) + .await?; + keys.extend(page.keys); + match page.next_page_token { + Some(next_page_token) => page_token = Some(next_page_token), + None => break, + } + } + keys.reverse(); + Ok(keys) + } + + /// Returns every pending operation stored in the index, validating its encoded location. + async fn list_all_journal_entries(&self) -> io::Result> { + let mut entries = Vec::new(); + for (primary_namespace, secondary_namespace, key) in + MigratableKVStore::list_all_keys(self.store.as_ref()).await? + { + if primary_namespace != INDEX_JOURNAL_PRIMARY_NAMESPACE { + continue; + } + let encoded = + KVStore::read(self.store.as_ref(), &primary_namespace, &secondary_namespace, &key) + .await?; + let entry = JournalEntry::deserialize(&encoded)?; + if TierStoreIndex::namespace_id(&entry.primary_namespace, &entry.secondary_namespace) + != secondary_namespace + || entry.key != key + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Journal entry identity does not match its location", + )); + } + entries.push(entry); + } + Ok(entries) + } + + /// Clears a pending operation after all of its intended effects are durable. + async fn remove_journal_entry( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result<()> { + KVStore::remove( + self.store.as_ref(), + INDEX_JOURNAL_PRIMARY_NAMESPACE, + &Self::namespace_id(primary_namespace, secondary_namespace), + key, + false, + ) + .await + } + + /// Reads the index database identity, creating and persisting one when it is absent. + async fn read_or_create_database_id( + store: &DynStore, + ) -> io::Result<[u8; INDEX_DATABASE_ID_LEN]> { + match KVStore::read(store, INDEX_METADATA_PRIMARY_NAMESPACE, "", INDEX_DATABASE_ID_KEY) + .await + { + Ok(bytes) => bytes.try_into().map_err(|_| { + io::Error::new(io::ErrorKind::InvalidData, "Invalid tier-store index database ID") + }), + Err(e) if e.kind() == io::ErrorKind::NotFound => { + let mut database_id = [0; INDEX_DATABASE_ID_LEN]; + getrandom::fill(&mut database_id).map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("Failed to generate tier-store index database ID: {e}"), + ) + })?; + KVStore::write( + store, + INDEX_METADATA_PRIMARY_NAMESPACE, + "", + INDEX_DATABASE_ID_KEY, + database_id.to_vec(), + ) + .await?; + Ok(database_id) + }, + Err(e) => Err(e), + } + } +} + +/// A 3-tiered [`KVStore`] implementation that routes data across +/// storage backends that may be local or remote: +/// - a primary store for durable, authoritative persistence, +/// - an optional backup store that maintains an additional durable copy of +/// primary-backed data, and +/// - an optional ephemeral store for non-critical, rebuildable cached data. +/// +/// When a backup store is configured, writes and removals for primary-backed data +/// are issued to the primary and backup stores concurrently and only succeed once +/// both stores complete successfully. +/// +/// Reads and lists do not consult the backup store during normal operation. +/// Ephemeral data is read from and written to the ephemeral store when configured. +/// Namespaces are indexed locally so unpaginated and paginated listings expose the +/// same logical contents in cross-tier creation order. Existing primary-store keys +/// are imported into the index before a namespace is first accessed. Cache values +/// imported from primary storage are then moved to ephemeral storage without changing +/// their index positions. +/// +/// Note that dual-store writes and removals are not atomic across the primary and +/// backup stores. If one store succeeds and the other fails, the operation +/// returns an error even though one store may already reflect the change. +pub(crate) struct TierStore { + inner: Arc, +} + +impl TierStore { + pub fn new(primary_store: Arc, logger: Arc) -> Self { + let inner = Arc::new(TierStoreInner::new(primary_store, Arc::clone(&logger))); + + Self { inner } + } + + /// Configures a backup store for primary-backed data. + /// + /// Once set, writes and removals targeting the primary tier succeed only if both + /// the primary and backup stores succeed. The two operations are issued + /// concurrently, and any failure is returned to the caller. + /// + /// Note: dual-store writes/removals are not atomic. An error may be returned + /// after the primary store has already been updated if the backup store fails. + /// + /// The backup store is not consulted for normal reads or lists. + pub fn set_backup_store(&mut self, backup: Arc) { + debug_assert_eq!(Arc::strong_count(&self.inner), 1); + + let inner = Arc::get_mut(&mut self.inner).expect( + "TierStore should not be shared during configuration. No other references should exist", + ); + + inner.backup_store = Some(backup); + } + + /// Configures the ephemeral store for non-critical, rebuildable data. + /// + /// When configured, selected cache-like data is routed to this store instead of + /// the primary store. + pub fn set_ephemeral_store(&mut self, ephemeral: Arc) { + debug_assert_eq!(Arc::strong_count(&self.inner), 1); + + let inner = Arc::get_mut(&mut self.inner).expect( + "TierStore should not be shared during configuration. No other references should exist", + ); + + inner.ephemeral_store = Some(ephemeral); + } + + pub(crate) fn set_index_store(&mut self, index: TierStoreIndex) { + debug_assert_eq!(Arc::strong_count(&self.inner), 1); + + let inner = Arc::get_mut(&mut self.inner).expect( + "TierStore should not be shared during configuration. No other references should exist", + ); + + inner.index = Some(index); + } + + /// Initializes the durable metadata used to determine whether the backup matches the primary. + /// + /// This must be called once after the optional backup store has been configured and before the + /// `TierStore` is used. If no backup is configured but a primary generation already exists, it + /// writes a new generation so any backup completed against the earlier generation will be + /// recognized as stale if it returns. If no generation exists, no backup has yet been tracked and + /// no metadata is created. If a backup is configured, it reads or creates the primary generation + /// and compares it and the current index database identity with the backup's completion record + /// without modifying that record. + /// + /// Returns [`BackupSyncStatus::NotConfigured`] when no backup is present, whether or not an + /// existing primary generation was rotated, [`BackupSyncStatus::Synchronized`] when the completion + /// matches both identities, or [`BackupSyncStatus::Required`] when the backup has no completion + /// record or records another generation or index. A `Required` result only classifies the backup; + /// it does not perform synchronization. + /// + /// Returns an error if a configured backup has no index, synchronization metadata cannot be read, + /// generated, or persisted, or stored metadata has an invalid length. + pub(crate) async fn initialize_backup_synchronization(&self) -> io::Result { + self.inner.initialize_backup_synchronization().await + } + + /// Makes a configured backup match the primary store before normal node operation begins. + /// + /// Pending journal entries are recovered first. Current primary values are then copied, stale + /// backup values are removed, and the backup's completion generation is updated last. The old + /// completion generation remains unchanged if any earlier step fails. + pub(crate) async fn synchronize_backup(&mut self) -> io::Result<()> { + let inner = Arc::get_mut(&mut self.inner).ok_or_else(|| { + io::Error::new( + io::ErrorKind::Other, + "TierStore must be exclusively owned during backup synchronization", + ) + })?; + inner.synchronize_backup().await + } +} + +pub(crate) async fn setup_index_store(data_dir: PathBuf) -> io::Result { + TierStoreIndex::new(data_dir).await +} + +impl KVStore for TierStore { + fn read( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> impl Future, io::Error>> + 'static + Send { + let inner = Arc::clone(&self.inner); + + let primary_namespace = primary_namespace.to_string(); + let secondary_namespace = secondary_namespace.to_string(); + let key = key.to_string(); + + async move { inner.read_internal(primary_namespace, secondary_namespace, key).await } + } + + fn write( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, + ) -> impl Future> + 'static + Send { + let inner = Arc::clone(&self.inner); + let locking_key = inner.build_locking_key(primary_namespace, secondary_namespace, key); + let (lock_ref, version) = inner.get_new_version_and_lock_ref(locking_key.clone()); + + let primary_namespace = primary_namespace.to_string(); + let secondary_namespace = secondary_namespace.to_string(); + let key = key.to_string(); + + async move { + inner + .write_internal( + primary_namespace, + secondary_namespace, + key, + buf, + lock_ref, + locking_key, + version, + ) + .await + } + } + + fn remove( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, + ) -> impl Future> + 'static + Send { + let inner = Arc::clone(&self.inner); + let locking_key = inner.build_locking_key(primary_namespace, secondary_namespace, key); + let (lock_ref, version) = inner.get_new_version_and_lock_ref(locking_key.clone()); + + let primary_namespace = primary_namespace.to_string(); + let secondary_namespace = secondary_namespace.to_string(); + let key = key.to_string(); + + async move { + inner + .remove_internal( + primary_namespace, + secondary_namespace, + key, + lazy, + lock_ref, + locking_key, + version, + ) + .await + } + } + + fn list( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> impl Future, io::Error>> + 'static + Send { + let inner = Arc::clone(&self.inner); + + let primary_namespace = primary_namespace.to_string(); + let secondary_namespace = secondary_namespace.to_string(); + + async move { inner.list_internal(primary_namespace, secondary_namespace).await } + } +} + +impl PaginatedKVStore for TierStore { + fn list_paginated( + &self, primary_namespace: &str, secondary_namespace: &str, page_token: Option, + ) -> impl Future> + 'static + Send { + let inner = Arc::clone(&self.inner); + + let primary_namespace = primary_namespace.to_string(); + let secondary_namespace = secondary_namespace.to_string(); + + async move { + inner.list_paginated_internal(primary_namespace, secondary_namespace, page_token).await + } + } +} + +impl MigratableKVStore for TierStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send { + let inner = Arc::clone(&self.inner); + async move { inner.list_all_keys_internal().await } + } +} + +struct TierStoreInner { + /// The authoritative store for durable data. + primary_store: Arc, + /// The store used for non-critical, rebuildable cached data. + ephemeral_store: Option>, + /// An optional second durable store for primary-backed data. + backup_store: Option>, + /// The local store used to index the logical contents across tiers. + index: Option, + /// The result of durable backup-synchronization initialization for this run. + backup_sync_status: Mutex>, + /// Per-namespace locks for serializing first-use index initialization. + index_initialization_locks: Mutex>>>, + /// Per-key locks for serializing primary+backup operations and skipping stale writes. + locks: Mutex>>>, + next_write_version: AtomicU64, + logger: Arc, +} + +impl TierStoreInner { + /// Creates a tier store with the primary data store. + pub fn new(primary_store: Arc, logger: Arc) -> Self { + Self { + primary_store, + ephemeral_store: None, + backup_store: None, + index: None, + backup_sync_status: Mutex::new(None), + index_initialization_locks: Mutex::new(HashMap::new()), + locks: Mutex::new(HashMap::new()), + next_write_version: AtomicU64::new(1), + logger, + } + } + + /// Rotates the primary generation or compares the backup's synchronization completion. + /// + /// Without a configured backup, this rotates an existing primary generation so primary-only + /// operation invalidates any earlier backup completion. It leaves primary metadata absent when no + /// backup has ever established a generation. With a configured backup, this compares the stored + /// completion with both the primary generation and the current index database identity. A missing + /// or different backup completion is classified as requiring synchronization; this method does not + /// perform that synchronization. + async fn initialize_backup_synchronization(&self) -> io::Result { + let status = if self.backup_store.is_none() { + match Self::read_generation_id(self.primary_store.as_ref(), PRIMARY_SYNC_GENERATION_KEY) + .await + { + Ok(_) => { + let generation_id = Self::generate_primary_generation_id()?; + self.write_primary_generation_id(generation_id).await?; + }, + Err(e) if e.kind() == io::ErrorKind::NotFound => {}, + Err(e) => return Err(e), + } + BackupSyncStatus::NotConfigured + } else { + let expected_completion = self.current_backup_sync_completion().await?; + match self.read_backup_sync_completion().await { + Ok(completion) if completion == expected_completion => { + BackupSyncStatus::Synchronized + }, + Ok(_) => BackupSyncStatus::Required, + Err(e) if e.kind() == io::ErrorKind::NotFound => BackupSyncStatus::Required, + Err(e) => return Err(e), + } + }; + *self.backup_sync_status.lock().expect("lock") = Some(status); + Ok(status) + } + + /// Generates an opaque random identity for one synchronization generation. + fn generate_primary_generation_id() -> io::Result<[u8; GENERATION_ID_LEN]> { + let mut generation_id = [0; GENERATION_ID_LEN]; + getrandom::fill(&mut generation_id).map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("Failed to generate tier-store backup synchronization generation: {e}"), + ) + })?; + Ok(generation_id) + } + + /// Reads the primary's current synchronization generation, creating and persisting one if absent. + /// + /// An existing generation is preserved so a matching backup completion remains valid across + /// restarts where the backup stays configured. + async fn read_or_create_primary_sync_generation_id( + &self, + ) -> io::Result<[u8; GENERATION_ID_LEN]> { + match Self::read_generation_id(self.primary_store.as_ref(), PRIMARY_SYNC_GENERATION_KEY) + .await + { + Ok(generation_id) => Ok(generation_id), + Err(e) if e.kind() == io::ErrorKind::NotFound => { + let generation_id = Self::generate_primary_generation_id()?; + self.write_primary_generation_id(generation_id).await?; + Ok(generation_id) + }, + Err(e) => Err(e), + } + } + + /// Returns the completion record a synchronized backup must contain for the current stores. + async fn current_backup_sync_completion(&self) -> io::Result { + let index_database_id = self + .index + .as_ref() + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::Other, + "TierStore index is required for backup synchronization", + ) + })? + .database_id(); + let primary_generation_id = self.read_or_create_primary_sync_generation_id().await?; + Ok(BackupSyncCompletion::new(primary_generation_id, index_database_id)) + } + + /// Persists the current synchronization generation directly in the authoritative primary store. + /// + /// The write intentionally bypasses backup replication: changing this record invalidates an old + /// backup, whose completion record must remain unchanged until synchronization actually finishes. + async fn write_primary_generation_id( + &self, generation_id: [u8; GENERATION_ID_LEN], + ) -> io::Result<()> { + KVStore::write( + self.primary_store.as_ref(), + BACKUP_SYNC_PRIMARY_NAMESPACE, + "", + PRIMARY_SYNC_GENERATION_KEY, + generation_id.to_vec(), + ) + .await + } + + /// Reads the configured backup's synchronization completion record. + /// + /// The record identifies both the primary generation and index database for which all durable + /// primary data was copied and stale backup data was removed. Its absence therefore means + /// synchronization is required; it must not be inferred from individual backup values. + /// + /// Returns [`io::ErrorKind::NotFound`] when no backup is configured or the configured backup has + /// no completion record. Other storage errors and malformed completion records are propagated so + /// callers cannot mistake an unreadable record for proof that the backup is current. + async fn read_backup_sync_completion(&self) -> io::Result { + let backup_store = self.backup_store.as_ref().ok_or_else(|| { + io::Error::new(io::ErrorKind::NotFound, "Backup store is not configured") + })?; + let encoded = KVStore::read( + backup_store.as_ref(), + BACKUP_SYNC_PRIMARY_NAMESPACE, + "", + BACKUP_SYNC_COMPLETED_GENERATION_KEY, + ) + .await?; + BackupSyncCompletion::decode(encoded) + } + + /// Reads and validates the primary store's synchronization generation. + /// + /// Returns [`io::ErrorKind::NotFound`] when the generation is absent and + /// [`io::ErrorKind::InvalidData`] when its value is not exactly + /// [`GENERATION_ID_LEN`] bytes. All other underlying storage errors are propagated. + async fn read_generation_id( + store: &DynStore, key: &str, + ) -> io::Result<[u8; GENERATION_ID_LEN]> { + let generation_id = KVStore::read(store, BACKUP_SYNC_PRIMARY_NAMESPACE, "", key).await?; + generation_id.try_into().map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + "Invalid tier-store backup synchronization generation", + ) + }) + } + + /// Records that the configured backup matches the current primary generation and index database. + /// + /// This must be written only after journal recovery, primary-value copying, and stale-value + /// removal have all succeeded. Writing it last ensures a failed synchronization leaves the + /// previous completion generation unchanged and will be retried on the next startup. + /// + /// Returns an error when no backup is configured or the completion record cannot be persisted. + async fn write_backup_sync_completion( + &self, completion: BackupSyncCompletion, + ) -> io::Result<()> { + let backup_store = self.backup_store.as_ref().ok_or_else(|| { + io::Error::new(io::ErrorKind::NotFound, "Backup store is not configured") + })?; + KVStore::write( + backup_store.as_ref(), + BACKUP_SYNC_PRIMARY_NAMESPACE, + "", + BACKUP_SYNC_COMPLETED_GENERATION_KEY, + completion.encode(), + ) + .await + } + + /// Copies all durable primary data into a stale backup and removes backup-only data. + async fn synchronize_backup(&self) -> io::Result<()> { + match *self.backup_sync_status.lock().expect("lock") { + Some(BackupSyncStatus::Synchronized) => return Ok(()), + Some(BackupSyncStatus::Required) => {}, + Some(BackupSyncStatus::NotConfigured) | None => { + return Err(io::Error::new( + io::ErrorKind::Other, + "Backup synchronization is not required or has not been initialized", + )); + }, + } + let backup_store = self.backup_store.as_ref().ok_or_else(|| { + io::Error::new(io::ErrorKind::NotFound, "Backup store is not configured") + })?; + self.recover_all_journal_entries().await?; + let completion = self.current_backup_sync_completion().await?; + let primary_keys: HashSet<_> = + MigratableKVStore::list_all_keys(self.primary_store.as_ref()) + .await? + .into_iter() + .filter(|(primary_namespace, _, _)| { + primary_namespace != BACKUP_SYNC_PRIMARY_NAMESPACE + }) + .filter(|(primary_namespace, secondary_namespace, key)| { + self.ephemeral_store.is_none() + || !is_ephemeral_cached_key(primary_namespace, secondary_namespace, key) + }) + .collect(); + + for (primary_namespace, secondary_namespace, key) in &primary_keys { + let value = KVStore::read( + self.primary_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + ) + .await?; + KVStore::write( + backup_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + value, + ) + .await?; + } + + for (primary_namespace, secondary_namespace, key) in + MigratableKVStore::list_all_keys(backup_store.as_ref()).await? + { + if primary_namespace == BACKUP_SYNC_PRIMARY_NAMESPACE { + continue; + } + if !primary_keys.contains(&( + primary_namespace.clone(), + secondary_namespace.clone(), + key.clone(), + )) { + KVStore::remove( + backup_store.as_ref(), + &primary_namespace, + &secondary_namespace, + &key, + false, + ) + .await?; + } + } + + self.write_backup_sync_completion(completion).await?; + *self.backup_sync_status.lock().expect("lock") = Some(BackupSyncStatus::Synchronized); + Ok(()) + } + + fn get_new_version_and_lock_ref(&self, locking_key: String) -> (Arc>, u64) { + let version = self.next_write_version.fetch_add(1, Ordering::Relaxed); + if version == u64::MAX { + panic!("TierStore version counter overflowed"); + } + + (self.get_lock_ref(locking_key), version) + } + + /// Returns the lock that serializes operations for one logical key. + fn get_lock_ref(&self, locking_key: String) -> Arc> { + let mut locks = self.locks.lock().expect("lock"); + Arc::clone(locks.entry(locking_key).or_insert_with(|| Arc::new(TokioMutex::new(0)))) + } + + fn clean_locks(&self, lock_ref: &Arc>, locking_key: String) { + let mut locks = self.locks.lock().expect("lock"); + let strong_count = Arc::strong_count(lock_ref); + debug_assert!(strong_count >= 2, "Unexpected TierStore lock strong count"); + if strong_count == 2 { + locks.remove(&locking_key); + } + } + + /// Returns the lock that serializes initialization of the given logical namespace. + fn get_index_initialization_lock( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> Arc> { + let namespace_id = TierStoreIndex::namespace_id(primary_namespace, secondary_namespace); + let mut locks = self.index_initialization_locks.lock().expect("lock"); + Arc::clone(locks.entry(namespace_id).or_insert_with(|| Arc::new(TokioMutex::new(())))) + } + + /// Removes an initialization lock after its final active user releases it. + fn clean_index_initialization_locks( + &self, lock_ref: &Arc>, primary_namespace: &str, secondary_namespace: &str, + ) { + let namespace_id = TierStoreIndex::namespace_id(primary_namespace, secondary_namespace); + let mut locks = self.index_initialization_locks.lock().expect("lock"); + if Arc::strong_count(lock_ref) == 2 { + locks.remove(&namespace_id); + } + } + + fn build_locking_key( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> String { + if primary_namespace.is_empty() { + key.to_owned() + } else { + format!("{}#{}#{}", primary_namespace, secondary_namespace, key) + } + } + + /// Reads from the primary data store. + async fn read_primary( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result> { + match KVStore::read( + self.primary_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + ) + .await + { + Ok(data) => Ok(data), + Err(e) => Err(e), + } + } + + /// Lists keys from the primary data store. + async fn list_primary( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result> { + match KVStore::list(self.primary_store.as_ref(), primary_namespace, secondary_namespace) + .await + { + Ok(keys) => Ok(keys), + Err(e) => { + log_error!( + self.logger, + "Failed to list from primary store for namespace {}/{}: {}.", + primary_namespace, + secondary_namespace, + e + ); + Err(e) + }, + } + } + + async fn write_primary_backup_async( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, + ) -> io::Result<()> { + if let Some(backup_store) = self.backup_store.as_ref() { + let primary_fut = KVStore::write( + self.primary_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + buf.clone(), + ); + + let backup_fut = KVStore::write( + backup_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + buf, + ); + + let (primary_res, backup_res) = tokio::join!(primary_fut, backup_fut); + + self.handle_primary_backup_results( + "write", + primary_namespace, + secondary_namespace, + key, + primary_res, + backup_res, + ) + } else { + KVStore::write( + self.primary_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + buf, + ) + .await + } + } + + async fn remove_primary_backup_async( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, + ) -> io::Result<()> { + let primary_fut = KVStore::remove( + self.primary_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + lazy, + ); + + if let Some(backup_store) = self.backup_store.as_ref() { + let backup_fut = KVStore::remove( + backup_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + lazy, + ); + + let (primary_res, backup_res) = tokio::join!(primary_fut, backup_fut); + + self.handle_primary_backup_results( + "removal", + primary_namespace, + secondary_namespace, + key, + primary_res, + backup_res, + ) + } else { + primary_fut.await + } + } + + async fn execute_locked_write( + &self, lock_ref: Arc>, locking_key: String, version: u64, callback: F, + ) -> io::Result<()> + where + F: FnOnce() -> Fut, + Fut: Future>, + { + let res = { + let mut last_written_version = lock_ref.lock().await; + + if version <= *last_written_version { + Ok(()) + } else { + let res = callback().await; + // A failed multi-store operation may still have updated one of its stores. We record + // the attempted version regardless so an older operation cannot overwrite newer state. + *last_written_version = version; + res + } + }; + + self.clean_locks(&lock_ref, locking_key); + res + } + + async fn read_internal( + &self, primary_namespace: String, secondary_namespace: String, key: String, + ) -> io::Result> { + check_namespace_key_validity( + primary_namespace.as_str(), + secondary_namespace.as_str(), + Some(key.as_str()), + "read", + )?; + self.ensure_namespace_indexed(&primary_namespace, &secondary_namespace).await?; + + let locking_key = self.build_locking_key(&primary_namespace, &secondary_namespace, &key); + let lock_ref = self.get_lock_ref(locking_key.clone()); + let result: io::Result> = async { + let _guard = lock_ref.lock().await; + self.prepare_key_locked(&primary_namespace, &secondary_namespace, &key).await?; + + if is_ephemeral_cached_key(&primary_namespace, &secondary_namespace, &key) { + if let Some(eph_store) = self.ephemeral_store.as_ref() { + // We don't retry ephemeral-store reads here. Local failures are treated as + // terminal for this access path rather than falling back to another store. + return KVStore::read( + eph_store.as_ref(), + &primary_namespace, + &secondary_namespace, + &key, + ) + .await; + } + } + + self.read_primary(&primary_namespace, &secondary_namespace, &key).await + } + .await; + self.clean_locks(&lock_ref, locking_key); + result + } + + async fn write_internal( + &self, primary_namespace: String, secondary_namespace: String, key: String, buf: Vec, + lock_ref: Arc>, locking_key: String, version: u64, + ) -> io::Result<()> { + check_namespace_key_validity( + primary_namespace.as_str(), + secondary_namespace.as_str(), + Some(key.as_str()), + "write", + )?; + self.ensure_namespace_indexed(&primary_namespace, &secondary_namespace).await?; + + self.execute_locked_write(lock_ref, locking_key, version, || async move { + self.write_locked(&primary_namespace, &secondary_namespace, &key, buf).await + }) + .await + } + + /// Prepares and writes one key while its per-key operation lock is held. + async fn write_locked( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, value: Vec, + ) -> io::Result<()> { + self.prepare_key_locked(primary_namespace, secondary_namespace, key).await?; + let tier = self.value_tier(primary_namespace, secondary_namespace, key); + let Some(index) = self.index.as_ref() else { + return self + .write_value(tier, primary_namespace, secondary_namespace, key, value) + .await; + }; + + let requires_backup = tier == ValueTier::Primary && self.backup_store.is_some(); + let operation = if index.contains_entry(primary_namespace, secondary_namespace, key).await? + { + if !requires_backup { + return self + .write_value(tier, primary_namespace, secondary_namespace, key, value) + .await; + } + JournalOperation::Update { value } + } else { + if self.value_exists(tier, primary_namespace, secondary_namespace, key).await? { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Value exists without a tier-store index entry", + )); + } + JournalOperation::Create { value } + }; + let journal_entry = JournalEntry { + primary_namespace: primary_namespace.to_string(), + secondary_namespace: secondary_namespace.to_string(), + key: key.to_string(), + tier, + requires_backup, + operation, + }; + index.write_journal_entry(&journal_entry).await?; + match &journal_entry.operation { + JournalOperation::Create { .. } => self.apply_pending_create(&journal_entry).await?, + JournalOperation::Update { .. } => self.apply_pending_update(&journal_entry).await?, + JournalOperation::Remove { .. } => { + unreachable!("write created a removal journal entry") + }, + } + index.remove_journal_entry(primary_namespace, secondary_namespace, key).await + } + + async fn remove_internal( + &self, primary_namespace: String, secondary_namespace: String, key: String, lazy: bool, + lock_ref: Arc>, locking_key: String, version: u64, + ) -> io::Result<()> { + check_namespace_key_validity( + primary_namespace.as_str(), + secondary_namespace.as_str(), + Some(key.as_str()), + "remove", + )?; + self.ensure_namespace_indexed(&primary_namespace, &secondary_namespace).await?; + + self.execute_locked_write(lock_ref, locking_key, version, || async move { + self.remove_locked(&primary_namespace, &secondary_namespace, &key, lazy).await + }) + .await + } + + /// Prepares and removes one key while its per-key operation lock is held. + async fn remove_locked( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, + ) -> io::Result<()> { + self.prepare_key_locked(primary_namespace, secondary_namespace, key).await?; + let tier = self.value_tier(primary_namespace, secondary_namespace, key); + let Some(index) = self.index.as_ref() else { + return self + .remove_value(tier, primary_namespace, secondary_namespace, key, lazy) + .await; + }; + + let journal_entry = JournalEntry { + primary_namespace: primary_namespace.to_string(), + secondary_namespace: secondary_namespace.to_string(), + key: key.to_string(), + tier, + requires_backup: tier == ValueTier::Primary && self.backup_store.is_some(), + operation: JournalOperation::Remove { lazy }, + }; + index.write_journal_entry(&journal_entry).await?; + self.apply_pending_remove(&journal_entry).await?; + index.remove_journal_entry(primary_namespace, secondary_namespace, key).await + } + + /// Selects the authoritative value tier for a logical key. + fn value_tier( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> ValueTier { + if self.ephemeral_store.is_some() + && is_ephemeral_cached_key(primary_namespace, secondary_namespace, key) + { + ValueTier::Ephemeral + } else { + ValueTier::Primary + } + } + + /// Checks for an existing value before classifying an unindexed key as a new creation. + async fn value_exists( + &self, tier: ValueTier, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result { + let store = match tier { + ValueTier::Primary => &self.primary_store, + ValueTier::Ephemeral => self.ephemeral_store.as_ref().ok_or_else(|| { + io::Error::new(io::ErrorKind::NotFound, "Ephemeral store is unavailable") + })?, + }; + match KVStore::read(store.as_ref(), primary_namespace, secondary_namespace, key).await { + Ok(_) => Ok(true), + Err(e) if e.kind() == io::ErrorKind::NotFound => { + if tier == ValueTier::Primary { + if let Some(backup_store) = self.backup_store.as_ref() { + return match KVStore::read( + backup_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + ) + .await + { + Ok(_) => Err(io::Error::new( + io::ErrorKind::InvalidData, + "Backup value exists without a primary or index entry", + )), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(e), + }; + } + } + Ok(false) + }, + Err(e) => Err(e), + } + } + + /// Writes a value to its authoritative tier and any required backup. + async fn write_value( + &self, tier: ValueTier, primary_namespace: &str, secondary_namespace: &str, key: &str, + value: Vec, + ) -> io::Result<()> { + match tier { + ValueTier::Primary => { + self.write_primary_backup_async(primary_namespace, secondary_namespace, key, value) + .await + }, + ValueTier::Ephemeral => { + let store = self.ephemeral_store.as_ref().ok_or_else(|| { + io::Error::new(io::ErrorKind::NotFound, "Ephemeral store is unavailable") + })?; + KVStore::write(store.as_ref(), primary_namespace, secondary_namespace, key, value) + .await + }, + } + } + + /// Removes a value from its authoritative tier and any required backup. + async fn remove_value( + &self, tier: ValueTier, primary_namespace: &str, secondary_namespace: &str, key: &str, + lazy: bool, + ) -> io::Result<()> { + match tier { + ValueTier::Primary => { + self.remove_primary_backup_async(primary_namespace, secondary_namespace, key, lazy) + .await + }, + ValueTier::Ephemeral => { + let store = self.ephemeral_store.as_ref().ok_or_else(|| { + io::Error::new(io::ErrorKind::NotFound, "Ephemeral store is unavailable") + })?; + KVStore::remove(store.as_ref(), primary_namespace, secondary_namespace, key, lazy) + .await + }, + } + } + + /// Completes a journaled create and makes it visible in the listing index. + async fn apply_pending_create(&self, entry: &JournalEntry) -> io::Result<()> { + let JournalOperation::Create { value } = &entry.operation else { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Expected pending create")); + }; + self.prepare_pending_backup_recovery(entry).await?; + self.write_value( + entry.tier, + &entry.primary_namespace, + &entry.secondary_namespace, + &entry.key, + value.clone(), + ) + .await?; + self.index + .as_ref() + .expect("pending operations require an index") + .write_entry(&entry.primary_namespace, &entry.secondary_namespace, &entry.key) + .await + } + + /// Completes a journaled update without changing the key's existing index membership or order. + async fn apply_pending_update(&self, entry: &JournalEntry) -> io::Result<()> { + let JournalOperation::Update { value } = &entry.operation else { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Expected pending update")); + }; + self.prepare_pending_backup_recovery(entry).await?; + self.write_value( + entry.tier, + &entry.primary_namespace, + &entry.secondary_namespace, + &entry.key, + value.clone(), + ) + .await + } + + /// Completes a journaled removal, hiding the key before deleting its value copies. + async fn apply_pending_remove(&self, entry: &JournalEntry) -> io::Result<()> { + let JournalOperation::Remove { lazy } = &entry.operation else { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Expected pending removal")); + }; + self.prepare_pending_backup_recovery(entry).await?; + self.index + .as_ref() + .expect("pending operations require an index") + .remove_entry(&entry.primary_namespace, &entry.secondary_namespace, &entry.key, *lazy) + .await?; + self.remove_value( + entry.tier, + &entry.primary_namespace, + &entry.secondary_namespace, + &entry.key, + *lazy, + ) + .await + } + + /// Confirms that primary-only recovery has durably invalidated the absent backup. + /// + /// Operations journaled with `requires_backup` normally finish against both durable stores. If + /// this run intentionally omitted the backup, initialization must first rotate the primary + /// generation. Reading or creating that generation here confirms the invalidation is durable + /// before recovery changes primary state; future resilvering then repairs the absent backup. + async fn prepare_pending_backup_recovery(&self, entry: &JournalEntry) -> io::Result<()> { + if !entry.requires_backup || self.backup_store.is_some() { + return Ok(()); + } + if *self.backup_sync_status.lock().expect("lock") != Some(BackupSyncStatus::NotConfigured) { + return Err(io::Error::new( + io::ErrorKind::NotFound, + "Required backup store is unavailable and backup synchronization is not initialized", + )); + } + self.read_or_create_primary_sync_generation_id().await.map(|_| ()) + } + + async fn list_internal( + &self, primary_namespace: String, secondary_namespace: String, + ) -> io::Result> { + check_namespace_key_validity( + primary_namespace.as_str(), + secondary_namespace.as_str(), + None, + "list", + )?; + + self.prepare_namespace_for_listing(&primary_namespace, &secondary_namespace).await?; + if let Some(index) = self.index.as_ref() { + return index.list(&primary_namespace, &secondary_namespace).await; + } + + self.list_value_stores(&primary_namespace, &secondary_namespace).await + } + + /// Returns every logical TierStore key for data migration. + async fn list_all_keys_internal(&self) -> io::Result> { + self.recover_all_journal_entries().await?; + let mut keys: HashSet<_> = MigratableKVStore::list_all_keys(self.primary_store.as_ref()) + .await? + .into_iter() + .filter(|(primary_namespace, _, _)| primary_namespace != BACKUP_SYNC_PRIMARY_NAMESPACE) + .collect(); + + if let Some(ephemeral_store) = self.ephemeral_store.as_ref() { + for key in MigratableKVStore::list_all_keys(ephemeral_store.as_ref()).await? { + if is_ephemeral_cached_key(&key.0, &key.1, &key.2) { + keys.insert(key); + } + } + } + Ok(keys.into_iter().collect()) + } + + /// Imports a namespace's existing primary-store keys and makes its index authoritative. + /// + /// Primary pagination returns keys from newest to oldest, so the complete result is reversed + /// before insertion to preserve that order in the local index. Values already present in the + /// ephemeral store are discarded because their positions relative to primary keys cannot be + /// reconstructed without the index. Cache values still in the primary store are retained and + /// subsequently moved to ephemeral storage without changing their imported index positions. + async fn ensure_namespace_indexed( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result<()> { + let Some(index) = self.index.as_ref() else { + return Ok(()); + }; + if index.is_namespace_ready(primary_namespace, secondary_namespace).await? { + return Ok(()); + } + + let lock_ref = self.get_index_initialization_lock(primary_namespace, secondary_namespace); + let result: io::Result<()> = async { + let _guard = lock_ref.lock().await; + if index.is_namespace_ready(primary_namespace, secondary_namespace).await? { + Ok(()) + } else { + self.discard_unindexed_ephemeral_cache(primary_namespace, secondary_namespace) + .await?; + let mut keys = Vec::new(); + let mut page_token = None; + loop { + let page = PaginatedKVStore::list_paginated( + self.primary_store.as_ref(), + primary_namespace, + secondary_namespace, + page_token, + ) + .await?; + keys.extend(page.keys); + match page.next_page_token { + Some(next_page_token) => page_token = Some(next_page_token), + None => break, + } + } + + for key in keys.into_iter().rev() { + index.write_entry(primary_namespace, secondary_namespace, &key).await?; + } + index.mark_namespace_ready(primary_namespace, secondary_namespace).await + } + } + .await; + self.clean_index_initialization_locks(&lock_ref, primary_namespace, secondary_namespace); + result + } + + /// Prepares a complete namespace before exposing its index through a listing. + async fn prepare_namespace_for_listing( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result<()> { + self.ensure_namespace_indexed(primary_namespace, secondary_namespace).await?; + self.recover_namespace(primary_namespace, secondary_namespace).await?; + self.ensure_ephemeral_cache_reconciled(primary_namespace, secondary_namespace).await + } + + /// Discards cache values whose ordering cannot be recovered from a missing index. + async fn discard_unindexed_ephemeral_cache( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result<()> { + let Some(ephemeral_store) = self.ephemeral_store.as_ref() else { + return Ok(()); + }; + for key in ephemeral_cache_keys(primary_namespace) { + KVStore::remove( + ephemeral_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + false, + ) + .await?; + } + Ok(()) + } + + /// Reconciles every cache key before an operation that prepares the complete namespace. + async fn ensure_ephemeral_cache_reconciled( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result<()> { + let (Some(_), Some(_)) = (self.index.as_ref(), self.ephemeral_store.as_ref()) else { + return Ok(()); + }; + + for key in ephemeral_cache_keys(primary_namespace) { + let locking_key = self.build_locking_key(primary_namespace, secondary_namespace, key); + let lock_ref = self.get_lock_ref(locking_key.clone()); + let result: io::Result<()> = async { + let _guard = lock_ref.lock().await; + self.prepare_key_locked(primary_namespace, secondary_namespace, key).await + } + .await; + self.clean_locks(&lock_ref, locking_key); + result?; + } + Ok(()) + } + + /// Moves one indexed cache value to ephemeral storage while its per-key lock is held. + /// + /// Destination writes precede source removals. Repeating this after interruption either copies + /// the primary value again or removes a stale primary/backup copy after finding the destination. + async fn reconcile_ephemeral_cache_key_locked( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result<()> { + let (Some(index), Some(ephemeral_store)) = + (self.index.as_ref(), self.ephemeral_store.as_ref()) + else { + return Ok(()); + }; + if !is_ephemeral_cached_key(primary_namespace, secondary_namespace, key) + || index.is_cache_ready(primary_namespace, secondary_namespace, key).await? + { + return Ok(()); + } + + if index.contains_entry(primary_namespace, secondary_namespace, key).await? { + match KVStore::read( + ephemeral_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + ) + .await + { + Ok(_) => {}, + Err(e) if e.kind() == io::ErrorKind::NotFound => { + match self.read_primary(primary_namespace, secondary_namespace, key).await { + Ok(value) => { + KVStore::write( + ephemeral_store.as_ref(), + primary_namespace, + secondary_namespace, + key, + value, + ) + .await?; + }, + Err(e) if e.kind() == io::ErrorKind::NotFound => { + self.remove_primary_backup_async( + primary_namespace, + secondary_namespace, + key, + false, + ) + .await?; + index + .remove_entry(primary_namespace, secondary_namespace, key, false) + .await?; + return index + .mark_cache_ready(primary_namespace, secondary_namespace, key) + .await; + }, + Err(e) => return Err(e), + } + }, + Err(e) => return Err(e), + } + + self.remove_primary_backup_async(primary_namespace, secondary_namespace, key, false) + .await?; + } + + index.mark_cache_ready(primary_namespace, secondary_namespace, key).await + } + + /// Recovers and reconciles one key while its per-key operation lock is held by the caller. + async fn prepare_key_locked( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result<()> { + self.recover_key_locked(primary_namespace, secondary_namespace, key).await?; + self.reconcile_ephemeral_cache_key_locked(primary_namespace, secondary_namespace, key).await + } + + /// Completes journaled creates, updates, and removals before exposing a namespace. + async fn recover_namespace( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result<()> { + let Some(index) = self.index.as_ref() else { + return Ok(()); + }; + for key in index.list_journal_entries(primary_namespace, secondary_namespace).await? { + let locking_key = self.build_locking_key(primary_namespace, secondary_namespace, &key); + let lock_ref = self.get_lock_ref(locking_key.clone()); + let result: io::Result<()> = async { + let _guard = lock_ref.lock().await; + self.recover_key_locked(primary_namespace, secondary_namespace, &key).await + } + .await; + self.clean_locks(&lock_ref, locking_key); + result?; + } + Ok(()) + } + + /// Recovers every journal entry before taking a complete migration snapshot. + async fn recover_all_journal_entries(&self) -> io::Result<()> { + let Some(index) = self.index.as_ref() else { + return Ok(()); + }; + for entry in index.list_all_journal_entries().await? { + let locking_key = self.build_locking_key( + &entry.primary_namespace, + &entry.secondary_namespace, + &entry.key, + ); + let lock_ref = self.get_lock_ref(locking_key.clone()); + let result: io::Result<()> = async { + let _guard = lock_ref.lock().await; + self.recover_key_locked( + &entry.primary_namespace, + &entry.secondary_namespace, + &entry.key, + ) + .await + } + .await; + self.clean_locks(&lock_ref, locking_key); + result?; + } + Ok(()) + } + + /// Recovers one key while its per-key operation lock is held by the caller. + async fn recover_key_locked( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> io::Result<()> { + let Some(index) = self.index.as_ref() else { + return Ok(()); + }; + let entry = + match index.read_journal_entry(primary_namespace, secondary_namespace, key).await { + Ok(entry) => entry, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(e) => return Err(e), + }; + match &entry.operation { + JournalOperation::Create { .. } => self.apply_pending_create(&entry).await?, + JournalOperation::Update { .. } => self.apply_pending_update(&entry).await?, + JournalOperation::Remove { .. } => self.apply_pending_remove(&entry).await?, + } + index.remove_journal_entry(primary_namespace, secondary_namespace, key).await + } + + /// Lists the authoritative logical keys directly from the primary and ephemeral value stores. + /// + /// Ephemeral-routed keys are taken only from the ephemeral store, excluding stale primary + /// copies. This path is used when no local index store is configured. + async fn list_value_stores( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> io::Result> { + let mut keys = self.list_primary(primary_namespace, secondary_namespace).await?; + + let Some(ephemeral_store) = self.ephemeral_store.as_ref() else { + return Ok(keys); + }; + + if primary_namespace != NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE + && primary_namespace != SCORER_PERSISTENCE_PRIMARY_NAMESPACE + { + return Ok(keys); + } + + // The ephemeral store is authoritative for keys routed there. Exclude stale + // primary copies, then add only routed keys from the ephemeral store. + keys.retain(|key| !is_ephemeral_cached_key(primary_namespace, secondary_namespace, key)); + + let ephemeral_keys = + KVStore::list(ephemeral_store.as_ref(), primary_namespace, secondary_namespace).await?; + + for key in ephemeral_keys { + if is_ephemeral_cached_key(primary_namespace, secondary_namespace, &key) + && !keys.contains(&key) + { + keys.push(key); + } + } + + Ok(keys) + } + + async fn list_paginated_internal( + &self, primary_namespace: String, secondary_namespace: String, + page_token: Option, + ) -> io::Result { + check_namespace_key_validity( + primary_namespace.as_str(), + secondary_namespace.as_str(), + None, + "list_paginated", + )?; + + self.prepare_namespace_for_listing(&primary_namespace, &secondary_namespace).await?; + if let Some(index) = self.index.as_ref() { + return index + .list_paginated(&primary_namespace, &secondary_namespace, page_token) + .await; + } + + PaginatedKVStore::list_paginated( + self.primary_store.as_ref(), + &primary_namespace, + &secondary_namespace, + page_token, + ) + .await + } + + fn handle_primary_backup_results( + &self, op: &str, primary_namespace: &str, secondary_namespace: &str, key: &str, + primary_res: io::Result<()>, backup_res: io::Result<()>, + ) -> io::Result<()> { + match (primary_res, backup_res) { + (Ok(()), Ok(())) => Ok(()), + (Err(primary_err), Ok(())) => { + log_error!( + self.logger, + "Primary {} failed after backup {} succeeded for key {}/{}/{}; primary and backup may have diverged: {}", + op, + op, + primary_namespace, + secondary_namespace, + key, + primary_err + ); + Err(primary_err) + }, + (Ok(()), Err(backup_err)) => { + log_error!( + self.logger, + "Backup {} failed after primary {} succeeded for key {}/{}/{}; primary and backup may have diverged: {}", + op, + op, + primary_namespace, + secondary_namespace, + key, + backup_err + ); + Err(backup_err) + }, + (Err(primary_err), Err(backup_err)) => { + log_error!( + self.logger, + "Primary and backup {}s both failed for key {}/{}/{}: primary={}, backup={}", + op, + primary_namespace, + secondary_namespace, + key, + primary_err, + backup_err + ); + Err(primary_err) + }, + } + } +} + +fn is_ephemeral_cached_key(pn: &str, _sn: &str, key: &str) -> bool { + ephemeral_cache_keys(pn).any(|cache_key| cache_key == key) +} + +fn ephemeral_cache_keys(primary_namespace: &str) -> impl Iterator + '_ { + [ + (NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_KEY), + (SCORER_PERSISTENCE_PRIMARY_NAMESPACE, SCORER_PERSISTENCE_KEY), + (SCORER_PERSISTENCE_PRIMARY_NAMESPACE, EXTERNAL_PATHFINDING_SCORES_CACHE_KEY), + ] + .into_iter() + .filter_map(move |(cache_namespace, cache_key)| { + (primary_namespace == cache_namespace).then_some(cache_key) + }) +} + +#[cfg(test)] +mod tests { + use std::future::Future; + use std::panic::RefUnwindSafe; + use std::path::PathBuf; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + + use lightning::util::logger::Level; + use lightning::util::persist::{ + CHANNEL_MANAGER_PERSISTENCE_KEY, CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + }; + use lightning_persister::fs_store::v2::FilesystemStoreV2; + + use super::*; + use crate::io::test_utils::{ + do_read_write_remove_list_persist, random_storage_path, InMemoryStore, + }; + use crate::io::tier_store::TierStore; + use crate::logger::Logger; + use crate::types::{DynStore, DynStoreWrapper}; + + impl RefUnwindSafe for TierStore {} + + struct CleanupDir(PathBuf); + impl Drop for CleanupDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + fn setup_test_environment() -> (PathBuf, Arc, CleanupDir) { + let base_dir = random_storage_path(); + let log_path = base_dir.join("tier_store_test.log").to_string_lossy().into_owned(); + let logger = Arc::new(Logger::new_fs_writer(log_path, Level::Trace).unwrap()); + let cleanup = CleanupDir(base_dir.clone()); + (base_dir, logger, cleanup) + } + + fn setup_tier_store(primary_store: Arc, logger: Arc) -> TierStore { + TierStore::new(primary_store, logger) + } + + fn set_test_index_store(tier: &mut TierStore) { + let store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + tier.set_index_store(TierStoreIndex::from_store(store)); + } + + fn test_backup_sync_completion( + primary_generation_id: [u8; GENERATION_ID_LEN], + ) -> BackupSyncCompletion { + BackupSyncCompletion::new(primary_generation_id, [1; INDEX_DATABASE_ID_LEN]) + } + + async fn read_test_backup_sync_completion(store: &DynStore) -> BackupSyncCompletion { + let encoded = store + .read(BACKUP_SYNC_PRIMARY_NAMESPACE, "", BACKUP_SYNC_COMPLETED_GENERATION_KEY) + .await + .unwrap(); + BackupSyncCompletion::decode(encoded).unwrap() + } + + #[test] + fn backup_sync_completion_roundtrips_and_rejects_invalid_length() { + let completion = + BackupSyncCompletion::new([2; GENERATION_ID_LEN], [3; INDEX_DATABASE_ID_LEN]); + assert_eq!(BackupSyncCompletion::decode(completion.encode()).unwrap(), completion); + assert_eq!( + BackupSyncCompletion::decode(vec![0; GENERATION_ID_LEN]).unwrap_err().kind(), + io::ErrorKind::InvalidData + ); + } + + #[tokio::test] + async fn backup_sync_metadata_remains_absent_when_a_backup_has_never_been_configured() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + + let tier = setup_tier_store(Arc::clone(&primary_store), Arc::clone(&logger)); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::NotConfigured + ); + drop(tier); + let restarted_tier = setup_tier_store(Arc::clone(&primary_store), logger); + assert_eq!( + restarted_tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::NotConfigured + ); + let error = + TierStoreInner::read_generation_id(primary_store.as_ref(), PRIMARY_SYNC_GENERATION_KEY) + .await + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::NotFound); + } + + #[tokio::test] + async fn backup_sync_generation_rotates_after_backup_is_removed() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + + let mut tier = setup_tier_store(Arc::clone(&primary_store), Arc::clone(&logger)); + tier.set_backup_store(backup_store); + set_test_index_store(&mut tier); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + let configured_generation_id = + TierStoreInner::read_generation_id(primary_store.as_ref(), PRIMARY_SYNC_GENERATION_KEY) + .await + .unwrap(); + drop(tier); + + let restarted_tier = setup_tier_store(Arc::clone(&primary_store), logger); + assert_eq!( + restarted_tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::NotConfigured + ); + let rotated_generation_id = + TierStoreInner::read_generation_id(primary_store.as_ref(), PRIMARY_SYNC_GENERATION_KEY) + .await + .unwrap(); + + assert_ne!(rotated_generation_id, configured_generation_id); + } + + #[tokio::test] + async fn backup_sync_status_tracks_persisted_completion() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + + let mut tier = setup_tier_store(Arc::clone(&primary_store), Arc::clone(&logger)); + tier.set_backup_store(Arc::clone(&backup_store)); + set_test_index_store(&mut tier); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + let expected_completion = tier.inner.current_backup_sync_completion().await.unwrap(); + + let mut stale_completion = tier.inner.current_backup_sync_completion().await.unwrap(); + stale_completion.primary_generation_id[0] ^= 1; + tier.inner.write_backup_sync_completion(stale_completion).await.unwrap(); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + + let completion = tier.inner.current_backup_sync_completion().await.unwrap(); + tier.inner.write_backup_sync_completion(completion).await.unwrap(); + drop(tier); + let mut restarted_tier = setup_tier_store(Arc::clone(&primary_store), logger); + restarted_tier.set_backup_store(backup_store); + set_test_index_store(&mut restarted_tier); + assert_eq!( + restarted_tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Synchronized + ); + assert_eq!( + TierStoreInner::read_generation_id(primary_store.as_ref(), PRIMARY_SYNC_GENERATION_KEY) + .await + .unwrap(), + expected_completion.primary_generation_id + ); + } + + #[tokio::test] + async fn replacing_the_index_requires_backup_synchronization() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + + let mut tier = setup_tier_store(Arc::clone(&primary_store), Arc::clone(&logger)); + tier.set_backup_store(Arc::clone(&backup_store)); + let first_index_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + tier.set_index_store(TierStoreIndex::from_store_with_database_id( + first_index_store, + [1; INDEX_DATABASE_ID_LEN], + )); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + let completion = tier.inner.current_backup_sync_completion().await.unwrap(); + tier.inner.write_backup_sync_completion(completion).await.unwrap(); + drop(tier); + + let mut restarted_tier = setup_tier_store(primary_store, logger); + restarted_tier.set_backup_store(backup_store); + let replacement_index_store: Arc = + Arc::new(DynStoreWrapper(InMemoryStore::new())); + restarted_tier.set_index_store(TierStoreIndex::from_store_with_database_id( + replacement_index_store, + [2; INDEX_DATABASE_ID_LEN], + )); + + assert_eq!( + restarted_tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + } + + #[tokio::test] + async fn backup_synchronization_copies_current_data_removes_stale_data_and_recovers_journal() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + primary_store.write("namespace", "", "current", vec![2]).await.unwrap(); + primary_store.write("namespace", "", "new", vec![3]).await.unwrap(); + backup_store.write("namespace", "", "current", vec![1]).await.unwrap(); + backup_store.write("namespace", "", "stale", vec![4]).await.unwrap(); + backup_store + .write(BACKUP_SYNC_PRIMARY_NAMESPACE, "", "other_metadata", vec![6]) + .await + .unwrap(); + + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + tier.set_backup_store(Arc::clone(&backup_store)); + set_test_index_store(&mut tier); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + let expected_completion = tier.inner.current_backup_sync_completion().await.unwrap(); + let pending = JournalEntry { + primary_namespace: "namespace".to_string(), + secondary_namespace: String::new(), + key: "pending".to_string(), + tier: ValueTier::Primary, + requires_backup: true, + operation: JournalOperation::Create { value: vec![5] }, + }; + tier.inner.index.as_ref().unwrap().write_journal_entry(&pending).await.unwrap(); + + tier.synchronize_backup().await.unwrap(); + + assert_eq!(backup_store.read("namespace", "", "current").await.unwrap(), vec![2]); + assert_eq!(backup_store.read("namespace", "", "new").await.unwrap(), vec![3]); + assert_eq!(backup_store.read("namespace", "", "pending").await.unwrap(), vec![5]); + assert!(backup_store.read("namespace", "", "stale").await.is_err()); + assert_eq!( + backup_store.read(BACKUP_SYNC_PRIMARY_NAMESPACE, "", "other_metadata").await.unwrap(), + vec![6] + ); + assert!(tier + .inner + .index + .as_ref() + .unwrap() + .read_journal_entry("namespace", "", "pending") + .await + .is_err()); + assert_eq!( + read_test_backup_sync_completion(backup_store.as_ref()).await, + expected_completion + ); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Synchronized + ); + } + + #[tokio::test] + async fn backup_synchronization_excludes_ephemeral_cache_values() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + for store in [&primary_store, &backup_store] { + store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + vec![1], + ) + .await + .unwrap(); + } + + let mut tier = setup_tier_store(primary_store, logger); + tier.set_backup_store(Arc::clone(&backup_store)); + tier.set_ephemeral_store(ephemeral_store); + set_test_index_store(&mut tier); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + + tier.synchronize_backup().await.unwrap(); + + assert!(backup_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .is_err()); + } + + #[tokio::test] + async fn failed_stale_cleanup_preserves_backup_completion_generation() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let remove_attempts = Arc::new(AtomicUsize::new(0)); + let backup_store: Arc = + Arc::new(DynStoreWrapper(InstrumentedStore::new(StoreBehavior::FailRemoveOnce { + attempts: Arc::clone(&remove_attempts), + }))); + primary_store.write("namespace", "", "current", vec![2]).await.unwrap(); + backup_store.write("namespace", "", "stale", vec![1]).await.unwrap(); + let old_completion = test_backup_sync_completion([7; GENERATION_ID_LEN]); + backup_store + .write( + BACKUP_SYNC_PRIMARY_NAMESPACE, + "", + BACKUP_SYNC_COMPLETED_GENERATION_KEY, + old_completion.encode(), + ) + .await + .unwrap(); + + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + tier.set_backup_store(Arc::clone(&backup_store)); + set_test_index_store(&mut tier); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + let expected_completion = tier.inner.current_backup_sync_completion().await.unwrap(); + + assert!(tier.synchronize_backup().await.is_err()); + + assert_eq!(read_test_backup_sync_completion(backup_store.as_ref()).await, old_completion); + assert_eq!(remove_attempts.load(Ordering::Relaxed), 1); + + tier.synchronize_backup().await.unwrap(); + + assert!(backup_store.read("namespace", "", "stale").await.is_err()); + assert_eq!( + read_test_backup_sync_completion(backup_store.as_ref()).await, + expected_completion + ); + } + + #[tokio::test] + async fn failed_journal_recovery_preserves_backup_completion_generation_and_can_retry() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let write_attempts = Arc::new(AtomicUsize::new(0)); + let backup_store: Arc = + Arc::new(DynStoreWrapper(InstrumentedStore::new(StoreBehavior::FailWriteOnce { + primary_namespace: "namespace", + key: "pending", + attempts: Arc::clone(&write_attempts), + }))); + let old_completion = test_backup_sync_completion([7; GENERATION_ID_LEN]); + backup_store + .write( + BACKUP_SYNC_PRIMARY_NAMESPACE, + "", + BACKUP_SYNC_COMPLETED_GENERATION_KEY, + old_completion.encode(), + ) + .await + .unwrap(); + + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + tier.set_backup_store(Arc::clone(&backup_store)); + set_test_index_store(&mut tier); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + let expected_completion = tier.inner.current_backup_sync_completion().await.unwrap(); + let pending = JournalEntry { + primary_namespace: "namespace".to_string(), + secondary_namespace: String::new(), + key: "pending".to_string(), + tier: ValueTier::Primary, + requires_backup: true, + operation: JournalOperation::Create { value: vec![1] }, + }; + tier.inner.index.as_ref().unwrap().write_journal_entry(&pending).await.unwrap(); + + assert!(tier.synchronize_backup().await.is_err()); + + assert_eq!(write_attempts.load(Ordering::Relaxed), 1); + assert_eq!(primary_store.read("namespace", "", "pending").await.unwrap(), vec![1]); + assert_eq!( + tier.inner + .index + .as_ref() + .unwrap() + .read_journal_entry("namespace", "", "pending") + .await + .unwrap(), + pending + ); + assert_eq!(read_test_backup_sync_completion(backup_store.as_ref()).await, old_completion); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + + tier.synchronize_backup().await.unwrap(); + + assert_eq!(backup_store.read("namespace", "", "pending").await.unwrap(), vec![1]); + assert!(tier + .inner + .index + .as_ref() + .unwrap() + .read_journal_entry("namespace", "", "pending") + .await + .is_err()); + assert_eq!( + read_test_backup_sync_completion(backup_store.as_ref()).await, + expected_completion + ); + } + + #[tokio::test] + async fn failed_primary_copy_preserves_backup_completion_generation_and_can_retry() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + primary_store.write("namespace", "", "current", vec![1]).await.unwrap(); + let write_attempts = Arc::new(AtomicUsize::new(0)); + let backup_store: Arc = + Arc::new(DynStoreWrapper(InstrumentedStore::new(StoreBehavior::FailWriteOnce { + primary_namespace: "namespace", + key: "current", + attempts: Arc::clone(&write_attempts), + }))); + let old_completion = test_backup_sync_completion([7; GENERATION_ID_LEN]); + backup_store + .write( + BACKUP_SYNC_PRIMARY_NAMESPACE, + "", + BACKUP_SYNC_COMPLETED_GENERATION_KEY, + old_completion.encode(), + ) + .await + .unwrap(); + + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + tier.set_backup_store(Arc::clone(&backup_store)); + set_test_index_store(&mut tier); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + let expected_completion = tier.inner.current_backup_sync_completion().await.unwrap(); + + assert!(tier.synchronize_backup().await.is_err()); + + assert_eq!(write_attempts.load(Ordering::Relaxed), 1); + assert!(backup_store.read("namespace", "", "current").await.is_err()); + assert_eq!(read_test_backup_sync_completion(backup_store.as_ref()).await, old_completion); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + + tier.synchronize_backup().await.unwrap(); + + assert_eq!(backup_store.read("namespace", "", "current").await.unwrap(), vec![1]); + assert_eq!( + read_test_backup_sync_completion(backup_store.as_ref()).await, + expected_completion + ); + } + + #[tokio::test] + async fn failed_completion_write_preserves_previous_generation_and_can_retry() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + primary_store.write("namespace", "", "current", vec![1]).await.unwrap(); + let write_attempts = Arc::new(AtomicUsize::new(0)); + let backup_store = InstrumentedStore::new(StoreBehavior::FailWriteOnce { + primary_namespace: BACKUP_SYNC_PRIMARY_NAMESPACE, + key: BACKUP_SYNC_COMPLETED_GENERATION_KEY, + attempts: Arc::clone(&write_attempts), + }); + let old_completion = test_backup_sync_completion([7; GENERATION_ID_LEN]); + backup_store + .inner + .write( + BACKUP_SYNC_PRIMARY_NAMESPACE, + "", + BACKUP_SYNC_COMPLETED_GENERATION_KEY, + old_completion.encode(), + ) + .await + .unwrap(); + backup_store.inner.write("namespace", "", "stale", vec![2]).await.unwrap(); + let backup_store: Arc = Arc::new(DynStoreWrapper(backup_store)); + + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + tier.set_backup_store(Arc::clone(&backup_store)); + set_test_index_store(&mut tier); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + let expected_completion = tier.inner.current_backup_sync_completion().await.unwrap(); + + assert!(tier.synchronize_backup().await.is_err()); + + assert_eq!(write_attempts.load(Ordering::Relaxed), 1); + assert_eq!(backup_store.read("namespace", "", "current").await.unwrap(), vec![1]); + assert!(backup_store.read("namespace", "", "stale").await.is_err()); + assert_eq!(read_test_backup_sync_completion(backup_store.as_ref()).await, old_completion); + assert_eq!( + tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + + tier.synchronize_backup().await.unwrap(); + + assert_eq!( + read_test_backup_sync_completion(backup_store.as_ref()).await, + expected_completion + ); + } + + #[test] + fn journal_entry_roundtrips() { + for operation in [ + JournalOperation::Create { value: vec![0, 1, 2, 255] }, + JournalOperation::Update { value: vec![255, 2, 1, 0] }, + JournalOperation::Remove { lazy: true }, + ] { + let entry = JournalEntry { + primary_namespace: "primary".to_string(), + secondary_namespace: "secondary".to_string(), + key: "key".to_string(), + tier: ValueTier::Primary, + requires_backup: true, + operation, + }; + assert_eq!(JournalEntry::deserialize(&entry.serialize()).unwrap(), entry); + } + } + + #[test] + fn page_token_roundtrips_and_validates_context() { + let database_id = [2; INDEX_DATABASE_ID_LEN]; + let namespace_id = TierStoreIndex::namespace_id("primary", "secondary"); + let token = TierStorePageToken::encode( + &database_id, + namespace_id.clone(), + PageToken::new("opaque:index-token".to_string()), + ); + + let decoded = + TierStorePageToken::decode(token.clone(), &database_id, &namespace_id).unwrap(); + assert_eq!(decoded.as_str(), "opaque:index-token"); + assert_eq!( + TierStorePageToken::decode(token.clone(), &[3; INDEX_DATABASE_ID_LEN], &namespace_id) + .unwrap_err() + .kind(), + io::ErrorKind::InvalidInput + ); + assert_eq!( + TierStorePageToken::decode(token, &database_id, "another-namespace") + .unwrap_err() + .kind(), + io::ErrorKind::InvalidInput + ); + assert_eq!( + TierStorePageToken::decode( + PageToken::new("not-a-tier-store-token".to_string()), + &database_id, + &namespace_id, + ) + .unwrap_err() + .kind(), + io::ErrorKind::InvalidInput + ); + let unsupported_version = TierStorePageToken { + format_version: PAGE_TOKEN_FORMAT_VERSION + 1, + index_database_id: database_id.to_vec(), + namespace_id: namespace_id.clone(), + index_page_token: "opaque:index-token".to_string(), + }; + let unsupported_version = + PageToken::new(Writeable::encode(&unsupported_version).to_lower_hex_string()); + assert_eq!( + TierStorePageToken::decode(unsupported_version, &database_id, &namespace_id) + .unwrap_err() + .kind(), + io::ErrorKind::InvalidInput + ); + } + + #[tokio::test] + async fn index_store_is_internal_persistent_sqlite_store() { + let base_dir = random_storage_path(); + let _cleanup = CleanupDir(base_dir.clone()); + + let index = setup_index_store(base_dir.clone()).await.unwrap(); + assert!(base_dir.join(SQLITE_TIER_INDEX_DB_FILE_NAME).exists()); + + let database_id = + TierStoreIndex::read_or_create_database_id(index.store.as_ref()).await.unwrap(); + let persisted_database_id = + TierStoreIndex::read_or_create_database_id(index.store.as_ref()).await.unwrap(); + assert_ne!(database_id, [0; INDEX_DATABASE_ID_LEN]); + assert_eq!(persisted_database_id, database_id); + } + + #[tokio::test] + async fn index_store_rejects_second_owner() { + let base_dir = random_storage_path(); + let _cleanup = CleanupDir(base_dir.clone()); + + let _index = setup_index_store(base_dir.clone()).await.unwrap(); + let error = match setup_index_store(base_dir).await { + Ok(_) => panic!("a second index-store owner must be rejected"), + Err(e) => e, + }; + assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); + } + + #[tokio::test] + async fn indexed_listing_orders_keys_across_primary_and_ephemeral_stores() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(primary_store, logger); + set_test_index_store(&mut tier); + tier.set_ephemeral_store(ephemeral_store); + + for key in ["primary-a", NETWORK_GRAPH_PERSISTENCE_KEY, "primary-b"] { + tier.write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + key, + vec![1], + ) + .await + .unwrap(); + } + + let page = PaginatedKVStore::list_paginated( + &tier, + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + None, + ) + .await + .unwrap(); + assert_eq!( + page.keys, + vec![ + "primary-b".to_string(), + NETWORK_GRAPH_PERSISTENCE_KEY.to_string(), + "primary-a".to_string(), + ] + ); + + let mut listed = KVStore::list( + &tier, + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + ) + .await + .unwrap(); + listed.sort(); + assert_eq!( + listed, + vec![ + NETWORK_GRAPH_PERSISTENCE_KEY.to_string(), + "primary-a".to_string(), + "primary-b".to_string(), + ] + ); + } + + #[tokio::test] + async fn indexed_listing_preserves_updates_and_reorders_recreated_keys() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(primary_store, logger); + set_test_index_store(&mut tier); + + for key in ["a", "b"] { + tier.write("namespace", "", key, vec![1]).await.unwrap(); + } + tier.write("namespace", "", "a", vec![2]).await.unwrap(); + + let page = PaginatedKVStore::list_paginated(&tier, "namespace", "", None).await.unwrap(); + assert_eq!(page.keys, vec!["b".to_string(), "a".to_string()]); + + tier.remove("namespace", "", "a", false).await.unwrap(); + tier.write("namespace", "", "a", vec![3]).await.unwrap(); + let page = PaginatedKVStore::list_paginated(&tier, "namespace", "", None).await.unwrap(); + assert_eq!(page.keys, vec!["a".to_string(), "b".to_string()]); + } + + #[tokio::test] + async fn adopting_ephemeral_storage_migrates_cache_keys_without_reordering_them() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + for key in ["old", NETWORK_GRAPH_PERSISTENCE_KEY, "new"] { + primary_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + key, + key.as_bytes().to_vec(), + ) + .await + .unwrap(); + backup_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + key, + key.as_bytes().to_vec(), + ) + .await + .unwrap(); + } + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + set_test_index_store(&mut tier); + tier.set_backup_store(Arc::clone(&backup_store)); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + let page = PaginatedKVStore::list_paginated( + &tier, + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + None, + ) + .await + .unwrap(); + assert_eq!( + page.keys, + vec!["new".to_string(), NETWORK_GRAPH_PERSISTENCE_KEY.to_string(), "old".to_string(),] + ); + assert_eq!( + ephemeral_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .unwrap(), + NETWORK_GRAPH_PERSISTENCE_KEY.as_bytes() + ); + assert!(primary_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .is_err()); + assert!(backup_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .is_err()); + } + + #[tokio::test] + async fn missing_index_discards_ephemeral_only_cache_data_before_reading() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + ephemeral_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + vec![1], + ) + .await + .unwrap(); + let mut tier = setup_tier_store(primary_store, logger); + set_test_index_store(&mut tier); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + assert!(tier + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .is_err()); + assert!(ephemeral_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .is_err()); + assert!(KVStore::list( + &tier, + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + ) + .await + .unwrap() + .is_empty()); + } + + #[tokio::test] + async fn missing_index_rebuilds_cache_from_primary_instead_of_ephemeral() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + primary_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + vec![2], + ) + .await + .unwrap(); + ephemeral_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + vec![1], + ) + .await + .unwrap(); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + set_test_index_store(&mut tier); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + assert_eq!( + tier.read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .unwrap(), + vec![2] + ); + assert!(primary_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .is_err()); + assert!(tier + .inner + .index + .as_ref() + .unwrap() + .contains_entry( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .unwrap()); + } + + #[tokio::test] + async fn namespace_preparation_finishes_interrupted_cache_migration() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + set_test_index_store(&mut tier); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + let index = tier.inner.index.as_ref().unwrap(); + for key in ["old", NETWORK_GRAPH_PERSISTENCE_KEY, "new"] { + index + .write_entry( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + key, + ) + .await + .unwrap(); + } + index + .mark_namespace_ready( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + ) + .await + .unwrap(); + primary_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + vec![1], + ) + .await + .unwrap(); + ephemeral_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + vec![1], + ) + .await + .unwrap(); + + let page = PaginatedKVStore::list_paginated( + &tier, + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + None, + ) + .await + .unwrap(); + assert_eq!( + page.keys, + vec!["new".to_string(), NETWORK_GRAPH_PERSISTENCE_KEY.to_string(), "old".to_string(),] + ); + assert!(primary_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .is_err()); + } + + #[tokio::test] + async fn reading_one_cache_key_only_reconciles_that_key() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + for (key, value) in + [(SCORER_PERSISTENCE_KEY, vec![1]), (EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, vec![2])] + { + primary_store + .write( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + key, + value, + ) + .await + .unwrap(); + } + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + set_test_index_store(&mut tier); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + assert_eq!( + tier.read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + SCORER_PERSISTENCE_KEY, + ) + .await + .unwrap(), + vec![1] + ); + assert!(primary_store + .read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + SCORER_PERSISTENCE_KEY, + ) + .await + .is_err()); + assert!(ephemeral_store + .read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, + ) + .await + .is_err()); + assert_eq!( + primary_store + .read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, + ) + .await + .unwrap(), + vec![2] + ); + + let index = tier.inner.index.as_ref().unwrap(); + assert!(index + .is_cache_ready( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + SCORER_PERSISTENCE_KEY, + ) + .await + .unwrap()); + assert!(!index + .is_cache_ready( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, + ) + .await + .unwrap()); + } + + #[tokio::test] + async fn writing_one_cache_key_only_reconciles_that_key() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + for (key, value) in + [(SCORER_PERSISTENCE_KEY, vec![1]), (EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, vec![2])] + { + primary_store + .write( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + key, + value, + ) + .await + .unwrap(); + } + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + set_test_index_store(&mut tier); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + tier.write( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + SCORER_PERSISTENCE_KEY, + vec![3], + ) + .await + .unwrap(); + assert!(primary_store + .read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + SCORER_PERSISTENCE_KEY, + ) + .await + .is_err()); + assert_eq!( + ephemeral_store + .read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + SCORER_PERSISTENCE_KEY, + ) + .await + .unwrap(), + vec![3] + ); + assert!(ephemeral_store + .read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, + ) + .await + .is_err()); + assert_eq!( + primary_store + .read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, + ) + .await + .unwrap(), + vec![2] + ); + } + + #[tokio::test] + async fn namespace_initialization_preserves_existing_primary_order_across_pages() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + for i in 0..55 { + primary_store + .write("namespace", "", &format!("existing-{i:02}"), vec![1]) + .await + .unwrap(); + } + let mut tier = setup_tier_store(primary_store, logger); + set_test_index_store(&mut tier); + + let mut actual = Vec::new(); + let mut page_token = None; + loop { + let page = + PaginatedKVStore::list_paginated(&tier, "namespace", "", page_token).await.unwrap(); + actual.extend(page.keys); + match page.next_page_token { + Some(next_page_token) => page_token = Some(next_page_token), + None => break, + } + } + + let expected = (0..55).rev().map(|i| format!("existing-{i:02}")).collect::>(); + assert_eq!(actual, expected); + assert_eq!(KVStore::list(&tier, "namespace", "").await.unwrap().len(), 55); + } + + #[tokio::test] + async fn paginated_listing_rejects_tokens_from_another_namespace_or_index() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), Arc::clone(&logger)); + let index_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + tier.set_index_store(TierStoreIndex::from_store_with_database_id( + index_store, + [1; INDEX_DATABASE_ID_LEN], + )); + for i in 0..51 { + tier.write("namespace", "", &format!("key-{i:02}"), vec![1]).await.unwrap(); + } + let token = PaginatedKVStore::list_paginated(&tier, "namespace", "", None) + .await + .unwrap() + .next_page_token + .unwrap(); + + let namespace_error = + PaginatedKVStore::list_paginated(&tier, "other-namespace", "", Some(token.clone())) + .await + .unwrap_err(); + assert_eq!(namespace_error.kind(), io::ErrorKind::InvalidInput); + + let replacement_index_store: Arc = + Arc::new(DynStoreWrapper(InMemoryStore::new())); + tier.set_index_store(TierStoreIndex::from_store_with_database_id( + replacement_index_store, + [2; INDEX_DATABASE_ID_LEN], + )); + let index_error = PaginatedKVStore::list_paginated(&tier, "namespace", "", Some(token)) + .await + .unwrap_err(); + assert_eq!(index_error.kind(), io::ErrorKind::InvalidInput); + } + + #[tokio::test] + async fn pending_creates_roll_forward_to_primary_and_backup_in_journal_order() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let index_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), Arc::clone(&logger)); + tier.set_backup_store(Arc::clone(&backup_store)); + tier.set_index_store(TierStoreIndex::from_store(Arc::clone(&index_store))); + tier.inner.ensure_namespace_indexed("namespace", "").await.unwrap(); + + for (key, value) in [("first", vec![1]), ("second", vec![2])] { + let entry = JournalEntry { + primary_namespace: "namespace".to_string(), + secondary_namespace: String::new(), + key: key.to_string(), + tier: ValueTier::Primary, + requires_backup: true, + operation: JournalOperation::Create { value: value.clone() }, + }; + tier.inner.index.as_ref().unwrap().write_journal_entry(&entry).await.unwrap(); + // Simulate the backup write winning the race before interruption. + backup_store.write("namespace", "", key, value).await.unwrap(); + } + + drop(tier); + let mut recovered_tier = setup_tier_store(Arc::clone(&primary_store), logger); + recovered_tier.set_backup_store(Arc::clone(&backup_store)); + recovered_tier.set_index_store(TierStoreIndex::from_store(index_store)); + + let page = + PaginatedKVStore::list_paginated(&recovered_tier, "namespace", "", None).await.unwrap(); + assert_eq!(page.keys, vec!["second".to_string(), "first".to_string()]); + for (key, value) in [("first", vec![1]), ("second", vec![2])] { + assert_eq!(primary_store.read("namespace", "", key).await.unwrap(), value); + assert_eq!(backup_store.read("namespace", "", key).await.unwrap(), value); + assert!(recovered_tier + .inner + .index + .as_ref() + .unwrap() + .read_journal_entry("namespace", "", key) + .await + .is_err()); + } + } + + #[tokio::test] + async fn pending_removal_hides_index_entry_before_removing_value_copies() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + tier.set_backup_store(Arc::clone(&backup_store)); + set_test_index_store(&mut tier); + tier.write("namespace", "", "key", vec![1]).await.unwrap(); + + let entry = JournalEntry { + primary_namespace: "namespace".to_string(), + secondary_namespace: String::new(), + key: "key".to_string(), + tier: ValueTier::Primary, + requires_backup: true, + operation: JournalOperation::Remove { lazy: false }, + }; + let index = tier.inner.index.as_ref().unwrap(); + index.write_journal_entry(&entry).await.unwrap(); + index.remove_entry("namespace", "", "key", false).await.unwrap(); + assert!(!index.contains_entry("namespace", "", "key").await.unwrap()); + assert!(primary_store.read("namespace", "", "key").await.is_ok()); + assert!(backup_store.read("namespace", "", "key").await.is_ok()); + assert!(index.read_journal_entry("namespace", "", "key").await.is_ok()); + + assert!(KVStore::list(&tier, "namespace", "").await.unwrap().is_empty()); + assert!(primary_store.read("namespace", "", "key").await.is_err()); + assert!(backup_store.read("namespace", "", "key").await.is_err()); + assert!(index.read_journal_entry("namespace", "", "key").await.is_err()); + } + + #[tokio::test] + async fn pending_backup_operations_roll_forward_without_the_backup_after_restart() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let index_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + for store in [&primary_store, &backup_store] { + store.write("namespace", "", "update", vec![1]).await.unwrap(); + store.write("namespace", "", "remove", vec![2]).await.unwrap(); + } + + let mut configured_tier = setup_tier_store(Arc::clone(&primary_store), Arc::clone(&logger)); + configured_tier.set_backup_store(Arc::clone(&backup_store)); + configured_tier.set_index_store(TierStoreIndex::from_store(Arc::clone(&index_store))); + assert_eq!( + configured_tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::Required + ); + configured_tier.inner.ensure_namespace_indexed("namespace", "").await.unwrap(); + let completion = configured_tier.inner.current_backup_sync_completion().await.unwrap(); + configured_tier.inner.write_backup_sync_completion(completion).await.unwrap(); + let entries = [ + ("create", JournalOperation::Create { value: vec![3] }), + ("update", JournalOperation::Update { value: vec![4] }), + ("remove", JournalOperation::Remove { lazy: false }), + ]; + for (key, operation) in entries { + configured_tier + .inner + .index + .as_ref() + .unwrap() + .write_journal_entry(&JournalEntry { + primary_namespace: "namespace".to_string(), + secondary_namespace: String::new(), + key: key.to_string(), + tier: ValueTier::Primary, + requires_backup: true, + operation, + }) + .await + .unwrap(); + } + drop(configured_tier); + + let mut primary_only_tier = setup_tier_store(Arc::clone(&primary_store), logger); + primary_only_tier.set_index_store(TierStoreIndex::from_store(index_store)); + assert_eq!( + primary_only_tier.initialize_backup_synchronization().await.unwrap(), + BackupSyncStatus::NotConfigured + ); + let keys = KVStore::list(&primary_only_tier, "namespace", "").await.unwrap(); + + assert!(keys.contains(&"create".to_string())); + assert!(keys.contains(&"update".to_string())); + assert!(!keys.contains(&"remove".to_string())); + assert_eq!(primary_store.read("namespace", "", "create").await.unwrap(), vec![3]); + assert_eq!(primary_store.read("namespace", "", "update").await.unwrap(), vec![4]); + assert!(primary_store.read("namespace", "", "remove").await.is_err()); + assert!(backup_store.read("namespace", "", "create").await.is_err()); + assert_eq!(backup_store.read("namespace", "", "update").await.unwrap(), vec![1]); + assert_eq!(backup_store.read("namespace", "", "remove").await.unwrap(), vec![2]); + let index = primary_only_tier.inner.index.as_ref().unwrap(); + for key in ["create", "update", "remove"] { + assert!(index.read_journal_entry("namespace", "", key).await.is_err()); + } + let rotated_generation_id = + TierStoreInner::read_generation_id(primary_store.as_ref(), PRIMARY_SYNC_GENERATION_KEY) + .await + .unwrap(); + assert_ne!(rotated_generation_id, completion.primary_generation_id); + assert_eq!(read_test_backup_sync_completion(backup_store.as_ref()).await, completion); + } + + #[tokio::test] + async fn pending_backup_recovery_requires_synchronization_initialization() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + primary_store.write("namespace", "", "key", vec![1]).await.unwrap(); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + set_test_index_store(&mut tier); + tier.inner.ensure_namespace_indexed("namespace", "").await.unwrap(); + let pending = JournalEntry { + primary_namespace: "namespace".to_string(), + secondary_namespace: String::new(), + key: "key".to_string(), + tier: ValueTier::Primary, + requires_backup: true, + operation: JournalOperation::Update { value: vec![2] }, + }; + let index = tier.inner.index.as_ref().unwrap(); + index.write_journal_entry(&pending).await.unwrap(); + assert_eq!(*tier.inner.backup_sync_status.lock().unwrap(), None); + + let error = tier.read("namespace", "", "key").await.unwrap_err(); + + assert_eq!(error.kind(), io::ErrorKind::NotFound); + assert_eq!(primary_store.read("namespace", "", "key").await.unwrap(), vec![1]); + assert!(matches!( + index.read_journal_entry("namespace", "", "key").await.unwrap().operation, + JournalOperation::Update { value } if value == vec![2] + )); + } + + #[tokio::test] + async fn read_recovers_only_the_requested_key() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + for (key, value) in [("stuck", vec![1]), ("readable", vec![2])] { + primary_store.write("namespace", "", key, value).await.unwrap(); + } + let mut tier = setup_tier_store(primary_store, logger); + set_test_index_store(&mut tier); + tier.inner.ensure_namespace_indexed("namespace", "").await.unwrap(); + let pending = JournalEntry { + primary_namespace: "namespace".to_string(), + secondary_namespace: String::new(), + key: "stuck".to_string(), + tier: ValueTier::Primary, + requires_backup: true, + operation: JournalOperation::Remove { lazy: false }, + }; + let index = tier.inner.index.as_ref().unwrap(); + index.write_journal_entry(&pending).await.unwrap(); + + assert_eq!(tier.read("namespace", "", "readable").await.unwrap(), vec![2]); + assert!(index.read_journal_entry("namespace", "", "stuck").await.is_ok()); + assert!(tier.read("namespace", "", "stuck").await.is_err()); + } + + #[tokio::test] + async fn writes_and_removals_recover_only_the_requested_key() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + for (key, value) in [("stuck", vec![1]), ("writable", vec![2]), ("removable", vec![3])] { + primary_store.write("namespace", "", key, value).await.unwrap(); + } + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + set_test_index_store(&mut tier); + tier.inner.ensure_namespace_indexed("namespace", "").await.unwrap(); + let pending = JournalEntry { + primary_namespace: "namespace".to_string(), + secondary_namespace: String::new(), + key: "stuck".to_string(), + tier: ValueTier::Primary, + requires_backup: true, + operation: JournalOperation::Remove { lazy: false }, + }; + let index = tier.inner.index.as_ref().unwrap(); + index.write_journal_entry(&pending).await.unwrap(); + + tier.write("namespace", "", "writable", vec![4]).await.unwrap(); + tier.remove("namespace", "", "removable", false).await.unwrap(); + + assert_eq!(primary_store.read("namespace", "", "writable").await.unwrap(), vec![4]); + assert!(primary_store.read("namespace", "", "removable").await.is_err()); + assert!(index.read_journal_entry("namespace", "", "stuck").await.is_ok()); + assert!(KVStore::list(&tier, "namespace", "").await.is_err()); + } + + #[tokio::test] + async fn read_recovers_pending_removal_before_returning_value() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + tier.set_backup_store(Arc::clone(&backup_store)); + set_test_index_store(&mut tier); + tier.write("namespace", "", "key", vec![1]).await.unwrap(); + + let pending = JournalEntry { + primary_namespace: "namespace".to_string(), + secondary_namespace: String::new(), + key: "key".to_string(), + tier: ValueTier::Primary, + requires_backup: true, + operation: JournalOperation::Remove { lazy: false }, + }; + let index = tier.inner.index.as_ref().unwrap(); + index.write_journal_entry(&pending).await.unwrap(); + index.remove_entry("namespace", "", "key", false).await.unwrap(); + + let error = tier.read("namespace", "", "key").await.unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::NotFound); + assert!(primary_store.read("namespace", "", "key").await.is_err()); + assert!(backup_store.read("namespace", "", "key").await.is_err()); + assert!(index.read_journal_entry("namespace", "", "key").await.is_err()); + } + + #[tokio::test] + async fn read_recovers_pending_cache_operation_without_relocking_the_key() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + primary_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + vec![1], + ) + .await + .unwrap(); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + set_test_index_store(&mut tier); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + tier.inner + .ensure_namespace_indexed( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + ) + .await + .unwrap(); + let pending = JournalEntry { + primary_namespace: NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE.to_string(), + secondary_namespace: NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE.to_string(), + key: NETWORK_GRAPH_PERSISTENCE_KEY.to_string(), + tier: ValueTier::Primary, + requires_backup: false, + operation: JournalOperation::Create { value: vec![2] }, + }; + let index = tier.inner.index.as_ref().unwrap(); + index.write_journal_entry(&pending).await.unwrap(); + + let value = tokio::time::timeout( + std::time::Duration::from_secs(5), + tier.read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ), + ) + .await + .expect("read deadlocked while preparing its cache key") + .unwrap(); + assert_eq!(value, vec![2]); + assert!(primary_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .is_err()); + assert_eq!( + ephemeral_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .unwrap(), + vec![2] + ); + assert!(index + .read_journal_entry( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .is_err()); + } + + #[tokio::test] + async fn failed_create_retains_journal_without_exposing_index_entry() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + let attempts = Arc::new(AtomicUsize::new(0)); + let backup_store: Arc = + Arc::new(DynStoreWrapper(InstrumentedStore::new(StoreBehavior::FailWrite { + attempts: Arc::clone(&attempts), + }))); + tier.set_backup_store(backup_store); + set_test_index_store(&mut tier); + + assert!(tier.write("namespace", "", "key", vec![1]).await.is_err()); + let index = tier.inner.index.as_ref().unwrap(); + assert!(primary_store.read("namespace", "", "key").await.is_ok()); + assert!(!index.contains_entry("namespace", "", "key").await.unwrap()); + assert!(matches!( + index.read_journal_entry("namespace", "", "key").await.unwrap().operation, + JournalOperation::Create { .. } + )); + assert_eq!(attempts.load(Ordering::Relaxed), 1); + } + + #[tokio::test] + async fn failed_update_retains_journal_without_changing_index_membership() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + set_test_index_store(&mut tier); + tier.write("namespace", "", "key", vec![1]).await.unwrap(); + let attempts = Arc::new(AtomicUsize::new(0)); + let backup_store: Arc = + Arc::new(DynStoreWrapper(InstrumentedStore::new(StoreBehavior::FailWrite { + attempts: Arc::clone(&attempts), + }))); + tier.set_backup_store(backup_store); + + assert!(tier.write("namespace", "", "key", vec![2]).await.is_err()); + let index = tier.inner.index.as_ref().unwrap(); + assert_eq!(primary_store.read("namespace", "", "key").await.unwrap(), vec![2]); + assert!(index.contains_entry("namespace", "", "key").await.unwrap()); + assert!(matches!( + index.read_journal_entry("namespace", "", "key").await.unwrap().operation, + JournalOperation::Update { value } if value == vec![2] + )); + assert_eq!(attempts.load(Ordering::Relaxed), 1); + } + + #[tokio::test] + async fn write_recovers_pending_create_under_key_lock_before_classifying() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let index_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + tier.set_index_store(TierStoreIndex::from_store(index_store)); + tier.inner.ensure_namespace_indexed("namespace", "").await.unwrap(); + + let pending = JournalEntry { + primary_namespace: "namespace".to_string(), + secondary_namespace: String::new(), + key: "key".to_string(), + tier: ValueTier::Primary, + requires_backup: false, + operation: JournalOperation::Create { value: vec![1] }, + }; + let index = tier.inner.index.as_ref().unwrap(); + index.write_journal_entry(&pending).await.unwrap(); + primary_store.write("namespace", "", "key", vec![1]).await.unwrap(); + + tier.write("namespace", "", "key", vec![2]).await.unwrap(); + assert_eq!(primary_store.read("namespace", "", "key").await.unwrap(), vec![2]); + assert!(index.contains_entry("namespace", "", "key").await.unwrap()); + assert!(index.read_journal_entry("namespace", "", "key").await.is_err()); + } + + #[tokio::test] + async fn update_rejects_value_without_index_entry() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + set_test_index_store(&mut tier); + tier.write("namespace", "", "key", vec![1]).await.unwrap(); + tier.inner + .index + .as_ref() + .unwrap() + .remove_entry("namespace", "", "key", false) + .await + .unwrap(); + + let error = tier.write("namespace", "", "key", vec![2]).await.unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert_eq!(primary_store.read("namespace", "", "key").await.unwrap(), vec![1]); + } + + #[tokio::test] + async fn failed_removal_retains_journal_and_hides_index_entry() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let backup_store: Arc = + Arc::new(DynStoreWrapper(InstrumentedStore::new(StoreBehavior::FailRemove))); + let mut tier = setup_tier_store(primary_store, logger); + tier.set_backup_store(backup_store); + set_test_index_store(&mut tier); + tier.write("namespace", "", "key", vec![1]).await.unwrap(); + + assert!(tier.remove("namespace", "", "key", false).await.is_err()); + let index = tier.inner.index.as_ref().unwrap(); + assert!(!index.contains_entry("namespace", "", "key").await.unwrap()); + assert!(matches!( + index.read_journal_entry("namespace", "", "key").await.unwrap().operation, + JournalOperation::Remove { .. } + )); + assert!(KVStore::list(&tier, "namespace", "").await.is_err()); + } + + enum StoreBehavior { + FailList, + FailWrite { + attempts: Arc, + }, + FailWriteOnce { + primary_namespace: &'static str, + key: &'static str, + attempts: Arc, + }, + FailRemove, + FailRemoveOnce { + attempts: Arc, + }, + } + + /// A store that injects selected failures or synchronization points while delegating other + /// operations to an inner [`InMemoryStore`]. + struct InstrumentedStore { + inner: InMemoryStore, + behavior: StoreBehavior, + } + + impl InstrumentedStore { + fn new(behavior: StoreBehavior) -> Self { + Self { inner: InMemoryStore::new(), behavior } + } + } + + impl KVStore for InstrumentedStore { + fn read( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, + ) -> impl Future, io::Error>> + 'static + Send { + KVStore::read(&self.inner, primary_namespace, secondary_namespace, key) + } + fn write( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec, + ) -> impl Future> + 'static + Send { + let write = match &self.behavior { + StoreBehavior::FailWrite { attempts } => { + attempts.fetch_add(1, Ordering::Relaxed); + None + }, + StoreBehavior::FailWriteOnce { + primary_namespace: failed_namespace, + key: failed_key, + attempts, + } if primary_namespace == *failed_namespace + && key == *failed_key + && attempts.fetch_add(1, Ordering::Relaxed) == 0 => + { + None + }, + _ => Some(KVStore::write( + &self.inner, + primary_namespace, + secondary_namespace, + key, + buf, + )), + }; + async move { + match write { + Some(write) => write.await, + None => Err(io::Error::new(io::ErrorKind::Other, "write failed")), + } + } + } + fn remove( + &self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool, + ) -> impl Future> + 'static + Send { + let remove = match &self.behavior { + StoreBehavior::FailRemove => None, + StoreBehavior::FailRemoveOnce { attempts } + if attempts.fetch_add(1, Ordering::Relaxed) == 0 => + { + None + }, + _ => Some(KVStore::remove( + &self.inner, + primary_namespace, + secondary_namespace, + key, + lazy, + )), + }; + async move { + match remove { + Some(remove) => remove.await, + None => Err(io::Error::new(io::ErrorKind::Other, "remove failed")), + } + } + } + fn list( + &self, primary_namespace: &str, secondary_namespace: &str, + ) -> impl Future, io::Error>> + 'static + Send { + let list = match &self.behavior { + StoreBehavior::FailList => None, + StoreBehavior::FailWrite { .. } + | StoreBehavior::FailWriteOnce { .. } + | StoreBehavior::FailRemove + | StoreBehavior::FailRemoveOnce { .. } => { + Some(KVStore::list(&self.inner, primary_namespace, secondary_namespace)) + }, + }; + async move { + match list { + Some(list) => list.await, + None => Err(io::Error::new(io::ErrorKind::Other, "list failed")), + } + } + } + } + + impl PaginatedKVStore for InstrumentedStore { + fn list_paginated( + &self, primary_namespace: &str, secondary_namespace: &str, + page_token: Option, + ) -> impl Future> + 'static + Send { + let fails = matches!(&self.behavior, StoreBehavior::FailList); + let list = PaginatedKVStore::list_paginated( + &self.inner, + primary_namespace, + secondary_namespace, + page_token, + ); + async move { + if fails { + return Err(io::Error::new(io::ErrorKind::Other, "list_paginated failed")); + } + list.await + } + } + } + + impl MigratableKVStore for InstrumentedStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send + { + MigratableKVStore::list_all_keys(&self.inner) + } + } + + #[tokio::test] + async fn write_read_list_remove() { + let (base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); + let tier = setup_tier_store(primary_store, logger); + + do_read_write_remove_list_persist(&tier).await; + } + + #[tokio::test] + async fn ephemeral_routing() { + let (base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + + let ephemeral_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("ephemeral")).unwrap())); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + let data = vec![42u8; 32]; + + tier.write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + data.clone(), + ) + .await + .unwrap(); + + tier.write( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + data.clone(), + ) + .await + .unwrap(); + + let primary_read_ng = primary_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await; + let ephemeral_read_ng = ephemeral_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await; + + let primary_read_cm = primary_store + .read( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + ) + .await; + let ephemeral_read_cm = ephemeral_store + .read( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + ) + .await; + + assert!(primary_read_ng.is_err()); + assert_eq!(ephemeral_read_ng.unwrap(), data); + + assert!(ephemeral_read_cm.is_err()); + assert_eq!(primary_read_cm.unwrap(), data); + } + + #[tokio::test] + async fn external_pathfinding_scores_cache_routes_to_ephemeral_store() { + let (base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + + let ephemeral_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("ephemeral")).unwrap())); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + let data = vec![42u8; 32]; + tier.write( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, + data.clone(), + ) + .await + .unwrap(); + + assert!(primary_store + .read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, + ) + .await + .is_err()); + assert_eq!( + tier.read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, + ) + .await + .unwrap(), + data + ); + + tier.remove( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, + false, + ) + .await + .unwrap(); + assert!(ephemeral_store + .read( + SCORER_PERSISTENCE_PRIMARY_NAMESPACE, + SCORER_PERSISTENCE_SECONDARY_NAMESPACE, + EXTERNAL_PATHFINDING_SCORES_CACHE_KEY, + ) + .await + .is_err()); + } + + #[tokio::test] + async fn list_exposes_primary_and_routed_ephemeral_keys() { + let (base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + + let ephemeral_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("ephemeral")).unwrap())); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + // A durable root-namespace key, routed to primary since it isn't ephemeral-cached. + tier.write( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + vec![1u8; 32], + ) + .await + .unwrap(); + + // The ephemeral-cached key, routed to the ephemeral store. + tier.write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + vec![2u8; 32], + ) + .await + .unwrap(); + + // A decoy sitting in the ephemeral store under an unrelated namespace. This must + // never leak into a listing for that namespace just because an ephemeral + // store happens to be configured. + ephemeral_store + .write( + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + "ephemeral-decoy", + vec![3u8; 32], + ) + .await + .unwrap(); + ephemeral_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + "ephemeral-root-decoy", + vec![4u8; 32], + ) + .await + .unwrap(); + + // This is `list("", "")`: CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE and + // NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE are the same empty string, so both + // keys live in the exact namespace. + let root_keys = KVStore::list( + &tier, + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + ) + .await + .unwrap(); + + // Unpaginated listing exposes the logical view across both tiers without leaking + // unrelated keys from the ephemeral store. + assert!(root_keys.contains(&CHANNEL_MANAGER_PERSISTENCE_KEY.to_string())); + assert!(root_keys.contains(&NETWORK_GRAPH_PERSISTENCE_KEY.to_string())); + assert!(!root_keys.contains(&"ephemeral-root-decoy".to_string())); + + let monitor_keys = KVStore::list( + &tier, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + ) + .await + .unwrap(); + + // The unrelated-namespace decoy sitting in the ephemeral store must not leak + // into a listing for a namespace it was never routed to. + assert!(!monitor_keys.contains(&"ephemeral-decoy".to_string())); + } + + #[tokio::test] + async fn list_paginated_only_exposes_primary_keys() { + let (base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + + let ephemeral_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("ephemeral")).unwrap())); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + tier.write( + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + "monitor-key", + vec![1u8; 32], + ) + .await + .unwrap(); + + // This decoy uses the same namespace but the opposite physical store, so it + // would show up if paginated listing routed to the wrong tier. + ephemeral_store + .write( + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + "ephemeral-decoy", + vec![2u8; 32], + ) + .await + .unwrap(); + + // This key shares the network graph's namespace tuple ("", "") but is not + // itself an ephemeral-cached key, standing in for durable root-namespace data + // such as `manager`/`output_sweeper`/`peers`. It must still be listed even + // though the ephemeral store is configured and authoritative for + // `network_graph`/`scorer` specifically. + primary_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + "other-root-namespace-key", + vec![3u8; 32], + ) + .await + .unwrap(); + + tier.write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + vec![4u8; 32], + ) + .await + .unwrap(); + + let primary_response = PaginatedKVStore::list_paginated( + &tier, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + None, + ) + .await + .unwrap(); + assert_eq!(primary_response.keys, vec!["monitor-key".to_string()]); + + let root_response = PaginatedKVStore::list_paginated( + &tier, + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + None, + ) + .await + .unwrap(); + + assert_eq!(root_response.keys, vec!["other-root-namespace-key".to_string()]); + } + + #[tokio::test] + async fn listings_only_consult_ephemeral_store_for_routed_namespaces() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + + // An ephemeral store whose `list`/`list_paginated` always fail. + let ephemeral_store: Arc = + Arc::new(DynStoreWrapper(InstrumentedStore::new(StoreBehavior::FailList))); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + // A durable key in a namespace that can never hold an ephemeral-cached key. + tier.write( + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + "monitor-key", + vec![1u8; 32], + ) + .await + .unwrap(); + + // Listing that namespace must not consult (or depend on) the ephemeral store, so it + // succeeds even though the ephemeral list would fail. + let monitor_keys = KVStore::list( + &tier, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + ) + .await + .unwrap(); + assert_eq!(monitor_keys, vec!["monitor-key".to_string()]); + + // The paginated path always exposes only the primary store. + let monitor_page = PaginatedKVStore::list_paginated( + &tier, + CHANNEL_MONITOR_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MONITOR_PERSISTENCE_SECONDARY_NAMESPACE, + None, + ) + .await + .unwrap(); + assert_eq!(monitor_page.keys, vec!["monitor-key".to_string()]); + + // An unpaginated root listing must consult the authoritative ephemeral store. + assert!(KVStore::list( + &tier, + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + ) + .await + .is_err()); + } + + #[tokio::test] + async fn list_hides_stale_primary_copy_when_ephemeral_key_is_missing() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + + let ephemeral_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + tier.set_ephemeral_store(Arc::clone(&ephemeral_store)); + + // A durable root-namespace key that must always be discoverable. + tier.write( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + vec![1u8; 32], + ) + .await + .unwrap(); + + // A stale copy of `network_graph` sitting in primary as if it had been persisted there + // before the ephemeral store was configured. The ephemeral store holds no copy, so + // `read` routes to ephemeral and would fail for this key. + primary_store + .write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + vec![2u8; 32], + ) + .await + .unwrap(); + + let root_keys = KVStore::list( + &tier, + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + ) + .await + .unwrap(); + + // The ephemeral store is authoritative for routed keys, so its missing entry hides + // the stale primary copy. + assert!(root_keys.contains(&CHANNEL_MANAGER_PERSISTENCE_KEY.to_string())); + assert!(!root_keys.contains(&NETWORK_GRAPH_PERSISTENCE_KEY.to_string())); + } + + #[tokio::test] + async fn primary_backed_writes_preserve_latest_call_order() { + let (base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); + let tier = setup_tier_store(primary_store, logger); + + let old_data = vec![1u8; 32]; + let new_data = vec![2u8; 32]; + + let old_write = tier.write( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + old_data, + ); + let new_write = tier.write( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + new_data.clone(), + ); + + new_write.await.unwrap(); + old_write.await.unwrap(); + + // Stale data doesn't overwrite latest + let persisted = tier + .read( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + ) + .await + .unwrap(); + assert_eq!(persisted, new_data); + } + + #[tokio::test] + async fn failed_newer_backup_write_still_supersedes_older_write() { + let (_base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + + let backup_write_attempts = Arc::new(AtomicUsize::new(0)); + let backup_store: Arc = + Arc::new(DynStoreWrapper(InstrumentedStore::new(StoreBehavior::FailWrite { + attempts: Arc::clone(&backup_write_attempts), + }))); + tier.set_backup_store(backup_store); + + let old_data = vec![1u8; 32]; + let new_data = vec![2u8; 32]; + let old_write = tier.write( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + old_data, + ); + let new_write = tier.write( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + new_data.clone(), + ); + + // The primary write succeeds, but the same newer write fails on the backup. + assert!(new_write.await.is_err()); + // The older operation must be treated as stale even though the newer operation failed. + old_write.await.unwrap(); + + let persisted = primary_store + .read( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + ) + .await + .unwrap(); + assert_eq!(persisted, new_data); + assert_eq!(backup_write_attempts.load(Ordering::Relaxed), 1); + } + + #[tokio::test] + async fn ephemeral_writes_preserve_latest_call_order() { + let (base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); + let mut tier = setup_tier_store(primary_store, logger); + + let ephemeral_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("ephemeral")).unwrap())); + tier.set_ephemeral_store(ephemeral_store); + + let old_data = vec![1u8; 32]; + let new_data = vec![2u8; 32]; + + let old_write = tier.write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + old_data, + ); + let new_write = tier.write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + new_data.clone(), + ); + + new_write.await.unwrap(); + old_write.await.unwrap(); + + let persisted = tier + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .unwrap(); + assert_eq!(persisted, new_data); + } + + #[tokio::test] + async fn ephemeral_removes_preserve_latest_call_order() { + let (base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); + let mut tier = setup_tier_store(primary_store, logger); + + let ephemeral_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("ephemeral")).unwrap())); + tier.set_ephemeral_store(ephemeral_store); + + let data = vec![2u8; 32]; + + let stale_remove = tier.remove( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + true, + ); + let new_write = tier.write( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + data.clone(), + ); + + new_write.await.unwrap(); + stale_remove.await.unwrap(); + + let persisted = tier + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .unwrap(); + assert_eq!(persisted, data); + } + + #[tokio::test] + async fn backup_write_is_part_of_success_path() { + let (base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + + let backup_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("backup")).unwrap())); + tier.set_backup_store(Arc::clone(&backup_store)); + + let data = vec![42u8; 32]; + + tier.write( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + data.clone(), + ) + .await + .unwrap(); + + let primary_read = primary_store + .read( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + ) + .await; + let backup_read = backup_store + .read( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + ) + .await; + + assert_eq!(primary_read.unwrap(), data); + assert_eq!(backup_read.unwrap(), data); + } + + #[tokio::test] + async fn backup_remove_is_part_of_success_path() { + let (base_dir, logger, _cleanup) = setup_test_environment(); + + let primary_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("primary")).unwrap())); + let mut tier = setup_tier_store(Arc::clone(&primary_store), logger); + + let backup_store: Arc = + Arc::new(DynStoreWrapper(FilesystemStoreV2::new(base_dir.join("backup")).unwrap())); + tier.set_backup_store(Arc::clone(&backup_store)); + + let data = vec![42u8; 32]; + let key = CHANNEL_MANAGER_PERSISTENCE_KEY; + + tier.write( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + key, + data, + ) + .await + .unwrap(); + + tier.remove( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + key, + true, + ) + .await + .unwrap(); + + let primary_read = primary_store + .read( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + key, + ) + .await; + let backup_read = backup_store + .read( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + key, + ) + .await; + + assert!(primary_read.is_err()); + assert!(backup_read.is_err()); + } +} diff --git a/src/io/utils.rs b/src/io/utils.rs index 30fc0c62d2..004590284c 100644 --- a/src/io/utils.rs +++ b/src/io/utils.rs @@ -780,7 +780,9 @@ mod read_objects_tests { use std::sync::Arc; use lightning::impl_writeable_tlv_based; - use lightning::util::persist::{KVStore, PageToken, PaginatedKVStore, PaginatedListResponse}; + use lightning::util::persist::{ + KVStore, MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse, + }; use lightning::util::ser::Writeable; use lightning::util::test_utils::TestLogger; @@ -980,6 +982,18 @@ mod read_objects_tests { } } + impl MigratableKVStore for StuckTokenStore { + fn list_all_keys( + &self, + ) -> impl std::future::Future< + Output = Result, lightning::io::Error>, + > + + 'static + + Send { + MigratableKVStore::list_all_keys(&self.inner) + } + } + #[tokio::test] async fn a_page_token_that_does_not_advance_is_an_error() { let stuck = StuckTokenStore { diff --git a/src/lib.rs b/src/lib.rs index 821304a532..c43598c868 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -147,6 +147,8 @@ use fee_estimator::{ }; #[cfg(feature = "uniffi")] use ffi::*; +#[cfg(all(feature = "uniffi", feature = "storage-tier"))] +pub use ffi::{DynStoreTrait, IOError, KVStoreKey, PaginatedListResponse}; use gossip::GossipSource; use graph::NetworkGraph; use io::utils::update_and_persist_node_metrics; diff --git a/src/peer_store.rs b/src/peer_store.rs index 8345bf7111..716e6d6008 100644 --- a/src/peer_store.rs +++ b/src/peer_store.rs @@ -174,7 +174,9 @@ mod tests { use std::sync::Arc; use bitcoin::io; - use lightning::util::persist::{PageToken, PaginatedKVStore, PaginatedListResponse}; + use lightning::util::persist::{ + MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse, + }; use lightning::util::test_utils::TestLogger; use super::*; @@ -219,6 +221,16 @@ mod tests { } } + impl MigratableKVStore for FailingStore { + fn list_all_keys( + &self, + ) -> impl std::future::Future, io::Error>> + + 'static + + Send { + async { Err(io::Error::new(io::ErrorKind::Other, "list_all_keys failed")) } + } + } + #[tokio::test] async fn peer_info_persistence() { let store: Arc = Arc::new(DynStoreWrapper(InMemoryStore::new())); diff --git a/src/types.rs b/src/types.rs index 1a61daa109..df292b369c 100644 --- a/src/types.rs +++ b/src/types.rs @@ -25,6 +25,8 @@ use lightning::routing::gossip; use lightning::routing::router::DefaultRouter; use lightning::routing::scoring::{CombinedScorer, ProbabilisticScoringFeeParameters}; use lightning::sign::InMemorySigner; +#[cfg(feature = "storage-tier")] +use lightning::util::persist::MigratableKVStore; use lightning::util::persist::{ KVStore, MonitorUpdatingPersisterAsync, PageToken, PaginatedKVStore, PaginatedListResponse, }; @@ -72,6 +74,16 @@ pub(crate) trait DynStoreTrait: Send + Sync { dyn Future> + Send + 'static, >, >; + #[cfg(feature = "storage-tier")] + fn list_all_keys_async( + &self, + ) -> Pin< + Box< + dyn Future, bitcoin::io::Error>> + + Send + + 'static, + >, + >; } impl<'a> KVStore for dyn DynStoreTrait + 'a { @@ -113,6 +125,16 @@ impl<'a> PaginatedKVStore for dyn DynStoreTrait + 'a { } } +#[cfg(feature = "storage-tier")] +impl<'a> MigratableKVStore for dyn DynStoreTrait + 'a { + fn list_all_keys( + &self, + ) -> impl Future, bitcoin::io::Error>> + Send + 'static + { + DynStoreTrait::list_all_keys_async(self) + } +} + pub(crate) type DynStore = dyn DynStoreTrait; // Newtype wrapper that implements `KVStore` for `Arc`. This is needed because `KVStore` @@ -161,9 +183,34 @@ impl PaginatedKVStore for DynStoreRef { } } +#[cfg(feature = "storage-tier")] +impl MigratableKVStore for DynStoreRef { + fn list_all_keys( + &self, + ) -> impl Future, bitcoin::io::Error>> + Send + 'static + { + DynStoreTrait::list_all_keys_async(&*self.0) + } +} + pub(crate) struct DynStoreWrapper(pub(crate) T); -impl DynStoreTrait for DynStoreWrapper { +// With tiered storage enabled, dynamic stores must support exhaustive key listing so backup +// synchronization can copy every primary value. Without tiered storage, stores only need to +// implement `PaginatedKVStore`, preserving compatibility for existing custom stores. +#[cfg(not(feature = "storage-tier"))] +trait DynStoreSource: PaginatedKVStore + Send + Sync {} + +#[cfg(feature = "storage-tier")] +trait DynStoreSource: PaginatedKVStore + MigratableKVStore + Send + Sync {} + +#[cfg(not(feature = "storage-tier"))] +impl DynStoreSource for T {} + +#[cfg(feature = "storage-tier")] +impl DynStoreSource for T {} + +impl DynStoreTrait for DynStoreWrapper { fn read_async( &self, primary_namespace: &str, secondary_namespace: &str, key: &str, ) -> Pin, bitcoin::io::Error>> + Send + 'static>> { @@ -202,6 +249,19 @@ impl DynStoreTrait for DynStoreWrapper { page_token, )) } + + #[cfg(feature = "storage-tier")] + fn list_all_keys_async( + &self, + ) -> Pin< + Box< + dyn Future, bitcoin::io::Error>> + + Send + + 'static, + >, + > { + Box::pin(MigratableKVStore::list_all_keys(&self.0)) + } } pub(crate) type AsyncPersister = MonitorUpdatingPersisterAsync< diff --git a/src/wallet/mod.rs b/src/wallet/mod.rs index b9c12b4a7f..d6d340aed3 100644 --- a/src/wallet/mod.rs +++ b/src/wallet/mod.rs @@ -2695,7 +2695,9 @@ mod tests { use bitcoin::hashes::Hash; use bitcoin::Network; use lightning::io; - use lightning::util::persist::{KVStore, PageToken, PaginatedKVStore, PaginatedListResponse}; + use lightning::util::persist::{ + KVStore, MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse, + }; use super::*; #[cfg(all(not(feature = "chain-esplora"), feature = "chain-electrum"))] @@ -2783,6 +2785,15 @@ mod tests { } } + impl MigratableKVStore for FailSwitchStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send + { + MigratableKVStore::list_all_keys(&*self.inner) + } + } + /// Constructs a `Wallet` around the given store, either creating a fresh BDK wallet or /// loading the one the store already holds. async fn new_test_wallet(store: Arc, load_existing: bool) -> Arc { @@ -3076,6 +3087,16 @@ mod tests { } } + impl MigratableKVStore for SnapshotStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send + { + let keys = self.data.lock().unwrap().keys().cloned().collect(); + async move { Ok(keys) } + } + } + #[tokio::test] async fn pool_survives_a_crash_at_any_point_during_refill() { let snapshot_store = SnapshotStore::new(); @@ -3190,6 +3211,15 @@ mod tests { } } + impl MigratableKVStore for GatedStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send + { + MigratableKVStore::list_all_keys(&*self.inner) + } + } + #[tokio::test] async fn aborting_a_refill_mid_persist_loses_no_reveals() { let gated_store = GatedStore::new(); @@ -3398,6 +3428,15 @@ mod tests { } } + impl MigratableKVStore for RecordOnlyStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send + { + MigratableKVStore::list_all_keys(&*self.inner) + } + } + #[tokio::test] async fn failed_get_new_address_leaves_the_pool_record_covering_the_pool() { let record_store = RecordOnlyStore::new(); @@ -3514,6 +3553,15 @@ mod tests { } } + impl MigratableKVStore for RecordFailStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send + { + MigratableKVStore::list_all_keys(&*self.inner) + } + } + #[tokio::test] async fn crash_after_a_failed_record_write_re_derives_the_same_indices() { let record_store = RecordFailStore::new(); @@ -3660,6 +3708,15 @@ mod tests { } } + impl MigratableKVStore for NamespaceGatedStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send + { + MigratableKVStore::list_all_keys(&*self.inner) + } + } + fn dummy_tx() -> Transaction { Transaction { version: bitcoin::transaction::Version::TWO, diff --git a/src/wallet/persist.rs b/src/wallet/persist.rs index 6384d0fcee..80c4c973a5 100644 --- a/src/wallet/persist.rs +++ b/src/wallet/persist.rs @@ -314,7 +314,9 @@ mod tests { use bdk_wallet::{AsyncWalletPersister, ChangeSet, Wallet as BdkWallet}; use bitcoin::Network; use lightning::io; - use lightning::util::persist::{KVStore, PageToken, PaginatedKVStore, PaginatedListResponse}; + use lightning::util::persist::{ + KVStore, MigratableKVStore, PageToken, PaginatedKVStore, PaginatedListResponse, + }; use super::KVStoreWalletPersister; use crate::io::test_utils::InMemoryStore; @@ -378,6 +380,15 @@ mod tests { } } + impl MigratableKVStore for GatedStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send + { + MigratableKVStore::list_all_keys(&*self.inner) + } + } + #[tokio::test] async fn retains_pending_changes_when_persist_is_cancelled() { let gated_store = GatedStore { diff --git a/tests/common/mod.rs b/tests/common/mod.rs index af6a49c69a..522f6645b1 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -53,6 +53,8 @@ use ldk_node::payment::{ PaymentDetails, PaymentDirection, PaymentKind, PaymentStatus, TransactionType, }; use ldk_node::probing::ProbingConfig; +#[cfg(all(feature = "uniffi", feature = "storage-tier"))] +use ldk_node::DynStoreTrait; use ldk_node::{ Builder, ChannelShutdownState, CustomTlvRecord, Event, LightningBalance, Node, NodeError, PendingSweepBalance, UserChannelId, @@ -60,9 +62,11 @@ use ldk_node::{ use lightning::io; use lightning::ln::msgs::SocketAddress; use lightning::routing::gossip::NodeAlias; +#[cfg(feature = "storage-tier")] +use lightning::util::persist::MigratableKVStore; use lightning::util::persist::{KVStore, PageToken, PaginatedKVStore, PaginatedListResponse}; use lightning_invoice::{Bolt11InvoiceDescription, Description}; -use lightning_persister::fs_store::v1::FilesystemStore; +use lightning_persister::fs_store::v2::FilesystemStoreV2; use lightning_types::payment::{PaymentHash, PaymentPreimage}; use logging::TestLogWriter; use rand::distr::Alphanumeric; @@ -717,6 +721,19 @@ macro_rules! setup_builder { pub(crate) use setup_builder; +#[cfg(all(feature = "uniffi", feature = "storage-tier"))] +pub(crate) fn into_builder_store(store: S) -> Arc +where + S: PaginatedKVStore + MigratableKVStore + Send + Sync + 'static, +{ + Arc::new(store) +} + +#[cfg(not(all(feature = "uniffi", feature = "storage-tier")))] +pub(crate) fn into_builder_store(store: S) -> S { + store +} + pub(crate) fn configure_chain_source( chain_source: &TestChainSource, builder: &mut Builder, config: &TestConfig, ) { @@ -860,7 +877,9 @@ pub(crate) fn setup_node(chain_source: &TestChainSource, config: TestConfig) -> let node = match config.store_type { TestStoreType::TestSyncStore => { let kv_store = TestSyncStore::new(config.node_config.storage_dir_path.into()); - builder.build_with_store(config.node_entropy.into(), kv_store).unwrap() + builder + .build_with_store(config.node_entropy.into(), into_builder_store(kv_store)) + .unwrap() }, #[cfg(feature = "storage-sqlite")] TestStoreType::Sqlite => builder.build(config.node_entropy.into()).unwrap(), @@ -1954,10 +1973,23 @@ impl PaginatedKVStore for TestSyncStore { } } +#[cfg(feature = "storage-tier")] +impl MigratableKVStore for TestSyncStore { + fn list_all_keys( + &self, + ) -> impl Future, io::Error>> + 'static + Send { + let inner = Arc::clone(&self.inner); + async move { + let _guard = inner.serializer.read().await; + MigratableKVStore::list_all_keys(&inner.test_store).await + } + } +} + struct TestSyncStoreInner { serializer: tokio::sync::RwLock<()>, test_store: InMemoryStore, - fs_store: FilesystemStore, + fs_store: FilesystemStoreV2, #[cfg(feature = "storage-sqlite")] sqlite_store: SqliteStore, } @@ -1967,7 +1999,7 @@ impl TestSyncStoreInner { let serializer = tokio::sync::RwLock::new(()); let mut fs_dir = dest_dir.clone(); fs_dir.push("fs_store"); - let fs_store = FilesystemStore::new(fs_dir); + let fs_store = FilesystemStoreV2::new(fs_dir).unwrap(); #[cfg(feature = "storage-sqlite")] let mut sql_dir = dest_dir.clone(); #[cfg(feature = "storage-sqlite")] diff --git a/tests/integration_tests_rust.rs b/tests/integration_tests_rust.rs index 0dad32d6ab..53ee3363f1 100644 --- a/tests/integration_tests_rust.rs +++ b/tests/integration_tests_rust.rs @@ -26,11 +26,12 @@ use common::{ expect_channel_pending_event, expect_channel_ready_event, expect_channel_ready_events, expect_event, expect_payment_claimable_event, expect_payment_received_event, expect_payment_successful_event, expect_splice_negotiated_event, generate_blocks_and_wait, - generate_listening_addresses, invalidate_blocks, open_channel, open_channel_no_wait, - open_channel_push_amt, open_channel_with_all, premine_and_distribute_funds, premine_blocks, - prepare_rbf, random_chain_source, random_config, setup_bitcoind_and_electrsd, setup_builder, - setup_node, setup_two_nodes, splice_in_with_all, wait_for_block, wait_for_tx, InMemoryStore, - NodePaymentExt, TestChainSource, TestConfig, TestStoreType, TestSyncStore, + generate_listening_addresses, into_builder_store, invalidate_blocks, open_channel, + open_channel_no_wait, open_channel_push_amt, open_channel_with_all, + premine_and_distribute_funds, premine_blocks, prepare_rbf, random_chain_source, random_config, + setup_bitcoind_and_electrsd, setup_builder, setup_node, setup_two_nodes, splice_in_with_all, + wait_for_block, wait_for_tx, InMemoryStore, NodePaymentExt, TestChainSource, TestConfig, + TestStoreType, TestSyncStore, }; use electrsd::corepc_node::{self, Node as BitcoinD}; use electrsd::ElectrsD; @@ -38,6 +39,8 @@ use ldk_node::config::{ AsyncPaymentsRole, EsploraSyncConfig, ADDRESS_POOL_SIZE, DEFAULT_FULL_SCAN_STOP_GAP, }; use ldk_node::entropy::NodeEntropy; +#[cfg(feature = "storage-tier")] +use ldk_node::io::sqlite_store::SqliteStore; use ldk_node::liquidity::LSPS2ServiceConfig; use ldk_node::payment::{ ConfirmationStatus, PayerProofOptions, PaymentDetails, PaymentDirection, PaymentKind, @@ -47,7 +50,15 @@ use ldk_node::{BuildError, Builder, Event, Node, NodeError, ReserveType}; use lightning::ln::channelmanager::PaymentId; use lightning::routing::gossip::{NodeAlias, NodeId}; use lightning::routing::router::RouteParametersConfig; +#[cfg(feature = "storage-tier")] +use lightning::util::persist::MigratableKVStore; use lightning::util::persist::{KVStore, PageToken, PaginatedKVStore, PaginatedListResponse}; +#[cfg(feature = "storage-tier")] +use lightning::util::persist::{ + CHANNEL_MANAGER_PERSISTENCE_KEY, CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_KEY, + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, +}; use lightning_invoice::{Bolt11InvoiceDescription, Description}; use lightning_types::payment::{PaymentHash, PaymentPreimage}; use log::LevelFilter; @@ -142,6 +153,16 @@ impl PaginatedKVStore for ContendedStore { } } +#[cfg(feature = "storage-tier")] +impl MigratableKVStore for ContendedStore { + fn list_all_keys( + &self, + ) -> impl Future, lightning::io::Error>> + 'static + Send + { + MigratableKVStore::list_all_keys(&*self.inner) + } +} + #[test] fn wallet_store_contention_does_not_stall_runtime() { let (ready_sender, ready_receiver) = mpsc::sync_channel(1); @@ -162,7 +183,10 @@ fn wallet_store_contention_does_not_stall_runtime() { wallet_write_started: Arc::new(tokio::sync::Notify::new()), }; let node = builder - .build_with_store(test_config.node_entropy.into(), store.clone()) + .build_with_store( + test_config.node_entropy.into(), + into_builder_store(store.clone()), + ) .map_err(|e| format!("failed to build node: {e:?}"))?; #[cfg(not(feature = "uniffi"))] let node = Arc::new(node); @@ -297,6 +321,16 @@ impl PaginatedKVStore for WalletPersistGatedStore { } } +#[cfg(feature = "storage-tier")] +impl MigratableKVStore for WalletPersistGatedStore { + fn list_all_keys( + &self, + ) -> impl Future, lightning::io::Error>> + 'static + Send + { + MigratableKVStore::list_all_keys(&*self.inner) + } +} + // LDK invokes the sync `SignerProvider::get_shutdown_scriptpubkey` callback on a runtime worker // thread while holding channel locks when a node accepts (or opens) a channel. If deriving the // shutdown script waits on wallet persistence, a contended wallet store wedges the event handler @@ -319,7 +353,9 @@ async fn channel_open_completes_while_wallet_persistence_is_stalled() { sync_config.background_sync_config = None; builder_b.set_chain_source_esplora(esplora_url, Some(sync_config)); let store = WalletPersistGatedStore::new(); - let node_b = builder_b.build_with_store(config_b.node_entropy.into(), store.clone()).unwrap(); + let node_b = builder_b + .build_with_store(config_b.node_entropy.into(), into_builder_store(store.clone())) + .unwrap(); node_b.start().unwrap(); // Fund both nodes so node B passes the anchor reserve check on the accept path. @@ -403,7 +439,9 @@ async fn address_pool_is_reloaded_on_restart() { setup_builder!(builder_b, config_b.node_config); builder_b.set_chain_source_esplora(esplora_url.clone(), Some(sync_config)); - let node_b = builder_b.build_with_store(config_b.node_entropy.into(), store.clone()).unwrap(); + let node_b = builder_b + .build_with_store(config_b.node_entropy.into(), into_builder_store(store.clone())) + .unwrap(); node_b.start().unwrap(); node_b.stop().unwrap(); drop(node_b); @@ -413,7 +451,9 @@ async fn address_pool_is_reloaded_on_restart() { let wallet_writes_before = store.wallet_writes_completed.load(Ordering::Acquire); setup_builder!(builder_b, config_b.node_config); builder_b.set_chain_source_esplora(esplora_url, Some(sync_config)); - let node_b = builder_b.build_with_store(config_b.node_entropy.into(), store.clone()).unwrap(); + let node_b = builder_b + .build_with_store(config_b.node_entropy.into(), into_builder_store(store.clone())) + .unwrap(); assert_eq!(store.wallet_writes_completed.load(Ordering::Acquire), wallet_writes_before); node_b.start().unwrap(); @@ -840,8 +880,9 @@ async fn start_stop_reinit() { setup_builder!(builder, config.node_config); builder.set_chain_source_esplora(esplora_url.clone(), Some(sync_config)); - let node = - builder.build_with_store(config.node_entropy.into(), test_sync_store.clone()).unwrap(); + let node = builder + .build_with_store(config.node_entropy.into(), into_builder_store(test_sync_store.clone())) + .unwrap(); node.start().unwrap(); let expected_node_id = node.node_id(); @@ -879,8 +920,9 @@ async fn start_stop_reinit() { setup_builder!(builder, config.node_config); builder.set_chain_source_esplora(esplora_url.clone(), Some(sync_config)); - let reinitialized_node = - builder.build_with_store(config.node_entropy.into(), test_sync_store).unwrap(); + let reinitialized_node = builder + .build_with_store(config.node_entropy.into(), into_builder_store(test_sync_store)) + .unwrap(); reinitialized_node.start().unwrap(); assert_eq!(reinitialized_node.node_id(), expected_node_id); @@ -4938,3 +4980,164 @@ async fn do_lsps2_multi_lsp_picks_cheapest(reverse_order: bool) { cheap.stop().unwrap(); expensive.stop().unwrap(); } +#[cfg(feature = "storage-tier")] +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn builder_routes_data_across_configured_storage_tiers() { + let (bitcoind, electrsd) = setup_bitcoind_and_electrsd(); + let chain_source = random_chain_source(&bitcoind, &electrsd); + + let config_a = random_config(); + let primary_store = TestSyncStore::new(common::random_storage_path()); + let preexisting_key = ("test", "", "preexisting"); + let preexisting_value = vec![42]; + primary_store + .write(preexisting_key.0, preexisting_key.1, preexisting_key.2, preexisting_value.clone()) + .await + .unwrap(); + let backup_dir = common::random_storage_path(); + let ephemeral_dir = common::random_storage_path(); + + setup_builder!(builder_a, config_a.node_config.clone()); + builder_a.set_chain_source_esplora( + format!("http://{}", electrsd.esplora_url.as_ref().unwrap()), + None, + ); + builder_a.set_filesystem_logger(None, None); + builder_a.set_backup_storage_dir_path(backup_dir.to_str().unwrap().to_owned()); + builder_a.set_ephemeral_storage_dir_path(ephemeral_dir.to_str().unwrap().to_owned()); + + let node_a = builder_a + .build_with_store(config_a.node_entropy.into(), into_builder_store(primary_store.clone())) + .unwrap(); + node_a.start().unwrap(); + assert!(node_a.status().is_running); + assert!(node_a.status().latest_fee_rate_cache_update_timestamp.is_some()); + + let mut config_b = random_config(); + config_b.node_config.manually_handle_unknown_bolt11_payments = true; + let node_b = setup_node(&chain_source, config_b); + + do_channel_full_cycle( + node_a, + node_b, + &bitcoind.client, + &electrsd.client, + false, + true, + true, + false, + ) + .await; + + let backup_store = SqliteStore::new( + backup_dir, + Some(ldk_node::io::sqlite_store::SQLITE_BACKUP_DB_FILE_NAME.to_string()), + Some(ldk_node::io::sqlite_store::KV_TABLE_NAME.to_string()), + ) + .unwrap(); + let ephemeral_store = SqliteStore::new( + ephemeral_dir, + Some(ldk_node::io::sqlite_store::SQLITE_EPHEMERAL_DB_FILE_NAME.to_string()), + Some(ldk_node::io::sqlite_store::KV_TABLE_NAME.to_string()), + ) + .unwrap(); + + assert_eq!( + backup_store.read(preexisting_key.0, preexisting_key.1, preexisting_key.2).await.unwrap(), + preexisting_value + ); + assert!( + ephemeral_store + .read(preexisting_key.0, preexisting_key.1, preexisting_key.2) + .await + .is_err(), + "ephemeral store contains pre-existing durable data" + ); + + for (pn, sn, key) in [ + ("bdk_wallet", "", "descriptor"), + ("bdk_wallet", "", "change_descriptor"), + ("bdk_wallet", "", "network"), + ("", "", "node_metrics"), + ("", "", "events"), + ("", "", "peers"), + ] { + let primary = primary_store.read(pn, sn, key).await.unwrap(); + let backup = backup_store.read(pn, sn, key).await.unwrap(); + + assert_eq!(backup, primary, "backup mismatch for {pn}/{sn}/{key}"); + assert!( + ephemeral_store.read(pn, sn, key).await.is_err(), + "ephemeral store contains durable value {pn}/{sn}/{key}" + ); + } + + let primary_channel_manager = primary_store + .read( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + ) + .await + .unwrap(); + let backup_channel_manager = backup_store + .read( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + ) + .await + .unwrap(); + assert_eq!(backup_channel_manager, primary_channel_manager); + assert!( + ephemeral_store + .read( + CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, + CHANNEL_MANAGER_PERSISTENCE_KEY, + ) + .await + .is_err(), + "ephemeral store contains channel manager data" + ); + + let mut primary_payments = primary_store.list("payments", "").await.unwrap(); + let mut backup_payments = backup_store.list("payments", "").await.unwrap(); + assert!(!primary_payments.is_empty()); + primary_payments.sort(); + backup_payments.sort(); + assert_eq!(backup_payments, primary_payments); + assert!(ephemeral_store.list("payments", "").await.unwrap().is_empty()); + + let ephemeral_network_graph = ephemeral_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .expect("ephemeral store should contain network graph data"); + assert!(!ephemeral_network_graph.is_empty()); + assert!( + primary_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .is_err(), + "primary store contains ephemeral network graph data" + ); + assert!( + backup_store + .read( + NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, + NETWORK_GRAPH_PERSISTENCE_KEY, + ) + .await + .is_err(), + "backup store contains ephemeral network graph data" + ); +}