diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index df762654135..e22df5d7473 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -530,6 +530,9 @@ jobs: - name: Local network test-pattern: test/e2e/localNetwork.spec.js restore_local_network_data: true + - name: Local network state sync + test-pattern: test/e2e/localNetworkStateSync.spec.js + restore_local_network_data: true - name: Testnet fullnode test-pattern: test/e2e/testnetFullnode.spec.js restore_local_network_data: false diff --git a/.pnp.cjs b/.pnp.cjs index f2c82efb91a..3db2927d9d3 100755 --- a/.pnp.cjs +++ b/.pnp.cjs @@ -9661,6 +9661,8 @@ const RAW_RUNTIME_STATE = ["@dashevo/dashcore-lib", "npm:0.22.0"],\ ["@dashevo/dashd-rpc", "npm:19.0.0"],\ ["@dashevo/docker-compose", "npm:0.24.4"],\ + ["@dashevo/dpns-contract", "workspace:packages/dpns-contract"],\ + ["@dashevo/evo-sdk", "workspace:packages/js-evo-sdk"],\ ["@dashevo/wallet-lib", "workspace:packages/wallet-lib"],\ ["@dashevo/withdrawals-contract", "workspace:packages/withdrawals-contract"],\ ["@oclif/core", "npm:3.26.5"],\ @@ -9674,6 +9676,7 @@ const RAW_RUNTIME_STATE = ["chai-as-promised", "virtual:595d7482cc8ddf98ee6aef33fc48b46393554ab5f17f851ef62e6e39315e53666c3e66226b978689aa0bc7f1e83a03081511a21db1c381362fe67614887077f9#npm:7.1.1"],\ ["chalk", "npm:4.1.2"],\ ["cron", "npm:2.1.0"],\ + ["dash", "workspace:packages/js-dash-sdk"],\ ["dashmate", "workspace:packages/dashmate"],\ ["dirty-chai", "virtual:595d7482cc8ddf98ee6aef33fc48b46393554ab5f17f851ef62e6e39315e53666c3e66226b978689aa0bc7f1e83a03081511a21db1c381362fe67614887077f9#npm:2.0.1"],\ ["diskusage-ng", "npm:1.0.4"],\ diff --git a/Cargo.lock b/Cargo.lock index 25f7110cd92..e9eb507bbe9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2975,7 +2975,7 @@ dependencies = [ [[package]] name = "grovedb" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=6c882c3ee7d2c331f1feda2eb4223add9a6f0e45#6c882c3ee7d2c331f1feda2eb4223add9a6f0e45" +source = "git+https://github.com/PastaPastaPasta/grovedb?branch=feat%2Fstate-sync-v2#10a63e1e2dab39b6cc59af8253dc42e187f071c6" dependencies = [ "axum 0.8.9", "bincode", @@ -3014,7 +3014,7 @@ dependencies = [ [[package]] name = "grovedb-bulk-append-tree" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=6c882c3ee7d2c331f1feda2eb4223add9a6f0e45#6c882c3ee7d2c331f1feda2eb4223add9a6f0e45" +source = "git+https://github.com/PastaPastaPasta/grovedb?branch=feat%2Fstate-sync-v2#10a63e1e2dab39b6cc59af8253dc42e187f071c6" dependencies = [ "bincode", "blake3", @@ -3032,7 +3032,7 @@ dependencies = [ [[package]] name = "grovedb-commitment-tree" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=6c882c3ee7d2c331f1feda2eb4223add9a6f0e45#6c882c3ee7d2c331f1feda2eb4223add9a6f0e45" +source = "git+https://github.com/PastaPastaPasta/grovedb?branch=feat%2Fstate-sync-v2#10a63e1e2dab39b6cc59af8253dc42e187f071c6" dependencies = [ "blake3", "grovedb-bulk-append-tree", @@ -3049,7 +3049,7 @@ dependencies = [ [[package]] name = "grovedb-costs" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=6c882c3ee7d2c331f1feda2eb4223add9a6f0e45#6c882c3ee7d2c331f1feda2eb4223add9a6f0e45" +source = "git+https://github.com/PastaPastaPasta/grovedb?branch=feat%2Fstate-sync-v2#10a63e1e2dab39b6cc59af8253dc42e187f071c6" dependencies = [ "integer-encoding", "intmap", @@ -3059,7 +3059,7 @@ dependencies = [ [[package]] name = "grovedb-dense-fixed-sized-merkle-tree" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=6c882c3ee7d2c331f1feda2eb4223add9a6f0e45#6c882c3ee7d2c331f1feda2eb4223add9a6f0e45" +source = "git+https://github.com/PastaPastaPasta/grovedb?branch=feat%2Fstate-sync-v2#10a63e1e2dab39b6cc59af8253dc42e187f071c6" dependencies = [ "bincode", "blake3", @@ -3073,7 +3073,7 @@ dependencies = [ [[package]] name = "grovedb-element" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=6c882c3ee7d2c331f1feda2eb4223add9a6f0e45#6c882c3ee7d2c331f1feda2eb4223add9a6f0e45" +source = "git+https://github.com/PastaPastaPasta/grovedb?branch=feat%2Fstate-sync-v2#10a63e1e2dab39b6cc59af8253dc42e187f071c6" dependencies = [ "bincode", "bincode_derive", @@ -3089,7 +3089,7 @@ dependencies = [ [[package]] name = "grovedb-epoch-based-storage-flags" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=6c882c3ee7d2c331f1feda2eb4223add9a6f0e45#6c882c3ee7d2c331f1feda2eb4223add9a6f0e45" +source = "git+https://github.com/PastaPastaPasta/grovedb?branch=feat%2Fstate-sync-v2#10a63e1e2dab39b6cc59af8253dc42e187f071c6" dependencies = [ "grovedb-costs", "hex", @@ -3101,7 +3101,7 @@ dependencies = [ [[package]] name = "grovedb-merk" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=6c882c3ee7d2c331f1feda2eb4223add9a6f0e45#6c882c3ee7d2c331f1feda2eb4223add9a6f0e45" +source = "git+https://github.com/PastaPastaPasta/grovedb?branch=feat%2Fstate-sync-v2#10a63e1e2dab39b6cc59af8253dc42e187f071c6" dependencies = [ "bincode", "bincode_derive", @@ -3127,7 +3127,7 @@ dependencies = [ [[package]] name = "grovedb-merkle-mountain-range" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=6c882c3ee7d2c331f1feda2eb4223add9a6f0e45#6c882c3ee7d2c331f1feda2eb4223add9a6f0e45" +source = "git+https://github.com/PastaPastaPasta/grovedb?branch=feat%2Fstate-sync-v2#10a63e1e2dab39b6cc59af8253dc42e187f071c6" dependencies = [ "bincode", "blake3", @@ -3140,7 +3140,7 @@ dependencies = [ [[package]] name = "grovedb-path" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=6c882c3ee7d2c331f1feda2eb4223add9a6f0e45#6c882c3ee7d2c331f1feda2eb4223add9a6f0e45" +source = "git+https://github.com/PastaPastaPasta/grovedb?branch=feat%2Fstate-sync-v2#10a63e1e2dab39b6cc59af8253dc42e187f071c6" dependencies = [ "hex", ] @@ -3148,7 +3148,7 @@ dependencies = [ [[package]] name = "grovedb-private-document-store" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=6c882c3ee7d2c331f1feda2eb4223add9a6f0e45#6c882c3ee7d2c331f1feda2eb4223add9a6f0e45" +source = "git+https://github.com/PastaPastaPasta/grovedb?branch=feat%2Fstate-sync-v2#10a63e1e2dab39b6cc59af8253dc42e187f071c6" dependencies = [ "blake3", "grovedb-bulk-append-tree", @@ -3161,7 +3161,7 @@ dependencies = [ [[package]] name = "grovedb-query" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=6c882c3ee7d2c331f1feda2eb4223add9a6f0e45#6c882c3ee7d2c331f1feda2eb4223add9a6f0e45" +source = "git+https://github.com/PastaPastaPasta/grovedb?branch=feat%2Fstate-sync-v2#10a63e1e2dab39b6cc59af8253dc42e187f071c6" dependencies = [ "bincode", "byteorder", @@ -3177,7 +3177,7 @@ dependencies = [ [[package]] name = "grovedb-storage" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=6c882c3ee7d2c331f1feda2eb4223add9a6f0e45#6c882c3ee7d2c331f1feda2eb4223add9a6f0e45" +source = "git+https://github.com/PastaPastaPasta/grovedb?branch=feat%2Fstate-sync-v2#10a63e1e2dab39b6cc59af8253dc42e187f071c6" dependencies = [ "blake3", "grovedb-costs", @@ -3196,7 +3196,7 @@ dependencies = [ [[package]] name = "grovedb-version" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=6c882c3ee7d2c331f1feda2eb4223add9a6f0e45#6c882c3ee7d2c331f1feda2eb4223add9a6f0e45" +source = "git+https://github.com/PastaPastaPasta/grovedb?branch=feat%2Fstate-sync-v2#10a63e1e2dab39b6cc59af8253dc42e187f071c6" dependencies = [ "thiserror 2.0.18", "versioned-feature-core 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3205,7 +3205,7 @@ dependencies = [ [[package]] name = "grovedb-visualize" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=6c882c3ee7d2c331f1feda2eb4223add9a6f0e45#6c882c3ee7d2c331f1feda2eb4223add9a6f0e45" +source = "git+https://github.com/PastaPastaPasta/grovedb?branch=feat%2Fstate-sync-v2#10a63e1e2dab39b6cc59af8253dc42e187f071c6" dependencies = [ "hex", "itertools 0.14.0", @@ -3214,7 +3214,7 @@ dependencies = [ [[package]] name = "grovedbg-types" version = "5.0.1" -source = "git+https://github.com/dashpay/grovedb?rev=6c882c3ee7d2c331f1feda2eb4223add9a6f0e45#6c882c3ee7d2c331f1feda2eb4223add9a6f0e45" +source = "git+https://github.com/PastaPastaPasta/grovedb?branch=feat%2Fstate-sync-v2#10a63e1e2dab39b6cc59af8253dc42e187f071c6" dependencies = [ "serde", "serde_with 3.21.0", diff --git a/Cargo.toml b/Cargo.toml index ffa84dbc455..6c003ec55c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -131,3 +131,32 @@ opt-level = 3 version = "4.2.0-dev.5" rust-version = "1.92" + +# ============================================================================ +# TEMPORARY — QA ONLY. DROP THIS AT THE REAL RE-PIN. +# +# Every grovedb crate is pinned in the package manifests to dashpay/grovedb +# rev 6c882c3, which carries a restore defect for sum trees: applying a state +# sync snapshot chunk fails GroveDB verification ("apply_snapshot_chunk grovedb +# verification failed with N incorrect hashes"), so a joining node can never +# complete a state sync. packages/rs-drive-abci/tests/sum_tree_sync_probe.rs +# pins that defect in-tree. +# +# dashpay/grovedb#840 (branch feat/state-sync-v2 on the PastaPastaPasta fork, +# head 10a63e1) fixes it, and its restore wire version is 1, which is inside +# drive-abci's supported set — so no protocol plumbing changes with it. +# +# This section exists so the state sync e2e can be run against the fix before +# the fork is merged and re-pinned by rev. It MUST be removed once the real +# pin lands: a branch reference is not reproducible, and a workspace [patch] +# silently overrides every manifest's rev. +# ============================================================================ +[patch."https://github.com/dashpay/grovedb"] +grovedb = { git = "https://github.com/PastaPastaPasta/grovedb", branch = "feat/state-sync-v2" } +grovedb-costs = { git = "https://github.com/PastaPastaPasta/grovedb", branch = "feat/state-sync-v2" } +grovedb-path = { git = "https://github.com/PastaPastaPasta/grovedb", branch = "feat/state-sync-v2" } +grovedb-storage = { git = "https://github.com/PastaPastaPasta/grovedb", branch = "feat/state-sync-v2" } +grovedb-version = { git = "https://github.com/PastaPastaPasta/grovedb", branch = "feat/state-sync-v2" } +grovedb-epoch-based-storage-flags = { git = "https://github.com/PastaPastaPasta/grovedb", branch = "feat/state-sync-v2" } +grovedb-query = { git = "https://github.com/PastaPastaPasta/grovedb", branch = "feat/state-sync-v2" } +grovedb-commitment-tree = { git = "https://github.com/PastaPastaPasta/grovedb", branch = "feat/state-sync-v2" } diff --git a/packages/dashmate/configs/defaults/getBaseConfigFactory.js b/packages/dashmate/configs/defaults/getBaseConfigFactory.js index 6e8a7132487..5d491183c59 100644 --- a/packages/dashmate/configs/defaults/getBaseConfigFactory.js +++ b/packages/dashmate/configs/defaults/getBaseConfigFactory.js @@ -356,6 +356,13 @@ export default function getBaseConfigFactory() { txProcessingTimeLimit: null, }, epochTime: 788400, + stateSync: { + snapshots: { + enabled: true, + frequencySeconds: 600, + maxCount: 6, + }, + }, }, tenderdash: { mode: 'full', @@ -460,6 +467,16 @@ export default function getBaseConfigFactory() { }, }, moniker: null, + // Serving snapshots to peers is always on in Tenderdash; `enabled` + // only makes a fresh node bootstrap from a snapshot, and Tenderdash + // ignores it once the node has local state, so it is safe on by + // default for existing nodes. + stateSync: { + enabled: true, + retries: 3, + chunkRequestTimeout: '15s', + fetchersCount: 4, + }, }, }, sourcePath: null, diff --git a/packages/dashmate/configs/defaults/getLocalConfigFactory.js b/packages/dashmate/configs/defaults/getLocalConfigFactory.js index 3ce6adc4851..26054578e7e 100644 --- a/packages/dashmate/configs/defaults/getLocalConfigFactory.js +++ b/packages/dashmate/configs/defaults/getLocalConfigFactory.js @@ -105,6 +105,11 @@ export default function getLocalConfigFactory(getBaseConfig) { metrics: { port: 46660, }, + // A local network genesis starts every node from scratch at the + // same time, so there is no populated peer to state sync from. + stateSync: { + enabled: false, + }, }, abci: { tokioConsole: { @@ -141,6 +146,11 @@ export default function getLocalConfigFactory(getBaseConfig) { rotation: false, }, }, + stateSync: { + snapshots: { + enabled: false, + }, + }, }, }, }, diff --git a/packages/dashmate/configs/getConfigFileMigrationsFactory.js b/packages/dashmate/configs/getConfigFileMigrationsFactory.js index 077ebae1225..ea6dca8f9c4 100644 --- a/packages/dashmate/configs/getConfigFileMigrationsFactory.js +++ b/packages/dashmate/configs/getConfigFileMigrationsFactory.js @@ -1764,6 +1764,23 @@ export default function getConfigFileMigrationsFactory(homeDir, defaultConfigs) return configFile; }, + '4.2.0-dev.6': (configFile) => { + // State sync options are required by the schema now. Pulled from the + // default config matching each config's name or group, so the local + // preset gets its disables while everything else gets the base + // defaults (consume and serve snapshots). + Object.entries(configFile.configs) + .forEach(([name, options]) => { + const defaultConfig = getDefaultConfigByNameOrGroup(name, options.group); + + options.platform.drive.tenderdash.stateSync = defaultConfig + .getStored('platform.drive.tenderdash.stateSync'); + options.platform.drive.abci.stateSync = defaultConfig + .getStored('platform.drive.abci.stateSync'); + }); + + return configFile; + }, }; } diff --git a/packages/dashmate/docker-compose.yml b/packages/dashmate/docker-compose.yml index 0f0384e2aaa..a5a21d55383 100644 --- a/packages/dashmate/docker-compose.yml +++ b/packages/dashmate/docker-compose.yml @@ -96,6 +96,11 @@ services: - GROVEDB_VISUALIZER_ADDRESS=0.0.0.0:${PLATFORM_DRIVE_ABCI_GROVEDB_VISUALIZER_PORT:?err} - PROPOSER_TX_PROCESSING_TIME_LIMIT=${PLATFORM_DRIVE_ABCI_PROPOSER_TX_PROCESSING_TIME_LIMIT} - NETWORK=${NETWORK:?err} + # Checkpoints live in the default CHECKPOINTS_PATH (DB_PATH/checkpoints), + # inside the drive_abci_data volume + - SNAPSHOTS_ENABLED=${PLATFORM_DRIVE_ABCI_STATE_SYNC_SNAPSHOTS_ENABLED:?err} + - SNAPSHOTS_FREQUENCY_SECONDS=${PLATFORM_DRIVE_ABCI_STATE_SYNC_SNAPSHOTS_FREQUENCY_SECONDS:?err} + - MAX_NUM_SNAPSHOTS=${PLATFORM_DRIVE_ABCI_STATE_SYNC_SNAPSHOTS_MAX_COUNT:?err} stop_grace_period: 30s expose: - 26658 diff --git a/packages/dashmate/docs/config/drive-abci.md b/packages/dashmate/docs/config/drive-abci.md index 3bbbf91ea44..3cc9341f590 100644 --- a/packages/dashmate/docs/config/drive-abci.md +++ b/packages/dashmate/docs/config/drive-abci.md @@ -124,6 +124,18 @@ These settings control developer and debugging tools: - Tokio Console: A debugging tool for Rust's async runtime - GroveDB Visualizer: A visualization tool for the GroveDB database structure +## State Sync Snapshots + +These settings control the serving side of state sync: Drive periodically takes GroveDB checkpoints and hands them to Tenderdash, which offers them to peers bootstrapping via state sync. The consuming side is configured on Tenderdash (see [Tenderdash configuration](./tenderdash.md#state-sync)). + +| Option | Description | Default | Example | +|--------|-------------|---------|---------| +| `platform.drive.abci.stateSync.snapshots.enabled` | Take and serve state sync snapshots | `true` | `false` | +| `platform.drive.abci.stateSync.snapshots.frequencySeconds` | How often to take a snapshot, in seconds, at least 60 | `600` | `3600` | +| `platform.drive.abci.stateSync.snapshots.maxCount` | Snapshots kept before pruning the oldest, at least 2 | `6` | `10` | + +Checkpoints are stored inside the Drive data volume under `db/checkpoints`. They hard-link unchanged data, so keeping several costs only a fraction of the database size. The local preset disables snapshots along with state sync consumption. + ## Other options | Option | Description | Default | Example | diff --git a/packages/dashmate/docs/config/tenderdash.md b/packages/dashmate/docs/config/tenderdash.md index ddb285243a4..38d22a0ec0b 100644 --- a/packages/dashmate/docs/config/tenderdash.md +++ b/packages/dashmate/docs/config/tenderdash.md @@ -80,6 +80,21 @@ The RPC interface is used for: - Submitting transactions - Fetching network status +## State Sync + +State sync bootstraps a fresh node from a recent state snapshot fetched from peers instead of replaying every block. These settings control the consuming side; serving snapshots to peers is always on in Tenderdash and is fed by Drive's snapshots (see [Drive ABCI configuration](./drive-abci.md#state-sync-snapshots)). + +| Option | Description | Default | Example | +|--------|-------------|---------|---------| +| `platform.drive.tenderdash.stateSync.enabled` | Bootstrap a fresh node from a snapshot | `true` | `false` | +| `platform.drive.tenderdash.stateSync.retries` | Retries before falling back to block sync, `0` disables retries | `3` | `5` | +| `platform.drive.tenderdash.stateSync.chunkRequestTimeout` | Timeout before re-requesting a snapshot chunk, at least `5s` | `15s` | `30s` | +| `platform.drive.tenderdash.stateSync.fetchersCount` | Concurrent chunk fetchers, 1 to 64 | `4` | `8` | + +- Enabling is safe for existing nodes: Tenderdash only attempts state sync when the node has no local state and disables it by itself otherwise. Only a freshly set up (or reset) node consumes a snapshot, and it ends up with a truncated block history starting at the snapshot height. +- Snapshots are verified through the P2P layer. The alternative RPC state provider needs at least two reachable Tenderdash RPC servers, but dashmate publishes the Tenderdash RPC on loopback only, without TLS and unproxied by the gateway, so the rendered config hardcodes `use-p2p = true`. +- The local preset disables state sync: a local network genesis starts every node from scratch, so there is no populated peer to sync from. + ## Metrics and Profiling These settings control monitoring and profiling tools: diff --git a/packages/dashmate/package.json b/packages/dashmate/package.json index 16759d1be09..3994d3c5d95 100644 --- a/packages/dashmate/package.json +++ b/packages/dashmate/package.json @@ -102,8 +102,11 @@ }, "devDependencies": { "@babel/core": "^7.26.10", + "@dashevo/dpns-contract": "workspace:*", + "@dashevo/evo-sdk": "workspace:*", "chai": "^4.3.10", "chai-as-promised": "^7.1.1", + "dash": "workspace:*", "dirty-chai": "^2.0.1", "eslint": "^9.18.0", "globby": "^11", diff --git a/packages/dashmate/src/config/configJsonSchema.js b/packages/dashmate/src/config/configJsonSchema.js index 263c1e4432d..0dd56b6cb2b 100644 --- a/packages/dashmate/src/config/configJsonSchema.js +++ b/packages/dashmate/src/config/configJsonSchema.js @@ -1087,9 +1087,37 @@ export default { required: ['txProcessingTimeLimit'], additionalProperties: false, }, + stateSync: { + type: 'object', + properties: { + snapshots: { + type: 'object', + properties: { + enabled: { + type: 'boolean', + description: 'Take state sync snapshots (GroveDB checkpoints) and serve them to peers', + }, + frequencySeconds: { + type: 'integer', + minimum: 60, + description: 'How often to take a snapshot, in seconds', + }, + maxCount: { + type: 'integer', + minimum: 2, + description: 'How many snapshots to keep before pruning the oldest', + }, + }, + required: ['enabled', 'frequencySeconds', 'maxCount'], + additionalProperties: false, + }, + }, + required: ['snapshots'], + additionalProperties: false, + }, }, additionalProperties: false, - required: ['docker', 'logs', 'tokioConsole', 'validatorSet', 'chainLock', 'epochTime', 'metrics', 'grovedbVisualizer', 'proposer'], + required: ['docker', 'logs', 'tokioConsole', 'validatorSet', 'chainLock', 'epochTime', 'metrics', 'grovedbVisualizer', 'proposer', 'stateSync'], }, tenderdash: { type: 'object', @@ -1337,8 +1365,45 @@ export default { genesis: { type: 'object', }, + stateSync: { + type: 'object', + properties: { + enabled: { + type: 'boolean', + description: 'Bootstrap a fresh node from a state sync snapshot instead of replaying' + + ' all blocks. Ignored once the node has local state', + }, + retries: { + type: 'integer', + minimum: 0, + description: 'How many times to retry state sync before falling back to block sync.' + + ' 0 disables retries', + }, + chunkRequestTimeout: { + description: 'Timeout before re-requesting a snapshot chunk. Tenderdash requires at least 5s', + allOf: [ + { + $ref: '#/definitions/duration', + }, + { + type: 'string', + // At least 5 seconds: 5s+, 5000ms+, or minutes/hours down to 0.1 + pattern: '^(([5-9]|[1-9][0-9]+)(\\.[0-9]+)?s|([5-9][0-9]{3}|[1-9][0-9]{4,})(\\.[0-9]+)?ms|([1-9][0-9]*(\\.[0-9]+)?|0\\.[1-9][0-9]*)[mh])$', + }, + ], + }, + fetchersCount: { + type: 'integer', + minimum: 1, + maximum: 64, + description: 'Number of concurrent snapshot chunk fetchers', + }, + }, + required: ['enabled', 'retries', 'chunkRequestTimeout', 'fetchersCount'], + additionalProperties: false, + }, }, - required: ['mode', 'docker', 'p2p', 'mempool', 'consensus', 'log', 'rpc', 'pprof', 'node', 'moniker', 'genesis', 'metrics'], + required: ['mode', 'docker', 'p2p', 'mempool', 'consensus', 'log', 'rpc', 'pprof', 'node', 'moniker', 'genesis', 'metrics', 'stateSync'], additionalProperties: false, }, }, diff --git a/packages/dashmate/src/createDIContainer.js b/packages/dashmate/src/createDIContainer.js index 0fca173c771..87178a3ce03 100644 --- a/packages/dashmate/src/createDIContainer.js +++ b/packages/dashmate/src/createDIContainer.js @@ -75,6 +75,7 @@ import startNodeTaskFactory from './listr/tasks/startNodeTaskFactory.js'; import createTenderdashRpcClient from './tenderdash/createTenderdashRpcClient.js'; import setupLocalPresetTaskFactory from './listr/tasks/setup/setupLocalPresetTaskFactory.js'; +import setupLocalJoinNodeTaskFactory from './listr/tasks/setup/local/setupLocalJoinNodeTaskFactory.js'; import setupRegularPresetTaskFactory from './listr/tasks/setup/setupRegularPresetTaskFactory.js'; import stopNodeTaskFactory from './listr/tasks/stopNodeTaskFactory.js'; import restartNodeTaskFactory from './listr/tasks/restartNodeTaskFactory.js'; @@ -322,6 +323,7 @@ export default async function createDIContainer(options = {}) { restartNodeTask: asFunction(restartNodeTaskFactory).singleton(), resetNodeTask: asFunction(resetNodeTaskFactory).singleton(), setupLocalPresetTask: asFunction(setupLocalPresetTaskFactory).singleton(), + setupLocalJoinNodeTask: asFunction(setupLocalJoinNodeTaskFactory).singleton(), setupRegularPresetTask: asFunction(setupRegularPresetTaskFactory).singleton(), configureCoreTask: asFunction(configureCoreTaskFactory).singleton(), configureTenderdashTask: asFunction(configureTenderdashTaskFactory).singleton(), diff --git a/packages/dashmate/src/docker/DockerCompose.js b/packages/dashmate/src/docker/DockerCompose.js index a5bbb0889f8..709c238b0eb 100644 --- a/packages/dashmate/src/docker/DockerCompose.js +++ b/packages/dashmate/src/docker/DockerCompose.js @@ -504,7 +504,10 @@ export default class DockerCompose { * @param {Config} config * @param {string[]} services * @param {Object} options - * @param {number} options.tail + * @param {number} [options.tail] + * @param {string} [options.since] - only logs after this time (RFC3339 or + * a relative value like `10m`), so a caller can read what a service has + * said since a known moment rather than everything the container kept * @return {Promise<{exitCode: number | null, out: string, err: string}>} */ async logs(config, services = [], options = {}) { @@ -515,6 +518,10 @@ export default class DockerCompose { args.unshift('--tail', options.tail.toString()); } + if (options.since) { + args.unshift('--since', options.since); + } + const commandOptions = this.#createOptions(config); try { diff --git a/packages/dashmate/src/doctor/analyse/analyseSystemResourcesFactory.js b/packages/dashmate/src/doctor/analyse/analyseSystemResourcesFactory.js index a5728a4b413..ca66dfc6c12 100644 --- a/packages/dashmate/src/doctor/analyse/analyseSystemResourcesFactory.js +++ b/packages/dashmate/src/doctor/analyse/analyseSystemResourcesFactory.js @@ -22,6 +22,14 @@ export default function analyseSystemResourcesFactory(verifySystemRequirements) diskIO, } = samples.getSystemInfo(); + let stateSyncSnapshotsEnabled = false; + try { + stateSyncSnapshotsEnabled = samples.getDashmateConfig() + .get('platform.drive.abci.stateSync.snapshots.enabled') === true; + } catch { + // A config collected by an older dashmate has no state sync options + } + // System requirements const problems = verifySystemRequirements( { @@ -33,6 +41,7 @@ export default function analyseSystemResourcesFactory(verifySystemRequirements) samples.getDashmateConfig().get('platform.enable'), { diskSpace: 5, + stateSyncSnapshotsEnabled, }, ); diff --git a/packages/dashmate/src/doctor/verifySystemRequirementsFactory.js b/packages/dashmate/src/doctor/verifySystemRequirementsFactory.js index f4ed0196a5a..c29991b698e 100644 --- a/packages/dashmate/src/doctor/verifySystemRequirementsFactory.js +++ b/packages/dashmate/src/doctor/verifySystemRequirementsFactory.js @@ -15,6 +15,7 @@ export default function verifySystemRequirementsFactory() { * @param {boolean} isHP * @param {Object} [overrideRequirements] * @param {Number} [overrideRequirements.diskSpace] + * @param {boolean} [overrideRequirements.stateSyncSnapshotsEnabled] * @returns {Problem[]} */ function verifySystemRequirements( @@ -30,7 +31,12 @@ export default function verifySystemRequirementsFactory() { const MINIMUM_CPU_CORES = isHP ? 4 : 2; const MINIMUM_CPU_FREQUENCY = 2.4; // GHz const MINIMUM_RAM = isHP ? 7.3 : 3.6; // GB - const MINIMUM_DISK_SPACE = overrideRequirements.diskSpace ?? (isHP ? 200 : 100); // GB + + // State sync snapshots are GroveDB checkpoints stored next to the database. + // They share unchanged data with it, so a small fixed headroom is enough. + const SNAPSHOTS_DISK_HEADROOM = overrideRequirements.stateSyncSnapshotsEnabled ? 10 : 0; // GB + const BASE_MINIMUM_DISK_SPACE = overrideRequirements.diskSpace ?? (isHP ? 200 : 100); // GB + const MINIMUM_DISK_SPACE = BASE_MINIMUM_DISK_SPACE + SNAPSHOTS_DISK_HEADROOM; // GB const problems = []; @@ -112,11 +118,17 @@ for required network services and avoid Proof-of-Service bans`, const availableDiskSpace = diskSpace.available / (1024 ** 3); // Convert to GB if (availableDiskSpace < MINIMUM_DISK_SPACE) { + const headroomNote = SNAPSHOTS_DISK_HEADROOM > 0 + ? ` (including ${SNAPSHOTS_DISK_HEADROOM}GB headroom for state sync snapshots)` + : ''; + const problem = new Problem( - `${availableDiskSpace.toFixed(2)}GB of available disk space detected. At least ${MINIMUM_DISK_SPACE}GB is required`, + `${availableDiskSpace.toFixed(2)}GB of available disk space detected. At least ${MINIMUM_DISK_SPACE}GB is required${headroomNote}`, `Consider increasing disk space to make sure the node can provide timely responses for required network services and avoid Proof-of-Service bans`, - MINIMUM_DISK_SPACE - availableDiskSpace < 5 ? SEVERITY.HIGH : SEVERITY.MEDIUM, + // Judged against the base minimum so that enabling snapshots can + // widen when a problem is raised but never downgrade its severity + BASE_MINIMUM_DISK_SPACE - availableDiskSpace < 5 ? SEVERITY.HIGH : SEVERITY.MEDIUM, ); problems.push(problem); diff --git a/packages/dashmate/src/listr/tasks/setup/local/configureTenderdashTaskFactory.js b/packages/dashmate/src/listr/tasks/setup/local/configureTenderdashTaskFactory.js index c3984d32c5a..dbe43749d15 100644 --- a/packages/dashmate/src/listr/tasks/setup/local/configureTenderdashTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/setup/local/configureTenderdashTaskFactory.js @@ -1,4 +1,5 @@ import { Listr } from 'listr2'; +import wireLocalTenderdashNode from './wireLocalTenderdashNode.js'; /** * @return {configureTenderdashTask} @@ -23,29 +24,8 @@ export default function configureTenderdashTaskFactory() { const randomChainIdPart = Math.floor(Math.random() * 60) + 1; const chainId = `dashmate_local_${randomChainIdPart}`; - platformConfigs.forEach((config, index) => { - config.set('platform.drive.tenderdash.genesis.chain_id', chainId); - - const p2pPeers = platformConfigs - .filter((_, i) => i !== index) - .map((innerConfig) => { - const nodeId = innerConfig.get('platform.drive.tenderdash.node.id'); - const port = innerConfig.get('platform.drive.tenderdash.p2p.port'); - - return { - id: nodeId, - host: config.get('externalIp'), - port, - }; - }); - - config.set('platform.drive.tenderdash.p2p.persistentPeers', p2pPeers); - - config.set( - 'platform.drive.tenderdash.genesis.validator_quorum_type', - config.get('platform.drive.abci.validatorSet.quorum.llmqType'), - ); - }); + platformConfigs.forEach((config) => ( + wireLocalTenderdashNode(config, chainId, platformConfigs))); }, }); diff --git a/packages/dashmate/src/listr/tasks/setup/local/setupLocalJoinNodeTaskFactory.js b/packages/dashmate/src/listr/tasks/setup/local/setupLocalJoinNodeTaskFactory.js new file mode 100644 index 00000000000..aa4812743e4 --- /dev/null +++ b/packages/dashmate/src/listr/tasks/setup/local/setupLocalJoinNodeTaskFactory.js @@ -0,0 +1,172 @@ +import { Listr } from 'listr2'; +import lodash from 'lodash'; +import { + PRESET_LOCAL, +} from '../../../../constants.js'; +import deriveTenderdashNodeId from '../../../../tenderdash/deriveTenderdashNodeId.js'; +import generateTenderdashNodeKey from '../../../../tenderdash/generateTenderdashNodeKey.js'; +import generateRandomString from '../../../../util/generateRandomString.js'; +import wireLocalTenderdashNode from './wireLocalTenderdashNode.js'; + +const { cloneDeep: lodashCloneDeep } = lodash; + +/** + * Config option paths that get a per-node host port offset on local networks. + * Mirrors the offsets applied to validators in setupLocalPresetTaskFactory. + * + * @type {string[]} + */ +const OFFSET_PORT_OPTIONS = [ + 'core.p2p.port', + 'core.rpc.port', + 'core.zmq.port', + 'dashmate.helper.api.port', + 'platform.drive.abci.grovedbVisualizer.port', + 'platform.drive.abci.tokioConsole.port', + 'platform.drive.abci.metrics.port', + 'platform.dapi.rsDapi.metrics.port', + 'platform.gateway.admin.port', + 'platform.gateway.listeners.dapiAndDrive.port', + 'platform.gateway.metrics.port', + 'platform.gateway.rateLimiter.metrics.port', + 'platform.drive.tenderdash.p2p.port', + 'platform.drive.tenderdash.rpc.port', + 'platform.drive.tenderdash.pprof.port', + 'platform.drive.tenderdash.metrics.port', +]; + +/** + * @param {ConfigFile} configFile + * @param {resolveDockerHostIp} resolveDockerHostIp + * @param {obtainSelfSignedCertificateTask} obtainSelfSignedCertificateTask + * @return {setupLocalJoinNodeTask} + */ +export default function setupLocalJoinNodeTaskFactory( + configFile, + resolveDockerHostIp, + obtainSelfSignedCertificateTask, +) { + /** + * Create a `local_join` config (or `configName`) for a full node (not a + * masternode, so no collateral registration) that joins an already set up + * local network and bootstraps Drive from a state sync snapshot. Kept as a + * DI task rather than a `group join` CLI command while the state sync e2e + * test is its only consumer. + * + * @typedef {setupLocalJoinNodeTask} + * @param {Config[]} groupConfigs - configs of the existing local group + * @param {Object} [options] + * @param {string} [options.configName] - name for the new config + * @param {number} [options.offsetIndex] - host port offset slot to occupy + * @return {Listr} + */ + function setupLocalJoinNodeTask(groupConfigs, options = {}) { + const { + configName = 'local_join', + // Local nodes local_1..local_N occupy offset indexes 0..N-1 and + // local_seed occupies N, so the first joining node continues at N + 1. + // A caller adding a second joiner passes the next slot explicitly so the + // two do not land on the same host ports. + offsetIndex = groupConfigs.length, + } = options; + + return new Listr([ + { + title: 'Create join node config', + task: async (ctx) => { + const nodeIndex = offsetIndex + 1; + + const config = configFile.createConfig(configName, PRESET_LOCAL); + + config.set('group', 'local'); + config.set('description', 'full node joining the local network'); + + OFFSET_PORT_OPTIONS.forEach((optionPath) => { + config.set(optionPath, config.get(optionPath) + (offsetIndex * 100)); + }); + + // Reads hand back a frozen snapshot, so build the new value and set + // it back rather than writing through the object get() returned. + const rpcUsers = lodashCloneDeep(config.get('core.rpc.users')); + Object.values(rpcUsers).forEach((rpcUser) => { + // eslint-disable-next-line no-param-reassign + rpcUser.password = generateRandomString(12); + }); + config.set('core.rpc.users', rpcUsers); + + config.set('externalIp', await resolveDockerHostIp()); + + const subnet = config.get('docker.network.subnet').split('.'); + subnet[2] = nodeIndex; + config.set('docker.network.subnet', subnet.join('.')); + + // A regular full node: no masternode registration is needed + config.set('core.masternode.enable', false); + + // Sync Core from the existing network + const seedConfigs = groupConfigs.filter((groupConfig) => ( + groupConfig.getName() === 'local_seed' + )); + + if (seedConfigs.length === 0) { + throw new Error('Cannot join the local network: no local_seed config in the group'); + } + + config.set('core.p2p.seeds', seedConfigs.map((seedConfig) => ({ + host: seedConfig.get('externalIp'), + port: seedConfig.get('core.p2p.port'), + }))); + + // Every group config carries the same spork keys (set during setup) + config.set('core.spork.address', groupConfigs[0].get('core.spork.address')); + config.set('core.spork.privateKey', groupConfigs[0].get('core.spork.privateKey')); + + // Platform full node with a fresh Tenderdash identity + config.set('platform.drive.tenderdash.mode', 'full'); + + const nodeKey = generateTenderdashNodeKey(); + + config.set('platform.drive.tenderdash.node.id', deriveTenderdashNodeId(nodeKey)); + config.set('platform.drive.tenderdash.node.key', nodeKey); + config.set('platform.drive.tenderdash.moniker', config.getName()); + + // A fresh node with state sync enabled bootstraps from a snapshot + // served by the existing nodes instead of replaying blocks + config.set('platform.drive.tenderdash.stateSync.enabled', true); + + // Join the existing Tenderdash mesh and genesis + const platformConfigs = groupConfigs.filter((groupConfig) => ( + groupConfig.get('platform.enable') + )); + + if (platformConfigs.length === 0) { + throw new Error('Cannot join the local network: no platform-enabled configs in the group'); + } + + const chainId = platformConfigs[0].get('platform.drive.tenderdash.genesis.chain_id'); + + // The genesis document is a network-wide artifact, and its consensus + // params decide node behaviour well past genesis — the evidence + // window, for one, sets how far below a restored snapshot this node + // backfills light blocks. Rendering the joiner's genesis from preset + // defaults only matches the network by accident, so inherit the + // group's genesis wholesale and wire the node-specific fields on top. + config.set( + 'platform.drive.tenderdash.genesis', + lodashCloneDeep(platformConfigs[0].get('platform.drive.tenderdash.genesis')), + ); + + wireLocalTenderdashNode(config, chainId, platformConfigs); + + ctx.joinNodeConfig = config; + }, + }, + { + title: 'Configure SSL certificate', + task: (ctx) => obtainSelfSignedCertificateTask(ctx.joinNodeConfig), + }, + ]); + } + + return setupLocalJoinNodeTask; +} diff --git a/packages/dashmate/src/listr/tasks/setup/local/wireLocalTenderdashNode.js b/packages/dashmate/src/listr/tasks/setup/local/wireLocalTenderdashNode.js new file mode 100644 index 00000000000..f6a66cffccc --- /dev/null +++ b/packages/dashmate/src/listr/tasks/setup/local/wireLocalTenderdashNode.js @@ -0,0 +1,28 @@ +/** + * Wire a node config into a local network's Tenderdash mesh: shared chain id, + * persistent peers (all given peer configs except the node itself) and the + * validator quorum type used in the genesis document. + * + * @param {Config} config - config of the node to wire + * @param {string} chainId - chain id shared by the local network + * @param {Config[]} peerConfigs - platform-enabled configs of the network + * @return {void} + */ +export default function wireLocalTenderdashNode(config, chainId, peerConfigs) { + config.set('platform.drive.tenderdash.genesis.chain_id', chainId); + + const p2pPeers = peerConfigs + .filter((peerConfig) => peerConfig.getName() !== config.getName()) + .map((peerConfig) => ({ + id: peerConfig.get('platform.drive.tenderdash.node.id'), + host: peerConfig.get('externalIp'), + port: peerConfig.get('platform.drive.tenderdash.p2p.port'), + })); + + config.set('platform.drive.tenderdash.p2p.persistentPeers', p2pPeers); + + config.set( + 'platform.drive.tenderdash.genesis.validator_quorum_type', + config.get('platform.drive.abci.validatorSet.quorum.llmqType'), + ); +} diff --git a/packages/dashmate/templates/platform/drive/tenderdash/config.toml.dot b/packages/dashmate/templates/platform/drive/tenderdash/config.toml.dot index 220f84df843..499d37e5925 100644 --- a/packages/dashmate/templates/platform/drive/tenderdash/config.toml.dot +++ b/packages/dashmate/templates/platform/drive/tenderdash/config.toml.dot @@ -81,7 +81,7 @@ filter-peers = false # Example for routed multi-app setup: # abci = "routed" # address = "Info:socket:unix:///tmp/socket.1,Info:socket:unix:///tmp/socket.2,CheckTx:socket:unix:///tmp/socket.1,*:socket:unix:///tmp/socket.3" -address = "CheckTx:grpc:drive_abci:26670,*:socket:tcp://drive_abci:26658" +address = "CheckTx:grpc:drive_abci:26670,ListSnapshots:grpc:drive_abci:26670,LoadSnapshotChunk:grpc:drive_abci:26670,*:socket:tcp://drive_abci:26658" # Transport mechanism to connect to the ABCI application: socket | grpc | routed transport = "routed" # Maximum number of simultaneous connections to the ABCI application @@ -97,6 +97,10 @@ transport = "routed" #] grpc-concurrency = [ { "check_tx" = {{= it.platform.drive.tenderdash.mempool.maxConcurrentCheckTx }} }, + # Snapshot serving: discovery is one request per peer, chunk downloads run + # several concurrent fetchers per syncing peer. + { "list_snapshots" = 10 }, + { "load_snapshot_chunk" = 100 }, ] @@ -418,29 +422,27 @@ ttl-num-blocks = {{=it.platform.drive.tenderdash.mempool.ttlNumBlocks}} # the network to take and serve state machine snapshots. State sync is not attempted if the node # has any local state (LastBlockHeight > 0). The node will have a truncated block history, # starting from the height of the snapshot. -enable = false +enable = {{? it.platform.drive.tenderdash.stateSync.enabled }}true{{??}}false{{?}} # State sync uses light client verification to verify state. This can be done either through the -# P2P layer or RPC layer. Set this to true to use the P2P layer. If false (default), RPC layer -# will be used. -use-p2p = false +# P2P layer or RPC layer. Set this to true to use the P2P layer. +# Hardcoded to P2P: the RPC mode needs at least two reachable RPC servers, but dashmate +# publishes the Tenderdash RPC on loopback only and does not proxy it through the gateway +# (no TLS or auth), so the RPC state provider is not viable here. +use-p2p = true # If using RPC, at least two addresses need to be provided. They should be compatible with net.Dial, # for example: "host.example.com:2125" rpc-servers = "" -# The hash and height of a trusted block. Must be within the trust-period. -trust-height = 0 -trust-hash = "" - -# The trust period should be set so that Tendermint can detect and gossip misbehavior before -# it is considered expired. For chains based on the Cosmos SDK, one day less than the unbonding -# period should suffice. -trust-period = "168h0m0s" - # Time to spend discovering snapshots before initiating a restore. discovery-time = "15s" +# The number of times to retry state sync. When retries are exhausted, the node falls back +# to block sync. Set to 0 to disable retries. In the pessimistic case it takes at least +# discovery-time * retries before falling back. +retries = {{= it.platform.drive.tenderdash.stateSync.retries }} + # Temporary directory for state sync snapshot chunks, defaults to os.TempDir(). # The synchronizer will create a new, randomly named directory within this directory # and remove it when the sync is complete. @@ -448,10 +450,10 @@ temp-dir = "" # The timeout duration before re-requesting a chunk, possibly from a different # peer (default: 15 seconds). -chunk-request-timeout = "15s" +chunk-request-timeout = "{{= it.platform.drive.tenderdash.stateSync.chunkRequestTimeout }}" # The number of concurrent chunk and block fetchers to run (default: 4). -fetchers = "4" +fetchers = "{{= it.platform.drive.tenderdash.stateSync.fetchersCount }}" ####################################################### ### Consensus Configuration Options ### diff --git a/packages/dashmate/test/e2e/lib/platformSdk.js b/packages/dashmate/test/e2e/lib/platformSdk.js new file mode 100644 index 00000000000..ad104b7d1e7 --- /dev/null +++ b/packages/dashmate/test/e2e/lib/platformSdk.js @@ -0,0 +1,486 @@ +import Dash from 'dash'; +import DashCoreLib from '@dashevo/dashcore-lib'; +import CoreService from '../../../src/core/CoreService.js'; +import wait from '../../../src/util/wait.js'; + +const { PrivateKey } = DashCoreLib; + +/** + * SDK plumbing for e2e specs that need to talk Platform to a node of a local + * dashmate network. + * + * The local gateway serves a self-signed certificate and the state sync suite + * moves every host port off its default, so nothing here may assume the stock + * ports the platform-test-suite reads out of `.env`. Every address is derived + * from the dashmate `Config` of the node being addressed, which is what lets a + * caller point one client at the validators and another at the joined node. + */ + +/** + * `WasmSdkError.name` for a transition family whose proof cannot bind the + * execution of one specific transition. + * + * @type {string} + */ +const EXECUTION_NOT_PROVED = 'ExecutionNotProved'; + +/** + * Shared EvoSDK instances, keyed by address. The WASM SDK multiplexes + * concurrent requests, and instantiating the WASM module repeatedly inside one + * mocha process is both slow and needless. + * + * @type {Map>} + */ +const evoSdkCache = new Map(); + +/** + * Host-facing DAPI address of a node, in `@dashevo/dapi-client` seed notation. + * + * @param {Config} config + * @return {string} + */ +export function getDapiAddress(config) { + const port = config.get('platform.gateway.listeners.dapiAndDrive.port'); + + return `127.0.0.1:${port}:self-signed`; +} + +/** + * Base URL of the local network's quorum list sidecar, which the WASM SDK's + * trusted context uses to learn quorum public keys. + * + * The sidecar runs on the seed node and its port is not offset per node, so + * any config of the group carries the right value. + * + * @param {Config} config + * @return {string} + */ +export function getQuorumListUrl(config) { + return `http://127.0.0.1:${config.get('platform.quorumList.api.port')}`; +} + +/** + * Get (or lazily create) an EvoSDK connected to one specific node. + * + * Proofs are switched on explicitly: every read this SDK performs is checked + * against a GroveDB proof and the Tenderdash quorum signature over the root + * hash, so a node that serves plausible-looking but unproven state fails + * rather than passes. Non-trusted mode is unavailable in WASM, so quorum + * public keys come from the local network's quorum list sidecar. + * + * @param {Config} config - node to query + * @param {Config} quorumListConfig - any config of the group (for the sidecar port) + * @return {Promise<{ evo: Object, sdk: Object }>} + */ +export function getEvoSdk(config, quorumListConfig) { + const address = `https://127.0.0.1:${config.get('platform.gateway.listeners.dapiAndDrive.port')}`; + + if (!evoSdkCache.has(address)) { + evoSdkCache.set(address, (async () => { + // The local gateway's certificate is self-signed and the WASM SDK's + // transport goes through fetch, which has no per-request TLS escape + // hatch. Quorum-verified proofs, not TLS, are the trust boundary here. + process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; + + const evo = await import('@dashevo/evo-sdk'); + + await evo.ensureInitialized(); + + // Connecting prefetches quorum keys and masternode addresses from the + // quorum list sidecar, whose per-masternode version checks need time + // after network start — early on it answers with no eligible + // masternodes and the connect fails. Retry instead of giving up: a + // rejected connect must also not stay cached, or one warm-up hiccup + // would poison every later proof verification. + const deadline = Date.now() + 180000; + + for (;;) { + const sdk = new evo.EvoSDK({ + network: 'local', + trusted: true, + quorumUrl: getQuorumListUrl(quorumListConfig), + addresses: [address], + proofs: true, + }); + + try { + await sdk.connect(); + + return { evo, sdk }; + } catch (error) { + if (Date.now() >= deadline) { + evoSdkCache.delete(address); + throw error; + } + + await wait(5000); + } + } + })()); + } + + return evoSdkCache.get(address); +} + +/** + * Drop every cached EvoSDK. Call between scenarios that restart nodes, so a + * later scenario cannot read through a connection pinned to a dead container. + * + * @return {void} + */ +export function resetEvoSdkCache() { + evoSdkCache.clear(); +} + +/** + * Read the error's kind without assuming it survives the WASM boundary. + * + * @param {*} error + * @return {string|undefined} + */ +function readErrorName(error) { + try { + return error && error.name; + } catch { + // A WASM error object whose memory is already released throws on access. + return undefined; + } +} + +/** + * Create the `IPlatformProofVerifier` that `Dash.Client` requires before it + * will broadcast anything. + * + * Mirrors the platform-test-suite verifier: an execution proof is demanded + * wherever the transition family can produce one, and the families that cannot + * fall back to a height-pinned snapshot of the affected state. + * + * @param {Config} config - node to verify against + * @param {Config} quorumListConfig + * @return {Object} + */ +export function createPlatformProofVerifier(config, quorumListConfig) { + // The WASM SDK's default per-request timeout is 10 seconds, but waiting for + // a state transition result must span at least one block interval — and the + // local network produces empty blocks every 30 seconds. Left at the default, + // every wait can exhaust its retries before a block was even due. + // + // `waitTimeoutMs` must NOT be set here: it makes rs-sdk wrap the wait in + // `tokio::time::timeout`, which reads `std::time::Instant` — unimplemented + // on wasm32 — and the whole module panics with `RuntimeError: unreachable`. + // The per-request `timeoutMs` bounds each attempt through the wasm-safe + // transport path instead, so the overall wait is still finite + // (retries x timeoutMs). + const waitSettings = { + timeoutMs: 120000, + retries: 2, + }; + + return { + async verifyStateTransitionResult({ serializedStateTransition }) { + const { evo, sdk } = await getEvoSdk(config, quorumListConfig); + + const stateTransition = evo.StateTransition.fromBytes( + new Uint8Array(serializedStateTransition), + ); + + try { + await sdk.stateTransitions.waitForResponse(stateTransition, waitSettings); + } catch (error) { + if (readErrorName(error) !== EXECUTION_NOT_PROVED) { + throw error; + } + + await sdk.stateTransitions.waitForAffectedState(stateTransition, waitSettings); + } + }, + + async verifyDataContractHistory({ contractId, startAtMs, limit }) { + const { sdk } = await getEvoSdk(config, quorumListConfig); + + const history = await sdk.contracts.getHistory({ + dataContractId: new Uint8Array(contractId), + limit, + startAtMs: Number(startAtMs), + }); + + return Array.from(history.entries()).map(([date, contract]) => ({ + date: BigInt(date), + value: contract.toBytes(), + })); + }, + }; +} + +/** + * Create a `Dash.Client` with a fresh, empty wallet. + * + * `skipSyncBeforeHeight` matters more than it looks: a local network has + * already mined thousands of blocks registering masternodes by the time this + * runs, and a wallet that scans all of them for a key created seconds ago + * spends minutes finding nothing. Starting the transaction scan at the current + * tip is what the platform-test-suite does for the same reason. + * + * @param {Config} config - node to talk to + * @param {Config} quorumListConfig + * @param {Object} [options] + * @param {number} [options.skipSyncBeforeHeight] + * @return {Client} + */ +export function createClient(config, quorumListConfig, { skipSyncBeforeHeight } = {}) { + const wallet = { + mnemonic: null, + waitForInstantLockTimeout: 120000, + }; + + if (skipSyncBeforeHeight) { + wallet.unsafeOptions = { + skipSynchronizationBeforeHeight: skipSyncBeforeHeight, + }; + } + + return new Dash.Client({ + network: 'regtest', + dapiAddresses: [getDapiAddress(config)], + platformProofVerifier: createPlatformProofVerifier(config, quorumListConfig), + // Per-request gRPC deadline. The 10s default is calibrated for reads; + // broadcasting an identity registration makes Tenderdash run CheckTx, + // which verifies the asset lock's InstantSend signature, and on a local + // network that can outlast 10s — the client then times out and retries a + // broadcast that was never rejected. + timeout: 60000, + wallet, + }); +} + +/** + * Current Core block height. + * + * @param {CoreService} coreService + * @return {Promise} + */ +export async function getCoreHeight(coreService) { + const { result } = await coreService.getRpcClient().getBlockCount(); + + return result; +} + +/** + * Fund a client's wallet straight from the seed node's Core wallet. + * + * The platform-test-suite funds through a second wallet-lib wallet holding the + * faucet key, but that wallet has to rediscover a coinbase output that was + * mined before it existed, over a DAPI whose validators carry no address + * index. Paying the client's address from Core instead means the only thing + * the wallet has to see is a transaction that arrives while it is already + * listening, which is the path wallet-lib is reliable on. + * + * @param {CoreService} coreService - Core of the seed node + * @param {Client} client + * @param {number} amount - duffs to send + * @param {Object} [options] + * @param {number} [options.timeoutMs] + * @param {function(string): void} [options.log] + * @return {Promise<{ address: string, balance: number }>} + */ +export async function fundClientFromCore(coreService, client, amount, { + timeoutMs = 600000, + log = () => {}, +} = {}) { + const account = await client.getWalletAccount(); + const { address } = account.getAddress(); + + const rpcClient = coreService.getRpcClient(); + + // sendToAddress takes DASH, and the wallet reports duffs + const { result: transactionId } = await rpcClient.sendToAddress(address, amount / 1e8); + + log(`sent ${amount} duffs to ${address} in ${transactionId}`); + + // The payment must be observable on the Core it was sent through before the + // wallet's stream can be blamed for not delivering it. A broadcast that + // silently never made it into the mempool used to surface here as a generic + // wallet-sync timeout minutes later, pointing every investigation at DAPI. + const { result: mempool } = await rpcClient.getRawMemPool(); + + if (!mempool.includes(transactionId)) { + throw new Error( + `funding transaction ${transactionId} did not enter the Core mempool after sendToAddress`, + ); + } + + const privateKey = new PrivateKey(); + const throwawayAddress = privateKey.toAddress('regtest').toString(); + + // Confirm the payment and verify it, rather than assuming two mined blocks + // did the job. + await rpcClient.generateToAddress(2, throwawayAddress, 10000000); + + const confirmDeadline = Date.now() + 60000; + let confirmations = 0; + + while (Date.now() < confirmDeadline) { + const { result: fundingTx } = await rpcClient.getTransaction(transactionId); + + confirmations = fundingTx.confirmations || 0; + + if (confirmations > 0) { + log(`funding transaction confirmed in block ${fundingTx.blockheight}` + + ` (${confirmations} confirmations)`); + break; + } + + await rpcClient.generateToAddress(1, throwawayAddress, 10000000); + await wait(2000); + } + + if (confirmations === 0) { + throw new Error( + `funding transaction ${transactionId} entered the mempool but was not mined within 60s`, + ); + } + + const deadline = Date.now() + timeoutMs; + + let balance = 0; + let polls = 0; + + while (Date.now() < deadline) { + balance = account.getTotalBalance(); + + if (balance >= amount) { + return { address, balance }; + } + + polls += 1; + + if (polls % 10 === 0) { + log(`waiting for wallet ${address}: ${balance} of ${amount} duffs`); + + // Nudge the chain occasionally: each new block re-triggers the wallet's + // stream processing without flooding it. Mining on every poll would give + // the sync more blocks to catch up on than it gains. + await rpcClient.generateToAddress(1, throwawayAddress, 10000000); + } + + await wait(3000); + } + + // The chain-side facts are known good at this point, so say so: this + // failure is in the wallet's transaction stream, not in the funding. + throw new Error( + `funding transaction ${transactionId} is confirmed on chain, but the wallet at ${address}` + + ` only saw ${balance} of ${amount} duffs within ${timeoutMs}ms`, + ); +} + +/** + * A CoreService wrapping a node's already-running Core container. + * + * @param {Object} diContainer - awilix DI container + * @param {Config} config + * @return {Promise} + */ +export async function getRunningCoreService(diContainer, config) { + const createRpcClient = diContainer.resolve('createRpcClient'); + const getConnectionHost = diContainer.resolve('getConnectionHost'); + const dockerCompose = diContainer.resolve('dockerCompose'); + const docker = diContainer.resolve('docker'); + + const [containerId] = await dockerCompose.getContainerIds(config, { + filterServiceNames: 'core', + }); + + if (!containerId) { + throw new Error(`Core is not running on ${config.getName()}`); + } + + const rpcClient = createRpcClient({ + port: config.get('core.rpc.port'), + user: 'dashmate', + pass: config.get('core.rpc.users.dashmate.password'), + host: await getConnectionHost(config, 'core', 'core.rpc.host'), + }); + + return new CoreService(config, rpcClient, docker.getContainer(containerId)); +} + +/** + * Sporks the local network turns on during setup. + * + * A node that joins later never learns them: setup activates them once on the + * seed while every node of the group is already connected, and a Core that + * finishes its masternode sync afterwards does not go back for them. Without + * SPORK_19 in particular the joiner treats ChainLocks as disabled, never + * obtains one, and drive-abci waits for a chain lock forever — so Tenderdash + * never finishes its ABCI handshake and state sync cannot even begin. + * + * @type {string[]} + */ +const LOCAL_NETWORK_SPORKS = [ + 'SPORK_2_INSTANTSEND_ENABLED', + 'SPORK_3_INSTANTSEND_BLOCK_FILTERING', + 'SPORK_9_SUPERBLOCKS_ENABLED', + 'SPORK_17_QUORUM_DKG_ENABLED', + 'SPORK_19_CHAINLOCKS_ENABLED', +]; + +/** + * Activate the local network's sporks on a node that joined after setup. + * + * The node's config carries the group's spork private key, so it can sign the + * spork messages itself. + * + * @param {Object} diContainer + * @param {Config} config + * @return {Promise} the sporks activated + */ +export async function activateLocalSporks(diContainer, config) { + const activateCoreSpork = diContainer.resolve('activateCoreSpork'); + + const coreService = await getRunningCoreService(diContainer, config); + + for (const spork of LOCAL_NETWORK_SPORKS) { + await activateCoreSpork(coreService.getRpcClient(), spork); + } + + return LOCAL_NETWORK_SPORKS; +} + +/** + * Mine coins to a fresh address on the seed node and hand back its key. + * + * Reuses dashmate's own `wallet mint` task rather than shelling out to the + * CLI, so the isolated home dir and per-suite ports are honoured. + * + * The task would otherwise start its own Core service, which fails once the + * network is up ("Service core is already running"). Handing it a CoreService + * wrapping the seed's running container makes it mine through that instead, + * and also stops it from tearing the container down afterwards. + * + * @param {Object} diContainer - awilix DI container + * @param {Config} seedConfig - the `local_seed` config + * @param {number} amount - dash to mine + * @return {Promise<{ address: string, privateKey: string, coreService: CoreService }>} + */ +export async function mintToNewAddress(diContainer, seedConfig, amount) { + const generateToAddressTask = diContainer.resolve('generateToAddressTask'); + + const coreService = await getRunningCoreService(diContainer, seedConfig); + + const context = await generateToAddressTask(seedConfig, amount).run({ + coreService, + address: null, + network: seedConfig.get('network'), + }); + + if (!context.privateKey) { + throw new Error('dashmate wallet mint did not return a private key'); + } + + return { + address: context.address, + privateKey: context.privateKey, + coreService, + }; +} diff --git a/packages/dashmate/test/e2e/lib/seedPlatformState.js b/packages/dashmate/test/e2e/lib/seedPlatformState.js new file mode 100644 index 00000000000..c4bff4fe79e --- /dev/null +++ b/packages/dashmate/test/e2e/lib/seedPlatformState.js @@ -0,0 +1,367 @@ +/** + * Put real state on a running local network so a state sync snapshot carries + * something worth verifying. + * + * A network whose only content is empty blocks proves very little: the + * reconstructed platform state would be almost entirely defaults, and a joined + * node could look healthy while having restored nothing. Every step here + * writes into a different subtree of Drive's state — identities, DPNS, data + * contracts (including one with a ranked index, whose secondary count trees + * only exist from protocol v14 on), documents, and identity balances — so the + * post-sync assertions can distinguish "restored the snapshot" from "started + * an empty chain". + * + * Seeding is deliberately granular. A step that cannot work on a local network + * records itself as skipped and the rest continue: the point of the suite is + * state sync, and losing the whole run because one optional write is + * unsupported would be a bad trade. + */ + +import crypto from 'crypto'; +import wait from '../../../src/util/wait.js'; + +/** + * How long to wait after a broadcast for the write to be queryable. + * + * @type {number} + */ +const ST_PROPAGATION_MS = 3000; + +/** + * Document type carrying a ranked index. `rankedCountable` needs + * `rangeCountable: true` (and therefore a countable index), may not sit on a + * unique index, and its terminal property's encoded key must stay under 247 + * bytes — a string costs 4x its `maxLength`, hence the 32 cap. + * + * @type {Object} + */ +const RANKED_DOCUMENT_SCHEMAS = { + rankedItem: { + type: 'object', + indices: [ + { + name: 'byCategory', + properties: [{ category: 'asc' }], + countable: 'countable', + rangeCountable: true, + rankedCountable: true, + }, + ], + properties: { + category: { + type: 'string', minLength: 1, maxLength: 32, position: 0, + }, + label: { type: 'string', maxLength: 63, position: 1 }, + }, + required: ['category'], + additionalProperties: false, + }, +}; + +/** + * A plain document type with ordinary indices, as a control alongside the + * ranked contract. + * + * @type {Object} + */ +const PLAIN_DOCUMENT_SCHEMAS = { + note: { + type: 'object', + indices: [ + { name: 'byOwnerAndTitle', properties: [{ $ownerId: 'asc' }, { title: 'asc' }] }, + ], + properties: { + title: { type: 'string', maxLength: 63, position: 0 }, + body: { type: 'string', maxLength: 255, position: 1 }, + }, + required: ['title'], + additionalProperties: false, + }, +}; + +/** + * Record the outcome of one seeding step. + * + * @param {Object} manifest + * @param {string} name + * @param {function(): Promise<*>} step + * @return {Promise<*|undefined>} the step's value, or undefined when it was skipped + */ +async function runStep(manifest, name, step) { + try { + const value = await step(); + + manifest.steps.push({ name, status: 'ok' }); + + return value; + } catch (error) { + manifest.steps.push({ name, status: 'skipped', reason: error.message }); + + return undefined; + } +} + +/** + * Seed identities, a DPNS name, two data contracts, documents and balance + * movements onto the network `client` is connected to. + * + * @param {Client} client - a funded `Dash.Client` + * @param {Object} [options] + * @param {function(string): void} [options.log] + * @return {Promise} manifest of what was seeded + */ +export default async function seedPlatformState(client, { log = () => {} } = {}) { + const manifest = { + steps: [], + identities: [], + name: undefined, + contracts: {}, + documents: [], + }; + + /** + * @param {number} amount + * @return {Promise} + */ + const registerIdentity = async (amount) => { + const identity = await client.platform.identities.register(amount); + + await wait(ST_PROPAGATION_MS); + + return identity; + }; + + const primaryIdentity = await runStep(manifest, 'register primary identity', async () => { + const identity = await registerIdentity(300000000); + + manifest.identities.push({ + id: identity.getId().toString(), + balance: identity.getBalance().toString(), + role: 'primary', + }); + + log(`seeded primary identity ${identity.getId().toString()}`); + + return identity; + }); + + const secondaryIdentity = await runStep(manifest, 'register secondary identity', async () => { + const identity = await registerIdentity(200000000); + + manifest.identities.push({ + id: identity.getId().toString(), + balance: identity.getBalance().toString(), + role: 'secondary', + }); + + log(`seeded secondary identity ${identity.getId().toString()}`); + + return identity; + }); + + if (!primaryIdentity) { + // Every remaining step signs with this identity, so there is nothing left + // to attempt. The caller decides whether an unseeded run is fatal. + return manifest; + } + + await runStep(manifest, 'register DPNS name', async () => { + // DPNS labels must be unique across the chain and may not start with a + // digit, so prefix the random suffix. + const label = `qa${crypto.randomBytes(6).toString('hex')}`; + + const domain = await client.platform.names.register( + `${label}.dash`, + { identity: primaryIdentity.getId() }, + primaryIdentity, + ); + + await wait(ST_PROPAGATION_MS); + + manifest.name = { + label, + normalizedLabel: domain.get('normalizedLabel'), + fullName: `${label}.dash`, + identityId: primaryIdentity.getId().toString(), + }; + + log(`seeded DPNS name ${label}.dash`); + + return domain; + }); + + /** + * Publish a contract and register it on the client under `appName`. + * + * @param {string} appName + * @param {Object} schemas + * @return {Promise} + */ + const publishContract = async (appName, schemas) => { + const contract = await client.platform.contracts.create(schemas, primaryIdentity); + + await client.platform.contracts.publish(contract, primaryIdentity); + + await wait(ST_PROPAGATION_MS); + + client.getApps().set(appName, { + contractId: contract.getId(), + contract, + }); + + manifest.contracts[appName] = { + id: contract.getId().toString(), + documentTypes: Object.keys(schemas), + }; + + log(`seeded data contract ${appName} ${contract.getId().toString()}`); + + return contract; + }; + + /** + * Create and broadcast one document. + * + * @param {string} appName + * @param {string} documentType + * @param {Object} data + * @return {Promise} + */ + const createDocument = async (appName, documentType, data) => { + const document = await client.platform.documents.create( + `${appName}.${documentType}`, + primaryIdentity, + data, + ); + + await client.platform.documents.broadcast({ create: [document] }, primaryIdentity); + + await wait(ST_PROPAGATION_MS); + + manifest.documents.push({ + id: document.getId().toString(), + appName, + documentType, + contractId: manifest.contracts[appName].id, + ownerId: primaryIdentity.getId().toString(), + data, + }); + + log(`seeded document ${appName}.${documentType} ${document.getId().toString()}`); + + return document; + }; + + const plainContract = await runStep( + manifest, + 'publish plain data contract', + () => publishContract('qaPlain', PLAIN_DOCUMENT_SCHEMAS), + ); + + if (plainContract) { + await runStep(manifest, 'create plain documents', async () => { + await createDocument('qaPlain', 'note', { title: 'state sync', body: 'seeded before the snapshot' }); + await createDocument('qaPlain', 'note', { title: 'second note', body: 'also seeded' }); + }); + } + + const rankedContract = await runStep( + manifest, + 'publish ranked-index data contract', + () => publishContract('qaRanked', RANKED_DOCUMENT_SCHEMAS), + ); + + if (rankedContract) { + await runStep(manifest, 'create ranked documents', async () => { + await createDocument('qaRanked', 'rankedItem', { category: 'alpha', label: 'first' }); + await createDocument('qaRanked', 'rankedItem', { category: 'alpha', label: 'second' }); + await createDocument('qaRanked', 'rankedItem', { category: 'beta', label: 'third' }); + }); + } + + // js-dash-sdk exposes no token factories at all: wasm-dpp's token bindings + // are getter-only and `contracts.create` forwards document schemas alone, so + // a token contract cannot be declared, minted or transferred from this + // stack. Recorded as a known gap rather than silently dropped, and kept + // distinct from `skipped` so an unexpected failure elsewhere still stands + // out in the report. + manifest.steps.push({ + name: 'token contract mint + transfer', + status: 'unsupported', + reason: 'js-dash-sdk has no token support (wasm-dpp token bindings are read-only ' + + 'and contracts.create forwards only document schemas)', + }); + + if (secondaryIdentity) { + await runStep(manifest, 'top up secondary identity', async () => { + const before = secondaryIdentity.getBalance(); + + await client.platform.identities.topUp(secondaryIdentity.getId(), 1000000); + + await wait(ST_PROPAGATION_MS); + + const after = await client.platform.identities.get(secondaryIdentity.getId()); + + manifest.topUp = { + identityId: secondaryIdentity.getId().toString(), + balanceBefore: before.toString(), + balanceAfter: after.getBalance().toString(), + }; + + log(`topped up ${secondaryIdentity.getId().toString()}`); + }); + } + + await runStep(manifest, 'withdraw credits', async () => { + const account = await client.getWalletAccount(); + const withdrawTo = await account.getUnusedAddress(); + + // Minimum is 190000 credits; go well above it so the withdrawal is not + // rejected for dust while still leaving the identity funded. + const metadata = await client.platform.identities.withdrawCredits( + primaryIdentity, + BigInt(1000000), + { toAddress: withdrawTo.address }, + ); + + await wait(ST_PROPAGATION_MS); + + manifest.withdrawal = { + identityId: primaryIdentity.getId().toString(), + toAddress: withdrawTo.address, + height: metadata && metadata.height ? metadata.height.toString() : undefined, + }; + + log(`withdrew credits from ${primaryIdentity.getId().toString()}`); + }); + + return manifest; +} + +/** + * Human-readable summary of a seeding manifest, for the run log. + * + * @param {Object} manifest + * @return {string} + */ +export function describeSeedManifest(manifest) { + return manifest.steps + .map(({ name, status, reason }) => ( + ` ${status.padEnd(11)} ${name}${reason ? ` — ${reason}` : ''}` + )) + .join('\n'); +} + +/** + * Steps that failed for a reason this suite does not already know about. + * + * A blanket "skipped" would let a real regression in, say, credit top-up look + * exactly like the token gap that can never work here, so the two are kept + * apart and the unexpected ones are surfaced. + * + * @param {Object} manifest + * @return {Object[]} + */ +export function getUnexpectedSkips(manifest) { + return manifest.steps.filter(({ status }) => status === 'skipped'); +} diff --git a/packages/dashmate/test/e2e/lib/stateSyncStatus.js b/packages/dashmate/test/e2e/lib/stateSyncStatus.js new file mode 100644 index 00000000000..3f607a1d41b --- /dev/null +++ b/packages/dashmate/test/e2e/lib/stateSyncStatus.js @@ -0,0 +1,452 @@ +import DAPIClient from '@dashevo/dapi-client'; +import wait from '../../../src/util/wait.js'; +import { getDapiAddress } from './platformSdk.js'; + +/** + * Observation helpers for a node that is (or has just finished) state syncing. + * + * Tenderdash's own RPC is the source of truth: rs-dapi's `getStatus` copies the + * state sync counters straight out of `sync_info`, and Drive reports none of + * them. Both paths are exercised here because the DAPI one is what operators + * actually reach for. + */ + +/** + * Fetch the full `/status` result from a node's Tenderdash RPC. + * + * @param {Config} config + * @return {Promise} + */ +export async function getTenderdashStatus(config) { + let host = config.get('platform.drive.tenderdash.rpc.host'); + + if (host === '0.0.0.0') { + host = '127.0.0.1'; + } + + const port = config.get('platform.drive.tenderdash.rpc.port'); + + const response = await fetch(`http://${host}:${port}/status`); + + const body = await response.json(); + + // Tenderdash wraps the response into `result` over HTTP JSON RPC + return body.result || body; +} + +/** + * Fetch `sync_info` from a node's Tenderdash RPC. + * + * @param {Config} config + * @return {Promise} + */ +export async function getTenderdashSyncInfo(config) { + const { sync_info: syncInfo } = await getTenderdashStatus(config); + + return syncInfo; +} + +/** + * State sync counters Tenderdash reports while restoring a snapshot. rs-dapi + * surfaces exactly these as `GetStatusResponseV0.StateSync`. + * + * @type {string[]} + */ +const STATE_SYNC_FIELDS = [ + 'total_synced_time', + 'remaining_time', + 'total_snapshots', + 'chunk_process_avg_time', + 'snapshot_height', + 'snapshot_chunks_count', + 'backfilled_blocks', + 'backfill_blocks_total', +]; + +/** + * Pick the state sync counters out of a `sync_info`, keeping only the ones + * that carry a meaningful (non-zero, non-empty) value. + * + * @param {Object} syncInfo + * @return {Object} + */ +export function pickStateSyncFields(syncInfo) { + const populated = {}; + + STATE_SYNC_FIELDS.forEach((field) => { + const value = syncInfo[field]; + + if (value === undefined || value === null || value === '' || value === '0' || value === 0) { + return; + } + + populated[field] = value; + }); + + return populated; +} + +/** + * Ask a node's DAPI for its status. + * + * Returns the raw error instead of throwing: this runs against a node that is + * mid-sync and may not be serving DAPI yet, and a failed observation is data + * rather than a test failure. + * + * @param {Config} config + * @return {Promise<{ ok: boolean, stateSync?: Object, chain?: Object, error?: string }>} + */ +export async function getDapiStatus(config) { + const client = new DAPIClient({ + dapiAddresses: [getDapiAddress(config)], + network: 'regtest', + }); + + try { + const response = await client.platform.getStatus(); + + const stateSync = response.getStateSyncStatus(); + const chain = response.getChainStatus(); + + return { + ok: true, + stateSync: { + snapshotHeight: stateSync.getSnapshotHeight().toString(), + snapshotChunkCount: stateSync.getSnapshotChunkCount().toString(), + totalSnapshots: stateSync.getTotalSnapshots(), + totalSyncedTime: stateSync.getTotalSyncedTime().toString(), + chunkProcessAverageTime: stateSync.getChunkProcessAverageTime().toString(), + backfilledBlocks: stateSync.getBackfilledBlocks().toString(), + backfilledBlockTotal: stateSync.getBackfilledBlockTotal().toString(), + }, + chain: { + catchingUp: chain.isCatchingUp(), + latestBlockHeight: chain.getLatestBlockHeight().toString(), + earliestBlockHeight: chain.getEarliestBlockHeight().toString(), + }, + }; + } catch (error) { + return { ok: false, error: error.message }; + } finally { + await client.disconnect().catch(() => {}); + } +} + +/** + * Watch a joining node until it finishes syncing, recording every distinct + * state sync observation seen along the way. + * + * Both transports are polled: Tenderdash RPC (which always carries the + * counters) and DAPI `getStatus` (which is what an operator would use). The + * loop ends when the node reports `catching_up: false`, or when the deadline + * passes. + * + * @param {Config} config + * @param {Object} [options] + * @param {number} [options.timeoutMs] + * @param {number} [options.intervalMs] + * @param {function(string): void} [options.log] + * @return {Promise<{ + * syncInfo: Object|undefined, + * tenderdashObservations: Object[], + * dapiObservations: Object[], + * dapiErrors: string[], + * }>} + */ +export async function watchStateSync(config, { + timeoutMs = 20 * 60 * 1000, + intervalMs = 2000, + log = () => {}, +} = {}) { + const deadline = Date.now() + timeoutMs; + + const tenderdashObservations = []; + const dapiObservations = []; + const dapiErrors = new Set(); + + let syncInfo; + let lastSeen = ''; + + while (Date.now() < deadline) { + try { + syncInfo = await getTenderdashSyncInfo(config); + + if (syncInfo) { + const populated = pickStateSyncFields(syncInfo); + + // Only record a change, so a slow sync does not produce hundreds of + // identical rows. + const fingerprint = JSON.stringify(populated); + + if (Object.keys(populated).length > 0 && fingerprint !== lastSeen) { + lastSeen = fingerprint; + + tenderdashObservations.push({ + at: new Date().toISOString(), + catchingUp: syncInfo.catching_up, + latestBlockHeight: syncInfo.latest_block_height, + earliestBlockHeight: syncInfo.earliest_block_height, + ...populated, + }); + + log(`state sync observation: ${fingerprint}`); + } + + if (syncInfo.catching_up === false + && parseInt(syncInfo.latest_block_height, 10) > 0) { + break; + } + } + } catch { + // Tenderdash RPC is not reachable yet + } + + const dapiStatus = await getDapiStatus(config); + + if (dapiStatus.ok) { + dapiObservations.push({ at: new Date().toISOString(), ...dapiStatus }); + } else { + dapiErrors.add(dapiStatus.error); + } + + await wait(intervalMs); + } + + return { + syncInfo, + tenderdashObservations, + dapiObservations, + dapiErrors: [...dapiErrors], + }; +} + +/** + * Lines worth putting in the run report: the whole sync lifecycle, including + * the peering and consensus handover around it. + * + * @type {RegExp} + */ +const LIFECYCLE_LOG_PATTERN = /snapshot|statesync|state sync|state_sync|chunk|backfill|switching to consensus|added peer|handshake/i; + +/** + * Lines only a node that actually state synced can emit. `added peer`, + * `handshake` and `switching to consensus` appear on every Tenderdash start + * regardless of how the node caught up, so they must not back an assertion + * that a state sync happened. + * + * @type {RegExp} + */ +const STATE_SYNC_LOG_PATTERN = /snapshot|statesync|state_sync|state sync|chunk|backfill/i; + +/** + * Pull the Tenderdash log lines that trace a state sync from offer to + * completion, for the run report. + * + * @param {DockerCompose} dockerCompose + * @param {Config} config + * @param {number} [tail] + * @return {Promise<{ lines: string[], stateSyncLines: string[] }>} + */ +export async function getStateSyncLogExcerpt(dockerCompose, config, tail = 4000) { + let output; + + try { + ({ out: output } = await dockerCompose.logs(config, ['drive_tenderdash'], { tail })); + } catch (error) { + return { + lines: [`unable to read drive_tenderdash logs: ${error.message}`], + stateSyncLines: [], + }; + } + + const all = output.split('\n').map((line) => line.trimEnd()); + + return { + lines: all.filter((line) => LIFECYCLE_LOG_PATTERN.test(line)), + stateSyncLines: all.filter((line) => STATE_SYNC_LOG_PATTERN.test(line)), + }; +} + +/** + * Wait until a node's DAPI answers, i.e. Drive is serving the restored state. + * + * dashmate's own `waitForNodeToBeReadyTask` cannot be used here: it hardcodes + * `no-ssl` when building its DAPI address, while the local preset obtains a + * self-signed certificate and `saveCertificateTask` sets + * `platform.gateway.ssl.enabled`, so the gateway speaks TLS. A plain HTTP + * request to it never succeeds, and that task retries forever with no + * deadline, so using it hangs the suite instead of failing it. + * + * @param {Config} config + * @param {Object} [options] + * @param {number} [options.timeoutMs] + * @param {number} [options.intervalMs] + * @return {Promise} the first successful status + */ +export async function waitForDapiReady(config, { + timeoutMs = 5 * 60 * 1000, + intervalMs = 2000, +} = {}) { + const deadline = Date.now() + timeoutMs; + + let lastError = 'never attempted'; + + while (Date.now() < deadline) { + const status = await getDapiStatus(config); + + if (status.ok) { + return status; + } + + lastError = status.error; + + await wait(intervalMs); + } + + throw new Error( + `${config.getName()} DAPI did not become ready within ${timeoutMs}ms: ${lastError}`, + ); +} + +/** + * The height drive-abci reports after restoring a snapshot, or undefined when + * it never restored one. + * + * This is the direct evidence that a node state synced. `earliest_block_height` + * alone is not, on a stock genesis: after a successful restore Tenderdash + * backfills light blocks *backwards* from the snapshot height to satisfy its + * evidence window (100000 blocks / 48 hours by default), and on a short chain + * that backfill reaches genesis — so a node that demonstrably restored a + * snapshot still ends up reporting 1, exactly like a node that replayed every + * block. The state sync suite shrinks the evidence window in genesis exactly + * so the backfill floor sits above genesis and the truncated history becomes + * observable; this log line remains the per-boot, ABCI-level proof of the + * restore itself, which block execution cannot produce. + * + * `since` scopes the read to one boot. Without it a node that restored once + * and was later wiped and restarted would still show the first restore, and + * the assertion that it re-synced would pass whether or not it actually did — + * silently excusing exactly the regression that scenario exists to catch. + * + * A failure to read the logs throws rather than reporting "no restore": the + * fallback scenario asserts on the absence of a restore, and a docker hiccup + * must not be able to masquerade as proof of it. + * + * @param {DockerCompose} dockerCompose + * @param {Config} config + * @param {Object} [options] + * @param {number} [options.tail] + * @param {string} [options.since] - RFC3339 time to read from + * @return {Promise} + */ +export async function getStateSyncRestoreHeight(dockerCompose, config, { + tail = 4000, + since, +} = {}) { + const { out: output } = await dockerCompose.logs(config, ['drive_abci'], { tail, since }); + + // drive-abci's tracing output colours its field names, so the literal text + // is `state_sync completed height=28`. Strip the escapes + // before matching or the height never parses. + // eslint-disable-next-line no-control-regex + const plain = output.replace(/\u001B\[[0-9;]*m/g, ''); + + // Matched within one line rather than assuming `height` sits immediately + // after the message, so adding a field to that tracing call cannot silently + // turn a successful restore into "never restored". + const matches = [...plain.matchAll(/state_sync completed[^\n]*?height\s*=\s*(\d+)/g)]; + + if (matches.length === 0) { + return undefined; + } + + return parseInt(matches[matches.length - 1][1], 10); +} + +/** + * Raw tail of one service's logs, for when a node fails in a way the filtered + * state sync excerpt cannot explain. + * + * @param {DockerCompose} dockerCompose + * @param {Config} config + * @param {string} serviceName + * @param {number} [tail] + * @return {Promise} + */ +export async function getServiceLogTail(dockerCompose, config, serviceName, tail = 80) { + try { + const { out } = await dockerCompose.logs(config, [serviceName], { tail }); + + return out.split('\n').map((line) => line.trimEnd()).filter(Boolean); + } catch (error) { + return [`unable to read ${serviceName} logs: ${error.message}`]; + } +} + +/** + * State of every container of a node, for diagnosing a service that came up + * and then died. + * + * @param {DockerCompose} dockerCompose + * @param {Config} config + * @return {Promise} + */ +export async function getContainerStates(dockerCompose, config) { + try { + const list = await dockerCompose.getContainersList(config, { all: true }); + + return list.map((entry) => `${entry.Service || entry.Name}: ${entry.State}`); + } catch (error) { + return [`unable to list containers: ${error.message}`]; + } +} + +/** + * Block until a joining node reports that a snapshot restore is genuinely + * under way, so a caller can disturb the network at a moment that matters. + * + * Returns the observation that proved it, or undefined when the node finished + * (or never started) syncing first — the caller decides whether that makes + * its scenario inconclusive rather than failed. + * + * @param {Config} config + * @param {Object} [options] + * @param {number} [options.timeoutMs] + * @param {number} [options.intervalMs] + * @return {Promise} + */ +export async function waitForStateSyncActivity(config, { + timeoutMs = 10 * 60 * 1000, + intervalMs = 1000, +} = {}) { + const deadline = Date.now() + timeoutMs; + + while (Date.now() < deadline) { + try { + const syncInfo = await getTenderdashSyncInfo(config); + + if (syncInfo) { + const populated = pickStateSyncFields(syncInfo); + + // A snapshot height means an offer was accepted; the chunk counters + // mean data is actually moving. + if (populated.snapshot_height + || populated.snapshot_chunks_count + || populated.chunk_process_avg_time) { + return populated; + } + + if (syncInfo.catching_up === false + && parseInt(syncInfo.latest_block_height, 10) > 0) { + return undefined; + } + } + } catch { + // Tenderdash RPC is not reachable yet + } + + await wait(intervalMs); + } + + return undefined; +} diff --git a/packages/dashmate/test/e2e/lib/verifySeededState.js b/packages/dashmate/test/e2e/lib/verifySeededState.js new file mode 100644 index 00000000000..3fac97c2bc3 --- /dev/null +++ b/packages/dashmate/test/e2e/lib/verifySeededState.js @@ -0,0 +1,153 @@ +/** + * Re-read seeded state from one specific node, with proofs. + * + * This is the assertion that separates "the joiner came up and answers RPC" + * from "the joiner actually restored the chain's state". Every read goes + * through the WASM SDK's proved path, so the node must produce a GroveDB proof + * that carries the value up to a root hash the validator quorum signed. A node + * that state synced incorrectly cannot fake that; it fails the proof instead + * of returning plausible-looking data. + */ + +import systemIds from '@dashevo/dpns-contract/lib/systemIds.js'; +import { getEvoSdk } from './platformSdk.js'; + +const { contractId: dpnsContractId } = systemIds; + +/** + * Unwrap a `ProofMetadataResponseTyped`, which carries the value alongside the + * proof and block metadata. + * + * @param {*} response + * @return {*} + */ +function unwrapProved(response) { + if (response && typeof response === 'object' && 'data' in response) { + return response.data; + } + + return response; +} + +/** + * Verify every item a seeding run recorded is readable, and proof-verified, + * from `config`'s node. + * + * @param {Config} config - node to query + * @param {Config} quorumListConfig + * @param {Object} manifest - result of seedPlatformState + * @return {Promise} one result per check + */ +export default async function verifySeededState(config, quorumListConfig, manifest) { + const { sdk } = await getEvoSdk(config, quorumListConfig); + + const checks = []; + + /** + * @param {string} name + * @param {function(): Promise<*>} read + * @param {function(*): boolean} isPresent + * @return {Promise} + */ + const check = async (name, read, isPresent) => { + try { + const value = unwrapProved(await read()); + + checks.push({ + name, + present: isPresent(value), + detail: undefined, + }); + } catch (error) { + checks.push({ name, present: false, detail: error.message }); + } + }; + + // Baseline, independent of whether seeding managed to write anything: DPNS + // is created in the genesis state, so every node that holds a correctly + // restored Drive can serve it under proof, and a node that restored nothing + // cannot. This keeps the check meaningful even on a run where seeding was + // blocked. + await check( + `DPNS system data contract ${dpnsContractId}`, + () => sdk.contracts.fetchWithProof(dpnsContractId), + (value) => Boolean(value), + ); + + for (const identity of manifest.identities) { + await check( + `identity ${identity.id}`, + () => sdk.identities.fetchWithProof(identity.id), + (value) => Boolean(value), + ); + } + + for (const [appName, contract] of Object.entries(manifest.contracts)) { + await check( + `data contract ${appName} ${contract.id}`, + () => sdk.contracts.fetchWithProof(contract.id), + (value) => Boolean(value), + ); + } + + for (const document of manifest.documents) { + await check( + `document ${document.appName}.${document.documentType} ${document.id}`, + () => sdk.documents.getWithProof( + document.contractId, + document.documentType, + document.id, + ), + (value) => Boolean(value), + ); + } + + if (manifest.name) { + await check( + `DPNS name ${manifest.name.fullName}`, + () => sdk.dpns.getUsernameByNameWithProof(manifest.name.fullName), + (value) => Boolean(value), + ); + } + + // `ranked` rather than `rankedWithProof`: verification is a property of the + // SDK (built with `proofs: true`), not of the method name. The `WithProof` + // suffix only decides whether the proof bytes come back to the caller — both + // variants verify before returning. + // + // A ranked index keeps ordered secondary trees beside the index itself. + // They are part of the snapshot and are not rebuilt by replaying blocks the + // joiner never saw, so answering a ranked query is a sharper check on the + // restored state than any plain document read. + const rankedContract = manifest.contracts.qaRanked; + + if (rankedContract) { + await check( + 'ranked query over the restored secondary trees', + () => sdk.documents.ranked({ + dataContractId: rankedContract.id, + documentTypeName: 'rankedItem', + groupBy: 'category', + aggregate: { type: 'count' }, + limit: 5, + }), + (value) => Boolean(value && value.entries && value.entries.length > 0), + ); + } + + return checks; +} + +/** + * Human-readable summary of verification results, for the run log. + * + * @param {Object[]} checks + * @return {string} + */ +export function describeVerification(checks) { + return checks + .map(({ name, present, detail }) => ( + ` ${present ? 'present' : 'MISSING'} ${name}${detail ? ` — ${detail}` : ''}` + )) + .join('\n'); +} diff --git a/packages/dashmate/test/e2e/localNetworkStateSync.spec.js b/packages/dashmate/test/e2e/localNetworkStateSync.spec.js new file mode 100644 index 00000000000..95c5edc53b4 --- /dev/null +++ b/packages/dashmate/test/e2e/localNetworkStateSync.spec.js @@ -0,0 +1,1097 @@ +import { asValue } from 'awilix'; +import createDIContainer from '../../src/createDIContainer.js'; +import HomeDir from '../../src/config/HomeDir.js'; +import wait from '../../src/util/wait.js'; +import { + activateLocalSporks, + createClient, + fundClientFromCore, + getCoreHeight, + mintToNewAddress, + resetEvoSdkCache, +} from './lib/platformSdk.js'; +import seedPlatformState, { + describeSeedManifest, + getUnexpectedSkips, +} from './lib/seedPlatformState.js'; +import verifySeededState, { describeVerification } from './lib/verifySeededState.js'; +import { + getContainerStates, + getServiceLogTail, + getStateSyncLogExcerpt, + getStateSyncRestoreHeight, + getTenderdashStatus, + getTenderdashSyncInfo, + waitForDapiReady, + waitForStateSyncActivity, + watchStateSync, +} from './lib/stateSyncStatus.js'; + +/** + * Brings up a three validator local network with frequent Drive snapshots, + * seeds real state onto it, then exercises Tenderdash state sync against that + * chain from several angles: + * + * - a fresh node joins and bootstraps from a snapshot instead of replaying + * blocks, ends up with the truncated block history state sync promises + * (earliest_block_height well above genesis), and the state it restored is + * re-read from it with proofs; + * - a second joiner survives the serving validator being restarted mid-sync; + * - a joined node whose platform data is wiped syncs again from scratch; + * - a joiner pointed at a network with snapshot serving turned off falls back + * to block sync rather than hanging — and keeps the full history from + * genesis, the contrast that makes the truncation assertion meaningful. + * + * The truncated history is only observable because setup shrinks the evidence + * window in the network's genesis (see the evidence constants below). With the + * stock 100000-block / 48-hour window Tenderdash backfills light blocks from + * the snapshot all the way to genesis on any local-sized chain, and + * earliest_block_height lands at 1 even after a genuine restore. Shrinking the + * window in the shared setup instead of adding a separate truncation spec + * keeps the property asserted on every scenario here without paying for a + * second multi-hour network bring-up. + * + * The scenarios share one network on purpose. Each bring-up costs many minutes + * and the later scenarios only need a config change on the running validators, + * so re-running setup for each would multiply the wall clock for no extra + * coverage. They do have to run in order: the fallback scenario disables + * snapshot serving and never turns it back on. + * + * No protocol version plumbing is needed for restorable snapshots: local + * network genesis carries no app_version, so drive-abci starts the chain at + * PlatformVersion::desired() (latest, >= v15) and writes reduced platform + * state from the genesis block on. + */ +describe('Local Network State Sync', function main() { + this.timeout(120 * 60 * 1000); // 120 minutes + this.bail(true); // bail on first failure + + let homeDir; + let container; + let configGroup; + let configFile; + let configFileRepository; + let writeConfigTemplates; + let assertLocalServicesRunning; + let dockerCompose; + let docker; + let joinConfig; + let churnConfig; + let fallbackConfig; + + // Starts empty so the post-sync state checks still run (against the genesis + // state) on a run where seeding could not proceed. + let seedManifest = { steps: [], identities: [], contracts: {}, documents: [] }; + let seedBlocker; + + const groupName = 'local'; + const joinConfigName = 'local_join'; + const churnConfigName = 'local_join_churn'; + const fallbackConfigName = 'local_join_fallback'; + + const joinConfigNames = [joinConfigName, churnConfigName, fallbackConfigName]; + + // How often validators create snapshot checkpoints + // (the config schema minimum is 60 seconds) + const snapshotFrequencySeconds = 60; + + // DB_PATH in docker-compose.yml plus the default checkpoints subdirectory + const driveCheckpointsPath = '/var/lib/dash/rs-drive-abci/db/checkpoints'; + + // Blocks the chain must have produced beyond the newest snapshot before a + // joiner is asked to restore it + const SNAPSHOT_HEIGHT_HEADROOM = 10; + + // Evidence window rendered into the network's genesis. After restoring a + // snapshot Tenderdash backfills light blocks below it until BOTH limits are + // covered, and only then does earliest_block_height settle. The window is + // shrunk so the backfill floor (snapshot height minus the window) sits well + // above genesis and the truncated history can be asserted. The duration leg + // is kept far below one empty-block interval (30s, set below) so the block + // count is the binding limit and the expected floor stays computable. + const EVIDENCE_MAX_AGE_NUM_BLOCKS = 10; + const EVIDENCE_MAX_AGE_DURATION_NS = '10000000000'; // 10 seconds + + // How far past the block-count window the backfill may legitimately run: + // the duration leg keeps it going while the blocks it walks are younger + // than EVIDENCE_MAX_AGE_DURATION_NS, which can add a few blocks when the + // ones below the snapshot were minted in a seeding burst rather than on the + // empty-block cadence. + const BACKFILL_SLACK_BLOCKS = 8; + + // The first joiner must restore a snapshot at least this high so the + // backfill floor is unambiguously above genesis even with the slack spent + // (20 - 10 - 8 = 2 > 1). Kept as low as that bound allows: on a run where + // seeding could not advance the chain, every extra block here is 30 wall + // clock seconds of empty-block waiting. + const TRUNCATION_MIN_SNAPSHOT_HEIGHT = 20; + + /** + * When each node last started, so a restore can be attributed to this boot + * rather than to an earlier one whose log lines the container still holds. + * + * @type {Map} + */ + const bootedAt = new Map(); + + /** + * Note that a node is starting now, a little in the past to absorb any skew + * between this process's clock and the docker daemon's. + * + * @param {Config} config + * @return {void} + */ + function markBoot(config) { + bootedAt.set(config.getName(), new Date(Date.now() - 5000).toISOString()); + } + + /** + * The snapshot height a node restored during its current boot, or undefined + * if it restored nothing. + * + * @param {Config} config + * @return {Promise} + */ + function getRestoreHeightSinceBoot(config) { + return getStateSyncRestoreHeight(dockerCompose, config, { + since: bootedAt.get(config.getName()), + }); + } + + // Host resources this run may claim. Overridable so two checkouts (or a run + // following one whose docker networks were left behind) can coexist. + const subnet = process.env.DASHMATE_E2E_STATE_SYNC_SUBNET || '172.31.0.0/24'; + const portBase = parseInt(process.env.DASHMATE_E2E_STATE_SYNC_PORT_BASE || '41000', 10); + const auxPortBase = parseInt(process.env.DASHMATE_E2E_STATE_SYNC_AUX_PORT_BASE || '42000', 10); + + /** + * Everything worth putting in a run report that assertions alone do not + * carry: seeding outcomes, mid-sync observations, log excerpts. + * + * @type {string[]} + */ + const report = []; + + /** + * @param {string} line + * @return {void} + */ + function record(line) { + report.push(line); + + // Mocha swallows stdout from hooks in some reporters, but the e2e suite + // runs with the spec reporter where this is the run's evidence trail. + // eslint-disable-next-line no-console + console.log(`[state-sync-qa] ${line}`); + } + + /** + * List heights of snapshot checkpoints a node's Drive has created so far + * + * @param {Config} config + * @return {Promise} + */ + async function getCheckpointHeights(config) { + let commandOutput; + try { + commandOutput = await dockerCompose.execCommand( + config, + 'drive_abci', + ['sh', '-c', `ls ${driveCheckpointsPath} 2>/dev/null || true`], + ); + } catch { + return []; + } + + return commandOutput.out + .split('\n') + .map((line) => parseInt(line.trim(), 10)) + .filter((height) => Number.isInteger(height) && height > 0); + } + + /** + * Wait until a validator has a snapshot checkpoint at or above the given + * height. + * + * @param {Config} validatorConfig + * @param {number} minHeight + * @param {number} [timeoutMs] + * @return {Promise} + */ + async function waitForRestorableCheckpoint( + validatorConfig, + minHeight, + timeoutMs = 25 * 60 * 1000, + ) { + const deadline = Date.now() + timeoutMs; + + let checkpointHeights = []; + while (Date.now() < deadline) { + checkpointHeights = await getCheckpointHeights(validatorConfig); + + const restorable = checkpointHeights.filter((height) => height >= minHeight); + + if (restorable.length > 0) { + // A snapshot needs headroom below the tip: Tenderdash verifies the + // light block at the snapshot height before accepting an offer, and on + // a chain only a block or two long there is nothing to verify against, + // so the joiner gives up on discovery and block syncs instead. + // + // The headroom is required above the OLDEST acceptable checkpoint, + // not the newest: snapshots appear about every two blocks here (60s + // frequency against a 30s empty-block cadence), so the tip can never + // outrun the newest checkpoint by much — a gate on the newest would + // wait forever on a quiet chain. The joiner prefers the newest offer + // and Tenderdash falls back to older snapshots when one cannot be + // verified, so one acceptable checkpoint with headroom is what the + // scenario actually needs; the chain also keeps growing during the + // joiner's multi-minute setup, giving the newer offers their own + // headroom by the time discovery happens. + let latestHeight = 0; + + try { + const syncInfo = await getTenderdashSyncInfo(validatorConfig); + latestHeight = parseInt(syncInfo.latest_block_height, 10); + } catch { + // validator RPC not reachable yet + } + + if (latestHeight >= Math.min(...restorable) + SNAPSHOT_HEIGHT_HEADROOM) { + break; + } + } + + await wait(5000); + } + + return checkpointHeights; + } + + /** + * Record everything needed to explain a joiner that failed to sync. + * + * A sync assertion that fails with nothing but "never answered RPC" cannot + * be acted on, and by the time the suite tears down the containers are gone. + * + * @param {Config} config + * @param {string} reason + * @return {Promise} + */ + async function dumpJoinerDiagnostics(config, reason) { + record(`DIAGNOSTICS for ${config.getName()}: ${reason}`); + + const states = await getContainerStates(dockerCompose, config); + record(' container states:'); + states.forEach((line) => record(` ${line}`)); + + for (const service of ['drive_tenderdash', 'drive_abci', 'core']) { + const lines = await getServiceLogTail(dockerCompose, config, service); + record(` last ${lines.length} ${service} log lines:`); + lines.forEach((line) => record(` ${line}`)); + } + } + + /** + * Set up, start and return the config of an extra node joining the network. + * + * @param {string} configName + * @param {number} offsetIndex + * @return {Promise} + */ + async function startJoinNode(configName, offsetIndex) { + const setupLocalJoinNodeTask = container.resolve('setupLocalJoinNodeTask'); + + await setupLocalJoinNodeTask(configGroup, { configName, offsetIndex }).run({ + isVerbose: true, + }); + + const config = configFile.getConfig(configName); + + await configFileRepository.write(configFile); + + writeConfigTemplates(config); + + const startNodeTask = container.resolve('startNodeTask'); + + markBoot(config); + + await startNodeTask(config).run({ + isVerbose: true, + }); + + // A node that joins after setup never learns the local network's sporks, + // and without SPORK_19 its Core reports no ChainLock, drive-abci waits for + // one forever and Tenderdash never completes the ABCI handshake. Activate + // them on the joiner with the group's spork key, exactly as setup does for + // the original nodes. + const sporks = await activateLocalSporks(container, config); + + record(`activated ${sporks.length} sporks on ${config.getName()}`); + + return config; + } + + before(async () => { + container = await createDIContainer(); + + homeDir = container.resolve('homeDir'); + if (process.env.DASHMATE_E2E_TESTS_LOCAL_HOMEDIR) { + homeDir.change(new HomeDir(process.env.DASHMATE_E2E_TESTS_LOCAL_HOMEDIR)); + } else { + homeDir.change(HomeDir.createTemp()); + } + + // Create config file + /** + * @type {ConfigFileJsonRepository} + */ + configFileRepository = container.resolve('configFileRepository'); + + const createConfigFile = container.resolve('createConfigFile'); + + if (process.env.DASHMATE_E2E_TESTS_LOCAL_HOMEDIR) { + configFile = configFileRepository.read(); + } else { + configFile = createConfigFile(); + } + + // Update local config template that will be used to setup nodes + // (and to create the join node config later) + const localConfig = configFile.getConfig(groupName); + + if (process.env.DASHMATE_E2E_TESTS_SKIP_IMAGE_BUILD !== 'true') { + localConfig.set('dashmate.helper.docker.build.enabled', true); + localConfig.set('platform.drive.abci.docker.build.enabled', true); + localConfig.set('platform.dapi.rsDapi.docker.build.enabled', true); + } + + // Offset from localNetwork.spec.js ports so leftovers of one suite + // don't collide with the other on a developer machine. + // + // Both the subnet and the two port blocks are overridable, because a + // developer machine can already carry an unrelated local network (or an + // orphaned docker network from an earlier run) sitting on these ranges, + // and docker refuses to create an overlapping pool. + localConfig.set('docker.network.subnet', subnet); + localConfig.set('dashmate.helper.api.port', portBase); + localConfig.set('core.p2p.port', portBase + 1); + localConfig.set('core.rpc.port', portBase + 2); + localConfig.set('platform.gateway.listeners.dapiAndDrive.port', portBase + 3); + localConfig.set('platform.drive.tenderdash.p2p.port', portBase + 4); + localConfig.set('platform.drive.tenderdash.rpc.port', portBase + 5); + localConfig.set('platform.drive.tenderdash.pprof.port', portBase + 6); + + // The remaining host-published ports (see the `ports:` sections in + // docker-compose.yml) are moved off their defaults too, so the suite can + // run next to another local network that keeps the stock ports + localConfig.set('core.zmq.port', auxPortBase + 1); + localConfig.set('platform.drive.abci.tokioConsole.port', auxPortBase + 2); + localConfig.set('platform.drive.abci.metrics.port', auxPortBase + 3); + localConfig.set('platform.drive.abci.grovedbVisualizer.port', auxPortBase + 4); + localConfig.set('platform.drive.tenderdash.metrics.port', auxPortBase + 5); + localConfig.set('platform.gateway.metrics.port', auxPortBase + 6); + localConfig.set('platform.gateway.admin.port', auxPortBase + 7); + localConfig.set('platform.gateway.rateLimiter.metrics.port', auxPortBase + 8); + localConfig.set('platform.quorumList.api.port', auxPortBase + 9); + + // Leftover join node configs from a previous run against this home dir + joinConfigNames.forEach((name) => { + if (configFile.isConfigExists(name)) { + configFile.removeConfig(name); + } + }); + + container.register({ + configFile: asValue(configFile), + }); + + writeConfigTemplates = container.resolve('writeConfigTemplates'); + assertLocalServicesRunning = container.resolve('assertLocalServicesRunning'); + dockerCompose = container.resolve('dockerCompose'); + docker = container.resolve('docker'); + }); + + // Teardown lives here rather than in a trailing `it()` because the suite + // bails on first failure, and Mocha never enters a describe block it has + // not reached yet. A failure — the thing this suite exists to catch — would + // otherwise strand three validators, up to three join nodes, their volumes + // and the docker network, which the next run then collides with on the same + // subnet and ports. Every step is best-effort so one failure cannot stop the + // rest of the cleanup. + after(async function teardown() { + this.timeout(30 * 60 * 1000); + + // Debugging aid: a failure in this suite is usually only diagnosable from + // the containers teardown is about to destroy. Keeping them alive is opt + // in and leaves the network, volumes and home dir for a post-mortem — the + // operator owns the cleanup. + if (process.env.DASHMATE_E2E_STATE_SYNC_KEEP_NETWORK === 'true') { + record('teardown skipped (DASHMATE_E2E_STATE_SYNC_KEEP_NETWORK=true); ' + + `the network is still running under ${homeDir.getPath()}`); + + // eslint-disable-next-line no-console + console.log(`\n[state-sync-qa] run report\n${report.map((line) => ` ${line}`).join('\n')}\n`); + + return; + } + + const joinConfigs = [joinConfig, churnConfig, fallbackConfig].filter(Boolean); + const allConfigs = [...joinConfigs, ...(configGroup || []).slice().reverse()]; + + if (allConfigs.length > 0) { + const stopNodeTask = container.resolve('stopNodeTask'); + + for (const config of allConfigs) { + try { + await stopNodeTask(config).run({ isForce: true }); + } catch (error) { + record(`teardown: could not stop ${config.getName()}: ${error.message}`); + } + } + + // Removes the containers and their volumes + const resetNodeTask = container.resolve('resetNodeTask'); + + for (const config of configFile.getGroupConfigs(groupName)) { + try { + await resetNodeTask(config).run({ + isHardReset: false, + isForce: true, + }); + } catch (error) { + record(`teardown: could not reset ${config.getName()}: ${error.message}`); + } + } + } + + try { + homeDir.remove(); + } catch (error) { + record(`teardown: could not remove home dir: ${error.message}`); + } + + if (report.length > 0) { + // eslint-disable-next-line no-console + console.log(`\n[state-sync-qa] run report\n${report.map((line) => ` ${line}`).join('\n')}\n`); + } + }); + + describe('setup', () => { + it('should setup local network', async function testSetup() { + if (process.env.DASHMATE_E2E_TESTS_LOCAL_HOMEDIR) { + this.skip('local network set up is provided'); + } + + const setupLocalPresetTask = await container.resolve('setupLocalPresetTask'); + const setupTask = setupLocalPresetTask(); + + await setupTask.run({ + nodeCount: 3, + debugLogs: true, + minerInterval: '2.5m', + isVerbose: true, + }); + + const configExists = configFile.isGroupExists(groupName); + + expect(configExists).to.be.true(); + + // Write configs + await configFileRepository.write(configFile); + + const writtenConfigGroup = configFile.getGroupConfigs(groupName); + writtenConfigGroup.forEach(writeConfigTemplates); + }); + + it('should enable frequent snapshots and a short evidence window', async () => { + configGroup = configFile.getGroupConfigs(groupName); + + for (const config of configGroup) { + if (config.get('platform.enable')) { + // The local preset disables snapshot serving because a network + // where every node starts from genesis has nothing to sync from. + // This test is exactly about a node joining later, so turn it on + // with the lowest allowed frequency. + config.set('platform.drive.abci.stateSync.snapshots.enabled', true); + config.set( + 'platform.drive.abci.stateSync.snapshots.frequencySeconds', + snapshotFrequencySeconds, + ); + + // Produce empty blocks often enough that checkpoints appear and + // the joiner catches up without waiting minutes between blocks + config.set('platform.drive.tenderdash.consensus.createEmptyBlocksInterval', '30s'); + + // Shrink the evidence window so post-restore backfill stops above + // genesis and joiners exhibit the truncated block history this + // suite asserts on. This must land in genesis before the first + // start: the genesis.json template renders + // platform.drive.tenderdash.genesis verbatim, and a genesis is + // immutable once the chain has started. Joiners inherit it through + // setupLocalJoinNodeTask, which copies the group genesis. + config.set('platform.drive.tenderdash.genesis.consensus_params.evidence', { + max_age: String(EVIDENCE_MAX_AGE_NUM_BLOCKS), + max_age_num_blocks: String(EVIDENCE_MAX_AGE_NUM_BLOCKS), + max_age_duration: EVIDENCE_MAX_AGE_DURATION_NS, + }); + } + } + + await configFileRepository.write(configFile); + + configGroup.forEach(writeConfigTemplates); + }); + + after(() => { + container.register({ + configGroup: asValue(configGroup), + }); + }); + }); + + describe('start', () => { + it('should start local network', async () => { + const startGroupNodesTask = await container.resolve('startGroupNodesTask'); + const task = startGroupNodesTask(configGroup); + + await task.run({ + isVerbose: true, + waitForReadiness: true, + }); + + await assertLocalServicesRunning(configGroup); + }); + }); + + describe('seed state', () => { + it('should seed identities, names, contracts and documents', async function seedState() { + const seedConfig = configGroup.find((config) => config.getName() === 'local_seed'); + const validatorConfig = configGroup.find((config) => config.get('platform.enable')); + + // Mine coins into the seed node's Core wallet. dashmate's own wallet + // task is reused so the isolated home dir and per-suite ports are + // honoured, and it also matures the coinbase outputs before returning. + const { coreService } = await mintToNewAddress(container, seedConfig, 50); + + // Start the wallet's transaction scan at the tip: everything below it + // predates the key and only costs time to walk. + const coreHeight = await getCoreHeight(coreService); + + record(`core height before funding: ${coreHeight}`); + + const client = createClient(validatorConfig, seedConfig, { + skipSyncBeforeHeight: coreHeight, + }); + + try { + const { address, balance } = await fundClientFromCore(coreService, client, 800000000, { + timeoutMs: 240000, + log: record, + }); + + record(`funded seeding wallet ${address} with ${balance} duffs`); + + seedManifest = await seedPlatformState(client, { log: record }); + } catch (error) { + // Funding the SDK wallet goes through wallet-lib, which learns about + // its coins from DAPI's Core transaction stream. When that stream does + // not deliver, nothing can be seeded — but the state sync scenarios + // that follow are what this suite exists for, and they do not depend + // on custom state, so the run continues against the genesis state + // instead of losing every later scenario to a seeding problem. + seedBlocker = error.message; + + record(`SEEDING BLOCKED — continuing against genesis state only: ${error.message}`); + } finally { + await client.disconnect().catch(() => {}); + } + + if (seedBlocker) { + this.skip(); + } + + record(`seeding outcomes:\n${describeSeedManifest(seedManifest)}`); + + // A seeding step that failed is only diagnosable from the serving + // side, and teardown destroys the containers before anyone can look. + if (seedManifest.identities.length === 0) { + for (const service of ['drive_abci', 'gateway']) { + const lines = await getServiceLogTail(dockerCompose, validatorConfig, service, 60); + record(`${validatorConfig.getName()} ${service} log tail (seeding produced no identity):`); + lines.forEach((line) => record(` ${line}`)); + } + } + + // Individual steps are allowed to skip (tokens have no JS SDK path at + // all), but a run where nothing landed would make every later state + // assertion vacuous. + expect( + seedManifest.identities.length, + `no identity was seeded; outcomes:\n${describeSeedManifest(seedManifest)}`, + ).to.be.above(0); + + expect( + Object.keys(seedManifest.contracts).length, + `no data contract was seeded; outcomes:\n${describeSeedManifest(seedManifest)}`, + ).to.be.above(0); + + expect( + seedManifest.documents.length, + `no document was seeded; outcomes:\n${describeSeedManifest(seedManifest)}`, + ).to.be.above(0); + + // A step that failed for a reason this suite does not already know about + // is the interesting kind: it may be a real regression wearing a skip's + // clothing, so it is called out separately from the known token gap. + const unexpected = getUnexpectedSkips(seedManifest); + + if (unexpected.length > 0) { + record(`UNEXPECTED seeding skips (${unexpected.length}), review these:`); + unexpected.forEach(({ name, reason }) => record(` ${name} — ${reason}`)); + } else { + record('no unexpected seeding skips'); + } + }); + }); + + describe('join node', () => { + it('should create a snapshot beyond the evidence window on a validator', async () => { + const validatorConfig = configGroup.find((config) => config.get('platform.enable')); + + // Wait until a checkpoint comfortably above the evidence window exists, + // so the joining node demonstrably restores a snapshot instead of + // replaying from genesis AND its post-restore backfill floor sits above + // genesis. Seeding already advanced the chain, so this checkpoint + // carries the seeded state rather than an empty tree. + const checkpointHeights = await waitForRestorableCheckpoint( + validatorConfig, + TRUNCATION_MIN_SNAPSHOT_HEIGHT, + ); + + expect( + checkpointHeights.some((height) => height >= TRUNCATION_MIN_SNAPSHOT_HEIGHT), + `no snapshot checkpoint at or above height ${TRUNCATION_MIN_SNAPSHOT_HEIGHT}` + + ` on ${validatorConfig.getName()},` + + ` found: [${checkpointHeights.join(', ')}]`, + ).to.be.true(); + + record(`validator checkpoints: [${checkpointHeights.join(', ')}]`); + }); + + it('should setup and start a join node', async () => { + joinConfig = await startJoinNode(joinConfigName, configGroup.length); + + expect(joinConfig.get('platform.drive.tenderdash.stateSync.enabled')).to.be.true(); + + await assertLocalServicesRunning([joinConfig]); + }); + + it('should state sync the join node instead of replaying blocks', async () => { + const { + syncInfo, + tenderdashObservations, + dapiObservations, + dapiErrors, + } = await watchStateSync(joinConfig, { log: record }); + + if (!syncInfo) { + await dumpJoinerDiagnostics(joinConfig, 'joiner never answered Tenderdash RPC'); + } + + expect(syncInfo, 'join node Tenderdash never responded on RPC').to.exist(); + expect(syncInfo.catching_up, 'join node is still catching up').to.be.false(); + expect(parseInt(syncInfo.latest_block_height, 10)).to.be.above(0); + + // Proof that the node restored a snapshot rather than executing every + // block, taken from the ABCI app itself. Block execution cannot produce + // this log line, so it pins down the path independently of the + // block-store shape asserted on below. + const restoreHeight = await getRestoreHeightSinceBoot(joinConfig); + + if (restoreHeight === undefined) { + // The joiner came up but never restored a snapshot. Whether the + // validators offered one at all is the whole question, and it is only + // answerable from both sides' logs. + await dumpJoinerDiagnostics(joinConfig, 'joiner block synced instead of state syncing'); + + const validatorConfig = configGroup.find((config) => config.get('platform.enable')); + const offered = await getStateSyncLogExcerpt(dockerCompose, validatorConfig); + + record(`serving validator ${validatorConfig.getName()} snapshot log lines:`); + offered.stateSyncLines.slice(-30).forEach((line) => record(` ${line}`)); + + const heights = await getCheckpointHeights(validatorConfig); + record(`serving validator checkpoints at failure: [${heights.join(', ')}]`); + } + + expect( + restoreHeight, + 'join node replayed blocks from genesis instead of restoring a snapshot', + ).to.be.a('number'); + + expect(restoreHeight, 'snapshot was restored at genesis').to.be.above(1); + + // The user-visible property of state sync: the block history starts at + // the backfill floor, not at genesis. The status is re-fetched here + // rather than reusing the poll's last sample, both so the assertion + // reads the settled post-backfill value and so the raw document lands + // in the run report (which sync_info/statesync fields this Tenderdash + // actually populates is itself worthwhile evidence). + const status = await getTenderdashStatus(joinConfig); + + record(`joiner raw /status after sync: ${JSON.stringify(status)}`); + + const earliestBlockHeight = parseInt(status.sync_info.earliest_block_height, 10); + + expect( + earliestBlockHeight, + 'a state synced node must have a truncated block history starting above genesis', + ).to.be.above(1); + + // The bounds below assume the joiner restored a snapshot at or above + // the minimum the checkpoint gate waited for. Tenderdash may fall back + // to an older offered snapshot, and one low enough would clip the + // backfill at genesis — fail that precondition by name rather than as + // a baffling bounds mismatch. + expect( + restoreHeight, + `the joiner restored a snapshot at ${restoreHeight}, below the` + + ` ${TRUNCATION_MIN_SNAPSHOT_HEIGHT} the truncation bounds assume`, + ).to.be.at.least(TRUNCATION_MIN_SNAPSHOT_HEIGHT); + + // ... and the floor is where the evidence window puts it: backfill runs + // from the snapshot down until the window is covered, so the earliest + // block sits at snapshot height minus the block-count window, give or + // take the duration leg (see BACKFILL_SLACK_BLOCKS). + const expectedEarliest = restoreHeight - EVIDENCE_MAX_AGE_NUM_BLOCKS; + + expect( + earliestBlockHeight, + `earliest_block_height=${earliestBlockHeight} is inconsistent with the` + + ` evidence window: the snapshot restored at ${restoreHeight} puts the` + + ` backfill floor at ${expectedEarliest} (slack ${BACKFILL_SLACK_BLOCKS} below, 2 above)`, + ).to.be.within(expectedEarliest - BACKFILL_SLACK_BLOCKS, expectedEarliest + 2); + + record(`joiner restored a snapshot at height ${restoreHeight} and kept a` + + ` truncated history: earliest_block_height=${earliestBlockHeight}` + + ` (backfill floor ${expectedEarliest} = ${restoreHeight} - ${EVIDENCE_MAX_AGE_NUM_BLOCKS}),` + + ` latest_block_height=${status.sync_info.latest_block_height}`); + + // Ops acceptance: the state sync counters an operator would watch. + // A sync that finishes between two polls legitimately leaves none, so + // this is recorded rather than asserted. + record(`mid-sync Tenderdash state sync observations: ${tenderdashObservations.length}`); + tenderdashObservations.forEach((observation) => { + record(` ${JSON.stringify(observation)}`); + }); + + record(`mid-sync DAPI getStatus observations: ${dapiObservations.length}`); + dapiObservations.slice(0, 5).forEach((observation) => { + record(` ${JSON.stringify(observation)}`); + }); + + if (dapiErrors.length > 0) { + record(`DAPI getStatus errors seen while syncing: ${JSON.stringify(dapiErrors)}`); + } + + // Drive and the other services survived applying the snapshot + await assertLocalServicesRunning([joinConfig]); + + // Drive serves the restored state: DAPI can fetch a system contract + const ready = await waitForDapiReady(joinConfig); + + record(`joined node DAPI is serving: ${JSON.stringify(ready.chain)}`); + record(`joined node DAPI StateSync fields: ${JSON.stringify(ready.stateSync)}`); + }); + + it('should serve the seeded state from the joined node with proofs', async () => { + const checks = await verifySeededState(joinConfig, configGroup[0], seedManifest); + + if (seedBlocker) { + record('NOTE: seeding was blocked, so this check covers the genesis state only'); + } + + record(`seeded state on the joined node:\n${describeVerification(checks)}`); + + const missing = checks.filter(({ present }) => !present); + + expect(checks.length, 'nothing was verified against the joined node').to.be.above(0); + + expect( + missing.length, + `the joined node did not serve seeded state:\n${describeVerification(missing)}`, + ).to.equal(0); + }); + + it('should report a healthy node through dashmate status', async () => { + const getPlatformScope = container.resolve('getPlatformScope'); + + const scope = await getPlatformScope(joinConfig); + + record(`dashmate platform status: tenderdash=${scope.tenderdash.serviceStatus}` + + ` drive=${scope.drive.serviceStatus}` + + ` height=${scope.tenderdash.latestBlockHeight}` + + ` peers=${scope.tenderdash.peers}`); + + expect(scope.tenderdash.catchingUp).to.be.false(); + expect(scope.tenderdash.serviceStatus).to.equal('up'); + expect(scope.drive.serviceStatus).to.equal('up'); + }); + + it('should capture the sync lifecycle from the joiner logs', async () => { + const { lines, stateSyncLines } = await getStateSyncLogExcerpt(dockerCompose, joinConfig); + + record(`joiner Tenderdash sync lifecycle log excerpt (${lines.length} lines):`); + lines.forEach((line) => record(` ${line}`)); + + // Peering and consensus handover lines appear on every Tenderdash start, + // so only the snapshot/chunk ones can back this assertion. + expect( + stateSyncLines.length, + 'no snapshot or chunk lines in the joiner logs', + ).to.be.above(0); + }); + }); + + describe('serving-side churn', () => { + it('should complete a sync while the serving validator restarts', async () => { + churnConfig = await startJoinNode(churnConfigName, configGroup.length + 1); + + // Wait until the joiner is demonstrably restoring a snapshot before + // disturbing anything. Firing the restart at an arbitrary moment could + // land before the transfer starts or after it finished, and the + // scenario would pass having tested nothing. + const activity = await waitForStateSyncActivity(churnConfig); + + if (activity) { + record(`churn joiner is mid-restore: ${JSON.stringify(activity)}`); + } else { + record('INCONCLUSIVE: the churn joiner never reported an in-progress restore,' + + ' so the restart below did not interrupt a chunk transfer'); + } + + // Which validator serves the snapshot is decided by Tenderdash's own + // peer discovery, so restarting one picked at random would leave open + // whether the serving node was ever touched. Restart them all, one at a + // time: every candidate server is bounced while the other two keep the + // chain producing blocks. + let restarted = 0; + + for (const config of configGroup) { + if (!config.get('platform.enable')) { + continue; + } + + const containerIds = await dockerCompose.getContainerIds(config, { + filterServiceNames: ['drive_abci', 'drive_tenderdash'], + }); + + for (const id of containerIds) { + await docker.getContainer(id).restart({ t: 5 }); + restarted += 1; + } + + record(`restarted ${containerIds.length} platform containers on ${config.getName()}`); + } + + expect(restarted, 'no validator containers were restarted').to.be.above(0); + + const { syncInfo } = await watchStateSync(churnConfig, { log: record }); + + expect(syncInfo, 'churn join node Tenderdash never responded on RPC').to.exist(); + expect( + syncInfo.catching_up, + 'churn join node did not finish syncing after the serving validators restarted', + ).to.be.false(); + + // Recorded, not asserted: a restart harsh enough to exhaust the state + // sync retries legitimately leaves the joiner block syncing instead, + // which is still a completed sync but a different path. + const churnRestoreHeight = await getRestoreHeightSinceBoot(churnConfig); + + record(`churn joiner finished at latest_block_height=${syncInfo.latest_block_height}` + + ` (${churnRestoreHeight === undefined + ? 'fell back to block sync' + : `restored a snapshot at height ${churnRestoreHeight}`})`); + + await assertLocalServicesRunning([churnConfig]); + }); + + it('should sync again after the joined node loses its platform data', async () => { + const stopNodeTask = container.resolve('stopNodeTask'); + + await stopNodeTask(joinConfig).run({ + isVerbose: true, + isForce: true, + platformOnly: true, + }); + + // Wipe platform volumes only: Core keeps its synced chain, so this is a + // node that has lost Drive and Tenderdash state and must state sync from + // scratch, not a brand new node. + const resetNodeTask = container.resolve('resetNodeTask'); + + await resetNodeTask(joinConfig).run({ + isVerbose: true, + isForce: true, + isPlatformOnlyReset: true, + isHardReset: false, + }); + + // A pooled connection to the old container would outlive the wipe + resetEvoSdkCache(); + + writeConfigTemplates(joinConfig); + + const startNodeTask = container.resolve('startNodeTask'); + + // From here on, only a restore logged after this moment counts. The wipe + // does not necessarily replace the container, so without this the first + // join's restore line would still be visible and would satisfy the + // assertion below even if this node never re-synced at all. + markBoot(joinConfig); + + await startNodeTask(joinConfig).run({ + isVerbose: true, + platformOnly: true, + }); + + const { syncInfo, tenderdashObservations } = await watchStateSync(joinConfig, { + log: record, + }); + + expect(syncInfo, 're-joined node Tenderdash never responded on RPC').to.exist(); + expect(syncInfo.catching_up, 're-joined node is still catching up').to.be.false(); + + const restoreHeight = await getRestoreHeightSinceBoot(joinConfig); + + expect( + restoreHeight, + 're-joined node replayed blocks from genesis instead of restoring a snapshot', + ).to.be.a('number'); + + // Freshly fetched so backfill has settled, as in the first join + const { sync_info: rejoinSyncInfo } = await getTenderdashStatus(joinConfig); + + expect( + parseInt(rejoinSyncInfo.earliest_block_height, 10), + 'the re-joined node must again have a truncated history starting above genesis', + ).to.be.above(1); + + record(`re-joined node restored a snapshot at height ${restoreHeight}` + + ` (earliest_block_height=${rejoinSyncInfo.earliest_block_height})` + + ` after ${tenderdashObservations.length} state sync observations`); + }); + }); + + describe('fallback ladder', () => { + it('should fall back to block sync when no validator serves snapshots', async () => { + // Turn snapshot serving off across the network. drive-abci answers + // ListSnapshots with an empty set when disabled regardless of the + // checkpoints still on disk, so a joiner finds nothing to offer. + for (const config of configGroup) { + if (config.get('platform.enable')) { + config.set('platform.drive.abci.stateSync.snapshots.enabled', false); + } + } + + await configFileRepository.write(configFile); + configGroup.forEach(writeConfigTemplates); + + const restartNodeTask = container.resolve('restartNodeTask'); + + for (const config of configGroup) { + if (config.get('platform.enable')) { + await restartNodeTask(config).run({ + isVerbose: true, + isForce: true, + platformOnly: true, + }); + } + } + + resetEvoSdkCache(); + + await assertLocalServicesRunning(configGroup); + + fallbackConfig = await startJoinNode(fallbackConfigName, configGroup.length + 2); + + expect(fallbackConfig.get('platform.drive.tenderdash.stateSync.enabled')).to.be.true(); + + // The joiner discovers no snapshots, exhausts its state sync retries and + // then block syncs. That path replays every block, so unlike the state + // synced nodes above it keeps the full history from genesis. + const { syncInfo } = await watchStateSync(fallbackConfig, { + log: record, + timeoutMs: 30 * 60 * 1000, + }); + + expect(syncInfo, 'fallback join node Tenderdash never responded on RPC').to.exist(); + expect( + syncInfo.catching_up, + 'fallback join node never finished syncing without snapshots', + ).to.be.false(); + + // The inverse of the join assertion: with nothing offered there must be + // no restore at all. Checked from the ABCI app rather than from + // earliest_block_height, which a backfill would make ambiguous. + const restoreHeight = await getRestoreHeightSinceBoot(fallbackConfig); + + expect( + restoreHeight, + 'fallback join node restored a snapshot even though serving was disabled', + ).to.be.undefined(); + + // The contrast that makes the join scenario's truncation assertion + // meaningful: a block synced node replayed and kept every block + // (drive-abci never requests pruning through retain_height), so even + // with the shrunk evidence window its history reaches genesis. + expect( + parseInt(syncInfo.earliest_block_height, 10), + 'a block synced node must keep the full history from genesis', + ).to.equal(1); + + record(`fallback joiner block synced to latest_block_height=${syncInfo.latest_block_height}` + + ` with earliest_block_height=${syncInfo.earliest_block_height}` + + ' and no snapshot restore'); + + const { lines } = await getStateSyncLogExcerpt(dockerCompose, fallbackConfig); + + record(`fallback joiner log excerpt (${lines.length} lines):`); + lines.slice(-40).forEach((line) => record(` ${line}`)); + }); + + it('should still serve the seeded state after block syncing', async () => { + await waitForDapiReady(fallbackConfig); + + const checks = await verifySeededState(fallbackConfig, configGroup[0], seedManifest); + + record(`seeded state on the block synced node:\n${describeVerification(checks)}`); + + const missing = checks.filter(({ present }) => !present); + + expect( + missing.length, + `the block synced node did not serve seeded state:\n${describeVerification(missing)}`, + ).to.equal(0); + }); + }); + + describe('stop', () => { + it('should stop join nodes and local network', async () => { + const stopNodeTask = await container.resolve('stopNodeTask'); + + const joinConfigs = [joinConfig, churnConfig, fallbackConfig].filter(Boolean); + + for (const config of [...joinConfigs, ...configGroup.slice().reverse()]) { + const task = stopNodeTask(config); + await task.run({ + isVerbose: true, + isForce: true, + }); + } + + await assertLocalServicesRunning([...configGroup, ...joinConfigs], false); + }); + }); +}); diff --git a/packages/dashmate/test/unit/config/configFile/migrateConfigFileFactory.spec.js b/packages/dashmate/test/unit/config/configFile/migrateConfigFileFactory.spec.js index 5fc22d104e0..265a2c160f4 100644 --- a/packages/dashmate/test/unit/config/configFile/migrateConfigFileFactory.spec.js +++ b/packages/dashmate/test/unit/config/configFile/migrateConfigFileFactory.spec.js @@ -234,6 +234,51 @@ describe('migrateConfigFileFactory', () => { } }); + it('should add state sync options to a config stamped before they existed', async () => { + // The schema now requires the state sync options, so a config written + // before they existed cannot be loaded until the migration adds them. + const fromVersion = '4.2.0-dev.5'; + const { version } = JSON.parse(fs.readFileSync(path.join(PACKAGE_ROOT_DIR, 'package.json'), 'utf8')); + + const configFileData = createConfigFile().toObject(); + configFileData.configFormatVersion = fromVersion; + for (const options of Object.values(configFileData.configs)) { + delete options.platform.drive.tenderdash.stateSync; + delete options.platform.drive.abci.stateSync; + } + + const migrated = migrateConfigFile(configFileData, fromVersion, version); + + for (const [name, options] of Object.entries(migrated.configs)) { + // A local network genesis starts every node from scratch, so the local + // preset neither consumes nor serves snapshots. + const enabled = !(name === 'local' || options.group === 'local'); + + expect(options.platform.drive.tenderdash.stateSync).to.deep.equal( + { + enabled, + retries: 3, + chunkRequestTimeout: '15s', + fetchersCount: 4, + }, + `tenderdash state sync options not added for ${name}`, + ); + expect(options.platform.drive.abci.stateSync).to.deep.equal( + { + snapshots: { + enabled, + frequencySeconds: 600, + maxCount: 6, + }, + }, + `drive snapshot options not added for ${name}`, + ); + + expect(() => new Config(name, options), `migrated ${name} config does not load`) + .to.not.throw(); + } + }); + it('should load a config a development build stamped with its own prerelease version', async () => { // A development build records its own package version in the config, so // every node running one is stamped at a prerelease of the next release. diff --git a/packages/dashmate/test/unit/config/stateSyncOptions.spec.js b/packages/dashmate/test/unit/config/stateSyncOptions.spec.js new file mode 100644 index 00000000000..b455106f294 --- /dev/null +++ b/packages/dashmate/test/unit/config/stateSyncOptions.spec.js @@ -0,0 +1,99 @@ +import HomeDir from '../../../src/config/HomeDir.js'; +import getBaseConfigFactory from '../../../configs/defaults/getBaseConfigFactory.js'; +import getLocalConfigFactory from '../../../configs/defaults/getLocalConfigFactory.js'; + +describe('state sync options', () => { + let getBaseConfig; + + beforeEach(() => { + getBaseConfig = getBaseConfigFactory(HomeDir.createTemp()); + }); + + describe('defaults', () => { + it('should enable consuming and serving snapshots on the base config', () => { + const config = getBaseConfig(); + + expect(config.get('platform.drive.tenderdash.stateSync')).to.deep.equal({ + enabled: true, + retries: 3, + chunkRequestTimeout: '15s', + fetchersCount: 4, + }); + + expect(config.get('platform.drive.abci.stateSync')).to.deep.equal({ + snapshots: { + enabled: true, + frequencySeconds: 600, + maxCount: 6, + }, + }); + }); + + // A local network genesis starts every node from scratch at the same time, + // so there is no populated peer to sync from and nothing worth serving. + it('should disable consuming and serving snapshots on the local preset', () => { + const config = getLocalConfigFactory(getBaseConfig)(); + + expect(config.get('platform.drive.tenderdash.stateSync.enabled')).to.be.false(); + expect(config.get('platform.drive.abci.stateSync.snapshots.enabled')).to.be.false(); + }); + }); + + describe('schema', () => { + let config; + + beforeEach(() => { + config = getBaseConfig(); + }); + + it('should accept retries of 0 (disable retries) but not negative', () => { + config.set('platform.drive.tenderdash.stateSync.retries', 0); + + expect(() => config.set('platform.drive.tenderdash.stateSync.retries', -1)) + .to.throw(); + }); + + it('should accept 1 to 64 fetchers only', () => { + config.set('platform.drive.tenderdash.stateSync.fetchersCount', 1); + config.set('platform.drive.tenderdash.stateSync.fetchersCount', 64); + + expect(() => config.set('platform.drive.tenderdash.stateSync.fetchersCount', 0)) + .to.throw(); + expect(() => config.set('platform.drive.tenderdash.stateSync.fetchersCount', 65)) + .to.throw(); + }); + + // Tenderdash 1.7 rejects a statesync chunk-request-timeout below 5 seconds. + it('should reject a chunk request timeout below the 5s Tenderdash minimum', () => { + ['5s', '15s', '1.5m', '0.5m', '2h', '0.1h', '5000ms', '30000ms'].forEach((valid) => { + config.set('platform.drive.tenderdash.stateSync.chunkRequestTimeout', valid); + }); + + ['0', '4s', '4.9s', '4999ms', '500ms', '0.05m', 'nonsense', 15].forEach((invalid) => { + expect(() => config.set('platform.drive.tenderdash.stateSync.chunkRequestTimeout', invalid), String(invalid)) + .to.throw(); + }); + }); + + it('should reject a snapshot frequency below one minute', () => { + config.set('platform.drive.abci.stateSync.snapshots.frequencySeconds', 60); + + expect(() => config.set('platform.drive.abci.stateSync.snapshots.frequencySeconds', 59)) + .to.throw(); + }); + + it('should keep at least two snapshots', () => { + config.set('platform.drive.abci.stateSync.snapshots.maxCount', 2); + + expect(() => config.set('platform.drive.abci.stateSync.snapshots.maxCount', 1)) + .to.throw(); + }); + + it('should reject unknown state sync options', () => { + expect(() => config.set('platform.drive.tenderdash.stateSync.maxConcurrentListSnapshots', 100)) + .to.throw(); + expect(() => config.set('platform.drive.abci.stateSync.snapshots.frequency', 5)) + .to.throw(); + }); + }); +}); diff --git a/packages/dashmate/test/unit/doctor/verifySystemRequirementsFactory.spec.js b/packages/dashmate/test/unit/doctor/verifySystemRequirementsFactory.spec.js index 38590b3fbdc..4f978ce530d 100644 --- a/packages/dashmate/test/unit/doctor/verifySystemRequirementsFactory.spec.js +++ b/packages/dashmate/test/unit/doctor/verifySystemRequirementsFactory.spec.js @@ -1,5 +1,6 @@ import verifySystemRequirementsFactory from '../../../src/doctor/verifySystemRequirementsFactory.js'; import Problem from '../../../src/doctor/Problem.js'; +import { SEVERITY } from '../../../src/doctor/Prescription.js'; describe('verifySystemRequirementsFactory', () => { let verifySystemRequirements; @@ -130,6 +131,53 @@ describe('verifySystemRequirementsFactory', () => { expect(problems[0]).to.be.an.instanceOf(Problem); expect(problems[0].getDescription()).to.include('50.00GB of available disk space detected'); }); + + it('should add headroom for state sync snapshots', () => { + const systemInfo = { + diskSpace: { available: 12 * 1024 ** 3 }, + }; + + // 12GB clears the 5GB override on its own... + const problems = verifySystemRequirements(systemInfo, false, { + diskSpace: 5, + }); + + expect(problems).to.have.lengthOf(0); + + // ...but not with the snapshot headroom on top + const problemsWithSnapshots = verifySystemRequirements(systemInfo, false, { + diskSpace: 5, + stateSyncSnapshotsEnabled: true, + }); + + expect(problemsWithSnapshots).to.have.lengthOf(1); + expect(problemsWithSnapshots[0].getDescription()) + .to.include('At least 15GB is required (including 10GB headroom for state sync snapshots)'); + }); + + it('should not downgrade severity when snapshot headroom widens the requirement', () => { + const systemInfo = { + diskSpace: { available: 2 * 1024 ** 3 }, + }; + + // 3GB short of the 5GB base minimum: HIGH + const problems = verifySystemRequirements(systemInfo, false, { + diskSpace: 5, + }); + + expect(problems).to.have.lengthOf(1); + expect(problems[0].getSeverity()).to.equal(SEVERITY.HIGH); + + // The 10GB headroom widens the deficit to 13GB, which must stay HIGH + // rather than fall over the 5GB near-threshold cutoff into MEDIUM + const problemsWithSnapshots = verifySystemRequirements(systemInfo, false, { + diskSpace: 5, + stateSyncSnapshotsEnabled: true, + }); + + expect(problemsWithSnapshots).to.have.lengthOf(1); + expect(problemsWithSnapshots[0].getSeverity()).to.equal(SEVERITY.HIGH); + }); }); it('should not return any problems if all requirements are met', () => { diff --git a/packages/dashmate/test/unit/tasks/setupLocalJoinNodeTaskFactory.spec.js b/packages/dashmate/test/unit/tasks/setupLocalJoinNodeTaskFactory.spec.js new file mode 100644 index 00000000000..cb4650ca6b2 --- /dev/null +++ b/packages/dashmate/test/unit/tasks/setupLocalJoinNodeTaskFactory.spec.js @@ -0,0 +1,278 @@ +import HomeDir from '../../../src/config/HomeDir.js'; +import ConfigFile from '../../../src/config/configFile/ConfigFile.js'; +import { PRESET_LOCAL } from '../../../src/constants.js'; +import getBaseConfigFactory from '../../../configs/defaults/getBaseConfigFactory.js'; +import getLocalConfigFactory from '../../../configs/defaults/getLocalConfigFactory.js'; +import generateTenderdashNodeKey from '../../../src/tenderdash/generateTenderdashNodeKey.js'; +import deriveTenderdashNodeId from '../../../src/tenderdash/deriveTenderdashNodeId.js'; +import wireLocalTenderdashNode from '../../../src/listr/tasks/setup/local/wireLocalTenderdashNode.js'; +import setupLocalJoinNodeTaskFactory from '../../../src/listr/tasks/setup/local/setupLocalJoinNodeTaskFactory.js'; + +describe('setupLocalJoinNodeTaskFactory', () => { + const CHAIN_ID = 'dashmate_local_42'; + const EXTERNAL_IP = '192.168.65.2'; + + let homeDir; + let configFile; + let groupConfigs; + let platformConfigs; + let templateConfig; + let resolveDockerHostIp; + let obtainSelfSignedCertificateTask; + let setupLocalJoinNodeTask; + + beforeEach(function beforeEach() { + homeDir = HomeDir.createTemp(); + + const getBaseConfig = getBaseConfigFactory(homeDir); + templateConfig = getLocalConfigFactory(getBaseConfig)(); + + configFile = new ConfigFile( + [templateConfig], + '4.2.0', + 'abcdef12', + null, + 'local', + ); + + // Recreate the shape of an already set up local group: + // three validators and a platform-disabled seed node + groupConfigs = ['local_1', 'local_2', 'local_3', 'local_seed'] + .map((name) => configFile.createConfig(name, PRESET_LOCAL)); + + groupConfigs.forEach((config, i) => { + config.set('group', 'local'); + config.set('externalIp', EXTERNAL_IP); + config.set('core.p2p.port', config.get('core.p2p.port') + (i * 100)); + + if (config.getName() === 'local_seed') { + config.set('platform.enable', false); + config.set('platform.drive.tenderdash.mode', 'seed'); + } else { + config.set('platform.drive.tenderdash.mode', 'validator'); + + const nodeKey = generateTenderdashNodeKey(); + config.set('platform.drive.tenderdash.node.id', deriveTenderdashNodeId(nodeKey)); + config.set('platform.drive.tenderdash.node.key', nodeKey); + config.set( + 'platform.drive.tenderdash.p2p.port', + config.get('platform.drive.tenderdash.p2p.port') + (i * 100), + ); + config.set('platform.drive.tenderdash.genesis.chain_id', CHAIN_ID); + } + + config.set('core.spork.address', 'spork-address'); + config.set('core.spork.privateKey', 'spork-private-key'); + }); + + platformConfigs = groupConfigs.filter((config) => config.get('platform.enable')); + + resolveDockerHostIp = this.sinon.stub().resolves(EXTERNAL_IP); + obtainSelfSignedCertificateTask = this.sinon.stub().resolves(); + + setupLocalJoinNodeTask = setupLocalJoinNodeTaskFactory( + configFile, + resolveDockerHostIp, + obtainSelfSignedCertificateTask, + ); + }); + + afterEach(() => { + homeDir.remove(); + }); + + describe('wireLocalTenderdashNode', () => { + it('should wire chain id, peers without self and quorum type', () => { + const config = groupConfigs[0]; + + wireLocalTenderdashNode(config, CHAIN_ID, platformConfigs); + + expect(config.get('platform.drive.tenderdash.genesis.chain_id')).to.equal(CHAIN_ID); + + const persistentPeers = config.get('platform.drive.tenderdash.p2p.persistentPeers'); + + expect(persistentPeers).to.have.length(2); + expect(persistentPeers.map((peer) => peer.id)).to.not.include( + config.get('platform.drive.tenderdash.node.id'), + ); + persistentPeers.forEach((peer) => { + expect(peer.host).to.equal(EXTERNAL_IP); + }); + + // The local preset's validator set quorum is llmqType 106 + expect(config.get('platform.drive.tenderdash.genesis.validator_quorum_type')) + .to.equal(106); + }); + }); + + describe('setupLocalJoinNodeTask', () => { + let joinConfig; + + beforeEach(async () => { + await setupLocalJoinNodeTask(groupConfigs).run(); + + joinConfig = configFile.getConfig('local_join'); + }); + + it('should create a platform-enabled full node config without masternode', () => { + expect(joinConfig.get('group')).to.equal('local'); + expect(joinConfig.get('platform.enable')).to.be.true(); + expect(joinConfig.get('platform.drive.tenderdash.mode')).to.equal('full'); + expect(joinConfig.get('core.masternode.enable')).to.be.false(); + }); + + it('should enable state sync for the joining node only', () => { + expect(joinConfig.get('platform.drive.tenderdash.stateSync.enabled')).to.be.true(); + + // Snapshot serving stays at the preset default for the joiner + expect(joinConfig.get('platform.drive.abci.stateSync.snapshots.enabled')) + .to.equal(templateConfig.get('platform.drive.abci.stateSync.snapshots.enabled')); + }); + + it('should wire the node into the existing Tenderdash network', () => { + expect(joinConfig.get('platform.drive.tenderdash.genesis.chain_id')).to.equal(CHAIN_ID); + + const persistentPeers = joinConfig.get('platform.drive.tenderdash.p2p.persistentPeers'); + + expect(persistentPeers).to.have.length(3); + expect(persistentPeers.map((peer) => peer.id)).to.have.members( + platformConfigs.map((config) => config.get('platform.drive.tenderdash.node.id')), + ); + expect(persistentPeers.map((peer) => peer.port)).to.have.members( + platformConfigs.map((config) => config.get('platform.drive.tenderdash.p2p.port')), + ); + }); + + it('should use a fresh Tenderdash node identity', () => { + const nodeId = joinConfig.get('platform.drive.tenderdash.node.id'); + + expect(nodeId).to.be.a('string').and.not.empty(); + expect(joinConfig.get('platform.drive.tenderdash.node.key')).to.be.a('string').and.not.empty(); + + const validatorNodeIds = platformConfigs + .map((config) => config.get('platform.drive.tenderdash.node.id')); + + expect(validatorNodeIds).to.not.include(nodeId); + }); + + it('should take the next port offset after the seed node', () => { + // 3 validators (offsets 0-2) + seed (3) means the joiner continues at 4 + const expectedOffset = 400; + + expect(joinConfig.get('core.p2p.port')) + .to.equal(templateConfig.get('core.p2p.port') + expectedOffset); + expect(joinConfig.get('platform.drive.tenderdash.p2p.port')) + .to.equal(templateConfig.get('platform.drive.tenderdash.p2p.port') + expectedOffset); + expect(joinConfig.get('platform.drive.tenderdash.rpc.port')) + .to.equal(templateConfig.get('platform.drive.tenderdash.rpc.port') + expectedOffset); + expect(joinConfig.get('platform.gateway.listeners.dapiAndDrive.port')) + .to.equal(templateConfig.get('platform.gateway.listeners.dapiAndDrive.port') + expectedOffset); + + const subnet = joinConfig.get('docker.network.subnet').split('.'); + expect(subnet[2]).to.equal('5'); + }); + + it('should join the Core network through the seed node with group sporks', () => { + const seedConfig = groupConfigs.find((config) => config.getName() === 'local_seed'); + + expect(joinConfig.get('core.p2p.seeds')).to.deep.equal([{ + host: EXTERNAL_IP, + port: seedConfig.get('core.p2p.port'), + }]); + + expect(joinConfig.get('core.spork.address')).to.equal('spork-address'); + expect(joinConfig.get('core.spork.privateKey')).to.equal('spork-private-key'); + + expect(joinConfig.get('core.rpc.users.dashmate.password')).to.not.equal( + templateConfig.get('core.rpc.users.dashmate.password'), + ); + }); + + it('should obtain a self-signed certificate for the joining node', () => { + expect(obtainSelfSignedCertificateTask).to.have.been.calledOnce(); + expect(obtainSelfSignedCertificateTask.firstCall.args[0]).to.equal(joinConfig); + }); + }); + + describe('genesis inheritance', () => { + it('should inherit the group genesis, customized consensus params included', async () => { + // The way the state sync e2e suite customizes the network genesis: + // an evidence window small enough that post-restore backfill stops + // above genesis. The joiner must render the same genesis or its + // Tenderdash disagrees with the network about the chain it is on. + platformConfigs.forEach((config) => { + config.set('platform.drive.tenderdash.genesis.consensus_params.evidence', { + max_age: '10', + max_age_num_blocks: '10', + max_age_duration: '10000000000', + }); + }); + + await setupLocalJoinNodeTask(groupConfigs).run(); + + const joinConfig = configFile.getConfig('local_join'); + + expect( + joinConfig.get('platform.drive.tenderdash.genesis.consensus_params.evidence'), + ).to.deep.equal({ + max_age: '10', + max_age_num_blocks: '10', + max_age_duration: '10000000000', + }); + + // The node-specific wiring still lands on top of the inherited genesis + expect(joinConfig.get('platform.drive.tenderdash.genesis.chain_id')).to.equal(CHAIN_ID); + expect(joinConfig.get('platform.drive.tenderdash.genesis.validator_quorum_type')) + .to.equal(106); + }); + }); + + describe('additional join nodes', () => { + it('should honour an explicit config name and port offset', async () => { + await setupLocalJoinNodeTask(groupConfigs).run(); + + await setupLocalJoinNodeTask(groupConfigs, { + configName: 'local_join_second', + offsetIndex: groupConfigs.length + 1, + }).run(); + + const first = configFile.getConfig('local_join'); + const second = configFile.getConfig('local_join_second'); + + // A second joiner must not land on the first one's host ports + expect(second.get('platform.gateway.listeners.dapiAndDrive.port')) + .to.equal(templateConfig.get('platform.gateway.listeners.dapiAndDrive.port') + 500); + expect(second.get('platform.drive.tenderdash.rpc.port')) + .to.equal(templateConfig.get('platform.drive.tenderdash.rpc.port') + 500); + + expect(second.get('platform.gateway.listeners.dapiAndDrive.port')) + .to.not.equal(first.get('platform.gateway.listeners.dapiAndDrive.port')); + + expect(second.get('docker.network.subnet').split('.')[2]).to.equal('6'); + + // ...and must be a distinct Tenderdash node + expect(second.get('platform.drive.tenderdash.node.id')) + .to.not.equal(first.get('platform.drive.tenderdash.node.id')); + + expect(second.get('platform.drive.tenderdash.stateSync.enabled')).to.be.true(); + }); + }); + + describe('invalid groups', () => { + it('should fail clearly when the group has no seed node', async () => { + const groupWithoutSeed = groupConfigs + .filter((config) => config.getName() !== 'local_seed'); + + await expect(setupLocalJoinNodeTask(groupWithoutSeed).run()) + .to.be.rejectedWith('no local_seed config'); + }); + + it('should fail clearly when the group has no platform-enabled nodes', async () => { + const seedOnlyGroup = groupConfigs + .filter((config) => config.getName() === 'local_seed'); + + await expect(setupLocalJoinNodeTask(seedOnlyGroup).run()) + .to.be.rejectedWith('no platform-enabled configs'); + }); + }); +}); diff --git a/packages/dashmate/test/unit/templates/tenderdashConfigTemplate.spec.js b/packages/dashmate/test/unit/templates/tenderdashConfigTemplate.spec.js new file mode 100644 index 00000000000..3d4f323fee6 --- /dev/null +++ b/packages/dashmate/test/unit/templates/tenderdashConfigTemplate.spec.js @@ -0,0 +1,51 @@ +import getBaseConfigFactory from '../../../configs/defaults/getBaseConfigFactory.js'; +import HomeDir from '../../../src/config/HomeDir.js'; +import renderServiceTemplatesFactory from '../../../src/templates/renderServiceTemplatesFactory.js'; +import renderTemplateFactory from '../../../src/templates/renderTemplateFactory.js'; + +describe('tenderdash config template', () => { + let config; + let renderServiceTemplates; + + beforeEach(() => { + const getBaseConfig = getBaseConfigFactory(HomeDir.createTemp()); + config = getBaseConfig(); + + const renderTemplate = renderTemplateFactory(); + renderServiceTemplates = renderServiceTemplatesFactory(renderTemplate); + }); + + const renderTenderdashConfig = () => renderServiceTemplates(config)['platform/drive/tenderdash/config.toml']; + + it('should render the statesync section from config defaults', () => { + const toml = renderTenderdashConfig(); + + expect(toml).to.include('enable = true'); + expect(toml).to.include('use-p2p = true'); + expect(toml).to.include('retries = 3'); + expect(toml).to.include('chunk-request-timeout = "15s"'); + expect(toml).to.include('fetchers = "4"'); + + // Light client trust options were removed in Tenderdash 1.7 + expect(toml).to.not.include('trust-height'); + expect(toml).to.not.include('trust-period'); + + expect(toml).to.not.include('undefined'); + }); + + it('should render statesync consuming disabled', () => { + config.set('platform.drive.tenderdash.stateSync.enabled', false); + + expect(renderTenderdashConfig()).to.include('enable = false'); + }); + + it('should route snapshot serving to the drive grpc app', () => { + const toml = renderTenderdashConfig(); + + expect(toml).to.include('ListSnapshots:grpc:drive_abci:26670'); + expect(toml).to.include('LoadSnapshotChunk:grpc:drive_abci:26670'); + expect(toml).to.include('*:socket:tcp://drive_abci:26658'); + expect(toml).to.include('{ "list_snapshots" = 10 }'); + expect(toml).to.include('{ "load_snapshot_chunk" = 100 }'); + }); +}); diff --git a/packages/js-dapi-client/lib/dapiAddressProvider/ListDAPIAddressProvider.js b/packages/js-dapi-client/lib/dapiAddressProvider/ListDAPIAddressProvider.js index c13217e98d4..c6aa3979974 100644 --- a/packages/js-dapi-client/lib/dapiAddressProvider/ListDAPIAddressProvider.js +++ b/packages/js-dapi-client/lib/dapiAddressProvider/ListDAPIAddressProvider.js @@ -31,8 +31,16 @@ class ListDAPIAddressProvider { // This is a temporary fix for a localhost masternode. // On macOS, internal docker IP is used to register masternode, and it's // not really possible to bind to that address, so that workaround is introduced. + // + // Only addresses carrying such an unreachable docker-internal host are + // rewritten. An explicitly configured loopback address already names the + // exact gateway to talk to — dashmate e2e suites move the stock ports on + // purpose — and clobbering it with the stock local ports silently + // redirects every request to whichever network squats those ports on the + // machine. const network = networks.get(this.options.network); - if (network && network.regtestEnabled) { + const isLoopback = ['127.0.0.1', 'localhost'].includes(liveAddress.getHost()); + if (network && network.regtestEnabled && !isLoopback) { const randomNodeIndex = Math.floor(Math.random() * liveAddresses.length); liveAddress.protocol = 'https'; diff --git a/packages/js-dapi-client/test/unit/dapiAddressProvider/ListDAPIAddressProvider.spec.js b/packages/js-dapi-client/test/unit/dapiAddressProvider/ListDAPIAddressProvider.spec.js index 8b887a4ad78..28d55334152 100644 --- a/packages/js-dapi-client/test/unit/dapiAddressProvider/ListDAPIAddressProvider.spec.js +++ b/packages/js-dapi-client/test/unit/dapiAddressProvider/ListDAPIAddressProvider.spec.js @@ -116,6 +116,28 @@ describe('ListDAPIAddressProvider', () => { expect(liveAddress.protocol).to.equal('https'); expect(liveAddress.allowSelfSignedCertificate).to.be.true(); }); + + it('should not modify an explicitly configured loopback address', async () => { + options = { + network: 'local', + }; + + // A local network that moved its ports off the stock 2443 range + // (dashmate e2e suites do) is addressed explicitly; rewriting the port + // would redirect every request to whatever squats the stock ports. + const loopbackAddress = new DAPIAddress('127.0.0.1:45003:self-signed'); + + listDAPIAddressProvider = new ListDAPIAddressProvider( + [loopbackAddress], + options, + ); + + const liveAddress = await listDAPIAddressProvider.getLiveAddress(); + + expect(liveAddress.host).to.equal('127.0.0.1'); + expect(liveAddress.port).to.equal(45003); + expect(liveAddress.allowSelfSignedCertificate).to.be.true(); + }); }); describe('#hasLiveAddresses', () => { diff --git a/packages/rs-dpp/src/lib.rs b/packages/rs-dpp/src/lib.rs index 7a7c90a8080..a8ebf21b25b 100644 --- a/packages/rs-dpp/src/lib.rs +++ b/packages/rs-dpp/src/lib.rs @@ -61,6 +61,8 @@ pub mod core_subsidy; pub mod fee; pub mod nft; pub mod prefunded_specialized_balance; +/// Reduced platform state stored in replicated state for state sync reconstruction +pub mod reduced_platform_state; pub mod serialization; #[cfg(any( feature = "message-signing", diff --git a/packages/rs-dpp/src/reduced_platform_state/mod.rs b/packages/rs-dpp/src/reduced_platform_state/mod.rs new file mode 100644 index 00000000000..05ab4151e77 --- /dev/null +++ b/packages/rs-dpp/src/reduced_platform_state/mod.rs @@ -0,0 +1,94 @@ +//! Reduced platform state +//! +//! A minimal subset of the Platform state that is stored inside the replicated GroveDB +//! state (under the Misc tree), allowing a node that syncs via ABCI state sync to +//! reconstruct the full Platform state. The full Platform state itself is only persisted +//! to GroveDB aux storage, which is not replicated by GroveDB state sync. + +use crate::serialization::{PlatformDeserializableFromVersionedStructure, PlatformSerializable}; +use crate::ProtocolError; +use bincode::{Decode, Encode}; +use platform_version::version::PlatformVersion; + +pub mod v0; + +use v0::ReducedPlatformStateV0; + +/// Reduced Platform State (platform-versioned wrapper) +#[derive(Clone, Debug, PartialEq, Encode, Decode, derive_more::From)] +pub enum ReducedPlatformState { + /// Version 0 + V0(ReducedPlatformStateV0), +} + +impl PlatformSerializable for ReducedPlatformState { + type Error = ProtocolError; + + fn serialize_to_bytes(&self) -> Result, Self::Error> { + let config = bincode::config::standard(); + bincode::encode_to_vec(self, config).map_err(|e| { + ProtocolError::PlatformSerializationError(format!( + "cannot serialize ReducedPlatformState: {}", + e + )) + }) + } +} + +impl PlatformDeserializableFromVersionedStructure for ReducedPlatformState { + fn versioned_deserialize( + data: &[u8], + _platform_version: &PlatformVersion, + ) -> Result + where + Self: Sized, + { + // The version of the structure is encoded in the enum discriminant, so the + // platform version is not needed to pick the variant. + let config = bincode::config::standard(); + bincode::decode_from_slice(data, config) + .map_err(|e| { + ProtocolError::PlatformDeserializationError(format!( + "cannot deserialize ReducedPlatformState: {}", + e + )) + }) + .map(|(object, _)| object) + } +} + +#[cfg(test)] +mod tests { + use super::v0::{ReducedBlockInfoV0, ReducedPlatformStateV0}; + use super::*; + use crate::block::block_info::BlockInfo; + + #[test] + fn should_roundtrip_reduced_platform_state_serialization() { + let state = ReducedPlatformState::V0(ReducedPlatformStateV0 { + last_committed_block_info: Some(ReducedBlockInfoV0 { + basic_info: BlockInfo::default_with_time(1_700_000_000_000), + app_hash: None, + quorum_hash: [1u8; 32].into(), + block_id_hash: None, + proposer_pro_tx_hash: [2u8; 32].into(), + signature: None, + round: 3, + }), + current_protocol_version_in_consensus: 15, + next_epoch_protocol_version: 15, + current_validator_set_quorum_hash: [4u8; 32].into(), + next_validator_set_quorum_hash: Some([5u8; 32].into()), + previous_fee_versions: [(0u16, 1u32)].into_iter().collect(), + quorum_positions: vec![[4u8; 32].into(), [5u8; 32].into()], + proposed_core_chain_locked_height: 1000, + }); + + let bytes = state.serialize_to_bytes().expect("should serialize"); + let restored = + ReducedPlatformState::versioned_deserialize(&bytes, PlatformVersion::latest()) + .expect("should deserialize"); + + assert_eq!(state, restored); + } +} diff --git a/packages/rs-dpp/src/reduced_platform_state/v0/mod.rs b/packages/rs-dpp/src/reduced_platform_state/v0/mod.rs new file mode 100644 index 00000000000..edda9bd2cdb --- /dev/null +++ b/packages/rs-dpp/src/reduced_platform_state/v0/mod.rs @@ -0,0 +1,61 @@ +use crate::block::block_info::BlockInfo; +use crate::fee::default_costs::EpochIndexFeeVersionsForStorage; +use crate::util::deserializer::ProtocolVersion; +use bincode::{Decode, Encode}; +use platform_value::Bytes32; + +/// Block information persisted as part of the reduced platform state. +/// +/// The reduced state is written while the block is still being executed, before it is +/// signed and before the resulting app hash is known, so `app_hash`, `block_id_hash` and +/// `signature` are `Option`s rather than zero-filled placeholders. They are `None` when +/// stored and are filled in (where possible) during state reconstruction. +#[derive(Clone, Debug, PartialEq, Encode, Decode)] +pub struct ReducedBlockInfoV0 { + /// Basic block info (height, core height, time, epoch) + pub basic_info: BlockInfo, + /// The app hash resulting from this block; unknown at store time + pub app_hash: Option, + /// The quorum that signed (or will sign) this block + pub quorum_hash: Bytes32, + /// The block id hash; unknown at store time + pub block_id_hash: Option, + /// The block proposer's pro tx hash + pub proposer_pro_tx_hash: Bytes32, + /// The block signature; unknown at store time + pub signature: Option<[u8; 96]>, + /// The consensus round that produced this block + pub round: u32, +} + +/// Reduced Platform State V0. +/// +/// This minimal version of the Platform state is written into GroveDB (under the Misc +/// tree, hence below the root hash) on every block proposal. Because it is part of the +/// replicated state, a freshly state-synced node can read it back and reconstruct the +/// full in-memory Platform state, which is otherwise only persisted to non-replicated +/// GroveDB aux storage. +#[derive(Clone, Debug, PartialEq, Encode, Decode)] +pub struct ReducedPlatformStateV0 { + /// Info about the block that was being processed when this state was written + /// (it becomes the last committed block once the block finalizes) + pub last_committed_block_info: Option, + /// Current protocol version in consensus + pub current_protocol_version_in_consensus: ProtocolVersion, + /// Upcoming protocol version + pub next_epoch_protocol_version: ProtocolVersion, + /// Current validator set quorum hash + pub current_validator_set_quorum_hash: Bytes32, + /// Next validator set quorum hash + pub next_validator_set_quorum_hash: Option, + /// Fee versions of previous epochs, stored by fee version number so they can be + /// restored faithfully on reconstruction + pub previous_fee_versions: EpochIndexFeeVersionsForStorage, + /// Ordered list of quorum hashes reflecting validator set quorum positions + // TODO: optimize this to not store the whole quorum hash, but only some index + pub quorum_positions: Vec, + /// Core chain locked height, as provided in RequestProcessProposal ABCI message; + /// note this can differ from the one in RequestPrepareProposal, as it can be + /// modified by the proposer. + pub proposed_core_chain_locked_height: u32, +} diff --git a/packages/rs-drive-abci/.env.local b/packages/rs-drive-abci/.env.local index c0e3ac3347a..4eb320b87aa 100644 --- a/packages/rs-drive-abci/.env.local +++ b/packages/rs-drive-abci/.env.local @@ -90,3 +90,10 @@ GROVEDB_VISUALIZER_ENABLED=false GROVEDB_VISUALIZER_ADDRESS=127.0.0.1:8083 NETWORK=regtest + +# ABCI state sync snapshots (serving side; disabled by default) +SNAPSHOTS_ENABLED=false +SNAPSHOTS_FREQUENCY_SECONDS=600 +MAX_NUM_SNAPSHOTS=3 +# CHECKPOINTS_PATH defaults to /checkpoints when unset +#CHECKPOINTS_PATH= diff --git a/packages/rs-drive-abci/.env.mainnet b/packages/rs-drive-abci/.env.mainnet index 65409c1d0a3..214b2b6c001 100644 --- a/packages/rs-drive-abci/.env.mainnet +++ b/packages/rs-drive-abci/.env.mainnet @@ -92,3 +92,10 @@ GROVEDB_VISUALIZER_ADDRESS=127.0.0.1:8083 PROPOSER_TX_PROCESSING_TIME_LIMIT=5000 NETWORK=mainnet + +# ABCI state sync snapshots (serving side; disabled by default) +SNAPSHOTS_ENABLED=false +SNAPSHOTS_FREQUENCY_SECONDS=600 +MAX_NUM_SNAPSHOTS=3 +# CHECKPOINTS_PATH defaults to /checkpoints when unset +#CHECKPOINTS_PATH= diff --git a/packages/rs-drive-abci/.env.testnet b/packages/rs-drive-abci/.env.testnet index 9e85d109c5f..b5a8379edc6 100644 --- a/packages/rs-drive-abci/.env.testnet +++ b/packages/rs-drive-abci/.env.testnet @@ -92,3 +92,10 @@ GROVEDB_VISUALIZER_ADDRESS=127.0.0.1:8083 PROPOSER_TX_PROCESSING_TIME_LIMIT=5000 NETWORK=testnet + +# ABCI state sync snapshots (serving side; disabled by default) +SNAPSHOTS_ENABLED=false +SNAPSHOTS_FREQUENCY_SECONDS=600 +MAX_NUM_SNAPSHOTS=3 +# CHECKPOINTS_PATH defaults to /checkpoints when unset +#CHECKPOINTS_PATH= diff --git a/packages/rs-drive-abci/src/abci/app/check_tx.rs b/packages/rs-drive-abci/src/abci/app/check_tx.rs index 170eb519599..4ed32b2fbed 100644 --- a/packages/rs-drive-abci/src/abci/app/check_tx.rs +++ b/packages/rs-drive-abci/src/abci/app/check_tx.rs @@ -1,7 +1,8 @@ -use crate::abci::app::PlatformApplication; +use crate::abci::app::{PlatformApplication, SnapshotManagerApplication}; use crate::abci::handler; use crate::error::Error; use crate::platform_types::platform::Platform; +use crate::platform_types::snapshot::SnapshotManager; use crate::rpc::core::CoreRPCLike; use crate::utils::spawn_blocking_task_with_name_if_supported; use async_trait::async_trait; @@ -22,6 +23,8 @@ where /// Platform platform: Arc>, core_rpc: Arc, + /// The snapshot manager, pinning checkpoints that are being served to peers + snapshot_manager: SnapshotManager, } impl PlatformApplication for CheckTxAbciApplication @@ -33,13 +36,26 @@ where } } +impl SnapshotManagerApplication for CheckTxAbciApplication +where + C: CoreRPCLike + Send + Sync + 'static, +{ + fn snapshot_manager(&self) -> &SnapshotManager { + &self.snapshot_manager + } +} + impl CheckTxAbciApplication where C: CoreRPCLike + Send + Sync + 'static, { /// Create new ABCI app pub fn new(platform: Arc>, core_rpc: Arc) -> Self { - Self { platform, core_rpc } + Self { + platform, + core_rpc, + snapshot_manager: SnapshotManager::new(), + } } } @@ -92,6 +108,24 @@ where .await .map_err(|error| tonic::Status::internal(format!("check tx panics: {}", error)))? } + + async fn list_snapshots( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + handler::list_snapshots(self, request.into_inner()) + .map(tonic::Response::new) + .map_err(error_into_status) + } + + async fn load_snapshot_chunk( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + handler::load_snapshot_chunk(self, request.into_inner()) + .map(tonic::Response::new) + .map_err(error_into_status) + } } pub fn error_into_status(error: Error) -> tonic::Status { diff --git a/packages/rs-drive-abci/src/abci/app/consensus.rs b/packages/rs-drive-abci/src/abci/app/consensus.rs index 43b6d518db8..8147a1a4f49 100644 --- a/packages/rs-drive-abci/src/abci/app/consensus.rs +++ b/packages/rs-drive-abci/src/abci/app/consensus.rs @@ -1,10 +1,13 @@ -use crate::abci::app::{BlockExecutionApplication, PlatformApplication, TransactionalApplication}; +use crate::abci::app::{ + BlockExecutionApplication, PlatformApplication, StateSyncApplication, TransactionalApplication, +}; use crate::abci::handler; use crate::abci::handler::error::error_into_exception; use crate::error::execution::ExecutionError; use crate::error::Error; use crate::execution::types::block_execution_context::BlockExecutionContext; use crate::platform_types::platform::Platform; +use crate::platform_types::snapshot::SnapshotFetchingSession; use crate::rpc::core::CoreRPCLike; use dpp::version::PlatformVersion; use drive::grovedb::Transaction; @@ -23,6 +26,8 @@ pub struct ConsensusAbciApplication<'a, C> { transaction: RwLock>>, /// The current block execution context block_execution_context: RwLock>, + /// The state sync transfer currently in progress, if any + snapshot_fetching_session: RwLock>>, } impl<'a, C> ConsensusAbciApplication<'a, C> { @@ -32,6 +37,7 @@ impl<'a, C> ConsensusAbciApplication<'a, C> { platform, transaction: Default::default(), block_execution_context: Default::default(), + snapshot_fetching_session: Default::default(), } } } @@ -42,6 +48,16 @@ impl PlatformApplication for ConsensusAbciApplication<'_, C> { } } +impl<'a, C> StateSyncApplication<'a, C> for ConsensusAbciApplication<'a, C> { + fn snapshot_fetching_session(&self) -> &RwLock>> { + &self.snapshot_fetching_session + } + + fn platform(&self) -> &'a Platform { + self.platform + } +} + impl BlockExecutionApplication for ConsensusAbciApplication<'_, C> { fn block_execution_context(&self) -> &RwLock> { &self.block_execution_context @@ -105,7 +121,7 @@ mod tests { crate::test::helpers::setup::TestPlatformBuilder::new().build_with_mock_rpc(); let app = ConsensusAbciApplication::::new(&platform.platform); - let _platform_ref = app.platform(); + let _platform_ref = PlatformApplication::platform(&app); } #[test] @@ -222,4 +238,18 @@ where ) -> Result { handler::verify_vote_extension(self, request).map_err(error_into_exception) } + + fn offer_snapshot( + &self, + request: proto::RequestOfferSnapshot, + ) -> Result { + handler::offer_snapshot(self, request).map_err(error_into_exception) + } + + fn apply_snapshot_chunk( + &self, + request: proto::RequestApplySnapshotChunk, + ) -> Result { + handler::apply_snapshot_chunk(self, request).map_err(error_into_exception) + } } diff --git a/packages/rs-drive-abci/src/abci/app/full.rs b/packages/rs-drive-abci/src/abci/app/full.rs index bd290b87156..e5274f01f30 100644 --- a/packages/rs-drive-abci/src/abci/app/full.rs +++ b/packages/rs-drive-abci/src/abci/app/full.rs @@ -1,10 +1,14 @@ -use crate::abci::app::{BlockExecutionApplication, PlatformApplication, TransactionalApplication}; +use crate::abci::app::{ + BlockExecutionApplication, PlatformApplication, SnapshotManagerApplication, + StateSyncApplication, TransactionalApplication, +}; use crate::abci::handler; use crate::abci::handler::error::error_into_exception; use crate::error::execution::ExecutionError; use crate::error::Error; use crate::execution::types::block_execution_context::BlockExecutionContext; use crate::platform_types::platform::Platform; +use crate::platform_types::snapshot::{SnapshotFetchingSession, SnapshotManager}; use crate::rpc::core::CoreRPCLike; use dpp::version::PlatformVersion; use drive::grovedb::Transaction; @@ -23,6 +27,10 @@ pub struct FullAbciApplication<'a, C> { pub transaction: RwLock>>, /// The current block execution context pub block_execution_context: RwLock>, + /// The snapshot manager, pinning checkpoints that are being served to peers + pub snapshot_manager: SnapshotManager, + /// The state sync transfer currently in progress, if any + pub snapshot_fetching_session: RwLock>>, } impl<'a, C> FullAbciApplication<'a, C> { @@ -32,6 +40,8 @@ impl<'a, C> FullAbciApplication<'a, C> { platform, transaction: Default::default(), block_execution_context: Default::default(), + snapshot_manager: SnapshotManager::new(), + snapshot_fetching_session: Default::default(), } } } @@ -42,6 +52,22 @@ impl PlatformApplication for FullAbciApplication<'_, C> { } } +impl SnapshotManagerApplication for FullAbciApplication<'_, C> { + fn snapshot_manager(&self) -> &SnapshotManager { + &self.snapshot_manager + } +} + +impl<'a, C> StateSyncApplication<'a, C> for FullAbciApplication<'a, C> { + fn snapshot_fetching_session(&self) -> &RwLock>> { + &self.snapshot_fetching_session + } + + fn platform(&self) -> &'a Platform { + self.platform + } +} + impl BlockExecutionApplication for FullAbciApplication<'_, C> { fn block_execution_context(&self) -> &RwLock> { &self.block_execution_context @@ -105,7 +131,7 @@ mod tests { crate::test::helpers::setup::TestPlatformBuilder::new().build_with_mock_rpc(); let app = FullAbciApplication::::new(&platform.platform); - let _platform_ref = app.platform(); + let _platform_ref = PlatformApplication::platform(&app); } #[test] @@ -241,4 +267,32 @@ where ) -> Result { handler::verify_vote_extension(self, request).map_err(error_into_exception) } + + fn list_snapshots( + &self, + request: proto::RequestListSnapshots, + ) -> Result { + handler::list_snapshots(self, request).map_err(error_into_exception) + } + + fn load_snapshot_chunk( + &self, + request: proto::RequestLoadSnapshotChunk, + ) -> Result { + handler::load_snapshot_chunk(self, request).map_err(error_into_exception) + } + + fn offer_snapshot( + &self, + request: proto::RequestOfferSnapshot, + ) -> Result { + handler::offer_snapshot(self, request).map_err(error_into_exception) + } + + fn apply_snapshot_chunk( + &self, + request: proto::RequestApplySnapshotChunk, + ) -> Result { + handler::apply_snapshot_chunk(self, request).map_err(error_into_exception) + } } diff --git a/packages/rs-drive-abci/src/abci/app/mod.rs b/packages/rs-drive-abci/src/abci/app/mod.rs index 27d7ef0794e..fc575f4066f 100644 --- a/packages/rs-drive-abci/src/abci/app/mod.rs +++ b/packages/rs-drive-abci/src/abci/app/mod.rs @@ -10,6 +10,7 @@ pub mod execution_result; mod full; use crate::execution::types::block_execution_context::BlockExecutionContext; +use crate::platform_types::snapshot::{SnapshotFetchingSession, SnapshotManager}; use crate::rpc::core::DefaultCoreRPC; #[cfg(test)] pub(crate) use check_tx::error_into_status; @@ -24,6 +25,23 @@ pub trait PlatformApplication { fn platform(&self) -> &Platform; } +/// ABCI application that serves state sync snapshots +pub trait SnapshotManagerApplication { + /// Returns the snapshot manager, which pins checkpoints that are actively being + /// served so pruning cannot delete them mid-transfer + fn snapshot_manager(&self) -> &SnapshotManager; +} + +/// ABCI application that can bootstrap its state via state sync +pub trait StateSyncApplication<'p, C = DefaultCoreRPC> { + /// Returns the state sync transfer currently in progress, if any + fn snapshot_fetching_session(&self) -> &RwLock>>; + + /// Returns Platform with the full `'p` lifetime, so a grovedb state sync session + /// borrowing the grove can be stored in the snapshot fetching session + fn platform(&self) -> &'p Platform; +} + /// Transactional ABCI application pub trait TransactionalApplication<'a> { /// Creates and keeps a new transaction diff --git a/packages/rs-drive-abci/src/abci/config.rs b/packages/rs-drive-abci/src/abci/config.rs index 7f80ab0e010..da8bda064ab 100644 --- a/packages/rs-drive-abci/src/abci/config.rs +++ b/packages/rs-drive-abci/src/abci/config.rs @@ -1,7 +1,8 @@ //! Configuration of ABCI Application server -use crate::utils::from_opt_str_or_number; +use crate::utils::{from_opt_str_or_number, from_str_or_native}; use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; // We allow changes in the ABCI configuration, but there should be a social process // involved in making this change. @@ -37,6 +38,78 @@ pub struct AbciConfig { /// Maximum time limit (in ms) to process state transitions to prepare proposal #[serde(default, deserialize_with = "from_opt_str_or_number")] pub proposer_tx_processing_time_limit: Option, + + /// State sync snapshot serving configuration + #[serde(flatten)] + pub state_sync: StateSyncAbciConfig, +} + +/// Configuration of ABCI state sync snapshot serving. +/// +/// NOTE: the field names (and thus the environment variable names `SNAPSHOTS_ENABLED`, +/// `SNAPSHOTS_FREQUENCY_SECONDS`, `MAX_NUM_SNAPSHOTS`, `CHECKPOINTS_PATH`) are a contract +/// with dashmate's env generation — do not rename them. +// @append_only +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct StateSyncAbciConfig { + /// Whether snapshots are offered to state-syncing peers. When enabled, the + /// snapshot frequency and retention below override the platform-version-driven + /// checkpoint parameters. + #[serde( + default = "StateSyncAbciConfig::default_snapshots_enabled", + deserialize_with = "from_str_or_native" + )] + pub snapshots_enabled: bool, + + /// How often (in seconds) a snapshot (grovedb checkpoint) is created + #[serde( + default = "StateSyncAbciConfig::default_snapshots_frequency_seconds", + deserialize_with = "from_str_or_native" + )] + pub snapshots_frequency_seconds: u32, + + /// Maximum number of snapshots kept on disk + #[serde( + default = "StateSyncAbciConfig::default_max_num_snapshots", + deserialize_with = "from_str_or_native" + )] + pub max_num_snapshots: usize, + + /// Directory where checkpoints are stored; defaults to `/checkpoints` + #[serde(default)] + pub checkpoints_path: Option, +} + +impl StateSyncAbciConfig { + pub(crate) fn default_snapshots_enabled() -> bool { + false + } + + pub(crate) fn default_snapshots_frequency_seconds() -> u32 { + 600 + } + + pub(crate) fn default_max_num_snapshots() -> usize { + 3 + } + + /// Resolves the checkpoints directory, defaulting to `/checkpoints` + pub fn resolved_checkpoints_path(&self, db_path: &Path) -> PathBuf { + self.checkpoints_path + .clone() + .unwrap_or_else(|| db_path.join("checkpoints")) + } +} + +impl Default for StateSyncAbciConfig { + fn default() -> Self { + Self { + snapshots_enabled: Self::default_snapshots_enabled(), + snapshots_frequency_seconds: Self::default_snapshots_frequency_seconds(), + max_num_snapshots: Self::default_max_num_snapshots(), + checkpoints_path: None, + } + } } impl AbciConfig { @@ -58,6 +131,7 @@ impl Default for AbciConfig { chain_id: "chain_id".to_string(), log: Default::default(), proposer_tx_processing_time_limit: Default::default(), + state_sync: Default::default(), } } } @@ -85,6 +159,42 @@ mod tests { assert_eq!(config.chain_id, "chain_id"); assert!(config.log.is_empty()); assert!(config.proposer_tx_processing_time_limit.is_none()); + assert!(!config.state_sync.snapshots_enabled); + assert_eq!(config.state_sync.snapshots_frequency_seconds, 600); + assert_eq!(config.state_sync.max_num_snapshots, 3); + assert!(config.state_sync.checkpoints_path.is_none()); + } + + #[test] + fn state_sync_config_resolves_default_checkpoints_path_from_db_path() { + let config = StateSyncAbciConfig::default(); + assert_eq!( + config.resolved_checkpoints_path(Path::new("/var/lib/drive/db")), + PathBuf::from("/var/lib/drive/db/checkpoints") + ); + + let config = StateSyncAbciConfig { + checkpoints_path: Some(PathBuf::from("/mnt/checkpoints")), + ..Default::default() + }; + assert_eq!( + config.resolved_checkpoints_path(Path::new("/var/lib/drive/db")), + PathBuf::from("/mnt/checkpoints") + ); + } + + #[test] + fn state_sync_config_deserializes_from_env_style_strings() { + // envy provides every value as a string; the custom deserializers must coerce + let json = r#"{"abci_consensus_bind_address": "tcp://x:1", "snapshots_enabled": "true", "snapshots_frequency_seconds": "120", "max_num_snapshots": "5", "checkpoints_path": "/tmp/checkpoints"}"#; + let config: AbciConfig = serde_json::from_str(json).expect("should deserialize"); + assert!(config.state_sync.snapshots_enabled); + assert_eq!(config.state_sync.snapshots_frequency_seconds, 120); + assert_eq!(config.state_sync.max_num_snapshots, 5); + assert_eq!( + config.state_sync.checkpoints_path, + Some(PathBuf::from("/tmp/checkpoints")) + ); } #[test] @@ -98,6 +208,7 @@ mod tests { chain_id: "test-chain".to_string(), log: Default::default(), proposer_tx_processing_time_limit: None, + state_sync: Default::default(), }; let serialized = serde_json::to_string(&config).expect("should serialize"); @@ -143,6 +254,7 @@ mod tests { chain_id: "clone-test".to_string(), log: Default::default(), proposer_tx_processing_time_limit: Some(1000), + state_sync: Default::default(), }; let cloned = config.clone(); diff --git a/packages/rs-drive-abci/src/abci/error.rs b/packages/rs-drive-abci/src/abci/error.rs index 306e0644956..f8168b2cbac 100644 --- a/packages/rs-drive-abci/src/abci/error.rs +++ b/packages/rs-drive-abci/src/abci/error.rs @@ -54,6 +54,14 @@ pub enum AbciError { #[error("bad commit signature: {0}")] BadCommitSignature(String), + /// Invalid state sync request received from Tenderdash or a peer + #[error("bad request state sync: {0}")] + StateSyncBadRequest(String), + + /// Internal error during state sync + #[error("internal error state sync: {0}")] + StateSyncInternalError(String), + /// The chain lock received was invalid #[error("invalid chain lock: {0}")] InvalidChainLock(String), diff --git a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs new file mode 100644 index 00000000000..8888b1307dc --- /dev/null +++ b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs @@ -0,0 +1,417 @@ +use crate::abci::app::StateSyncApplication; +use crate::abci::AbciError; +use crate::error::Error; +use crate::platform_types::platform_state::PlatformStateV0Methods; +use crate::platform_types::snapshot::{ + clear_restore_sentinel_best_effort, wipe_drive_for_restore, MAX_STATE_SYNC_CHUNK_ID_SIZE, + MAX_STATE_SYNC_CHUNK_SIZE, +}; +use crate::rpc::core::CoreRPCLike; +use tenderdash_abci::proto::abci as proto; +use tenderdash_abci::proto::abci::response_apply_snapshot_chunk; + +/// Applies one chunk of a state sync snapshot to the grovedb sync session. +/// +/// A chunk grovedb rejects does not kill the whole transfer: Tenderdash is asked to +/// refetch that chunk (from a different peer, if it identified the sender). When the +/// last chunk lands, the session is committed, grovedb is verified against the target +/// app hash, and the platform state is reconstructed from the reduced platform state +/// contained in the restored snapshot. +pub fn apply_snapshot_chunk<'a, 'db: 'a, A, C>( + app: &'a A, + request: proto::RequestApplySnapshotChunk, +) -> Result +where + A: StateSyncApplication<'db, C> + 'db, + C: CoreRPCLike + 'db, +{ + tracing::trace!( + chunk_id = hex::encode(&request.chunk_id), + chunk_len = request.chunk.len(), + "[state_sync] api apply_snapshot_chunk", + ); + + // Cap peer-supplied sizes before anything decodes them (issue #3773) + if request.chunk_id.len() > MAX_STATE_SYNC_CHUNK_ID_SIZE { + return Err(AbciError::StateSyncBadRequest(format!( + "apply_snapshot_chunk chunk id of {} bytes exceeds the {} byte limit", + request.chunk_id.len(), + MAX_STATE_SYNC_CHUNK_ID_SIZE + )) + .into()); + } + if request.chunk.len() > MAX_STATE_SYNC_CHUNK_SIZE { + return Err(AbciError::StateSyncBadRequest(format!( + "apply_snapshot_chunk chunk of {} bytes exceeds the {} byte limit", + request.chunk.len(), + MAX_STATE_SYNC_CHUNK_SIZE + )) + .into()); + } + + let platform_version = app.platform().state.load().current_platform_version()?; + let grove_version = &platform_version.drive.grove_version; + + let mut session_write_guard = app.snapshot_fetching_session().write().map_err(|_| { + AbciError::StateSyncInternalError( + "apply_snapshot_chunk unable to lock session (poisoned)".to_string(), + ) + })?; + + { + let session = session_write_guard + .as_mut() + .ok_or(AbciError::StateSyncBadRequest( + "apply_snapshot_chunk no state sync session in progress".to_string(), + ))?; + + let wire_version = session.wire_version; + let next_chunk_ids = match session.state_sync_info.apply_chunk( + &request.chunk_id, + &request.chunk, + wire_version, + grove_version, + ) { + Ok(next_chunk_ids) => next_chunk_ids, + Err(e) => { + let reject_senders = if request.sender.is_empty() { + vec![] + } else { + vec![request.sender.clone()] + }; + + // grovedb removes a chunk id from its pending set before processing it, + // so a chunk it has already seen (e.g. the refetch of one it rejected) + // cannot be re-applied within this session: ask Tenderdash to restart + // the snapshot instead (a same-height re-offer, which we accept). + // The string match is brittle by necessity (grovedb only exposes + // InternalError(String) here); if the wording ever changes, the fallback + // below is still safe — Tenderdash retries the chunk until it gives up + // and restarts the snapshot itself. + if matches!(&e, drive::grovedb::Error::InternalError(message) if message.contains("not expected")) + { + tracing::warn!( + chunk_id = hex::encode(&request.chunk_id), + sender = request.sender, + error = ?e, + "[state_sync] apply_snapshot_chunk cannot re-apply a chunk in this session, requesting snapshot restart", + ); + return Ok(proto::ResponseApplySnapshotChunk { + result: response_apply_snapshot_chunk::Result::RetrySnapshot.into(), + refetch_chunks: vec![], + reject_senders, + next_chunks: vec![], + }); + } + + // A chunk grovedb cannot apply (corrupted or tampered data) is + // recoverable: keep the session and ask Tenderdash to refetch the chunk, + // banning the peer that sent it so the refetch goes elsewhere. + tracing::warn!( + chunk_id = hex::encode(&request.chunk_id), + sender = request.sender, + error = ?e, + "[state_sync] apply_snapshot_chunk rejected a chunk, requesting refetch", + ); + return Ok(proto::ResponseApplySnapshotChunk { + result: response_apply_snapshot_chunk::Result::Retry.into(), + refetch_chunks: vec![request.chunk_id], + reject_senders, + next_chunks: vec![], + }); + } + }; + + if !session.state_sync_info.is_sync_completed() { + return Ok(proto::ResponseApplySnapshotChunk { + result: response_apply_snapshot_chunk::Result::Accept.into(), + refetch_chunks: vec![], + reject_senders: vec![], + next_chunks: next_chunk_ids, + }); + } + + if !next_chunk_ids.is_empty() { + return Err(AbciError::StateSyncInternalError( + "apply_snapshot_chunk session is completed but next_chunk_ids is not empty" + .to_string(), + ) + .into()); + } + } + + // The transfer is complete: consume the session and commit it + let session = session_write_guard + .take() + .expect("session presence was just checked"); + + // grovedb only makes the session durable once its own root-hash check passes, so a + // failure here leaves nothing committed — but the database is still WIPED from the + // offer, so the node cannot be left as it is. Route it through the same recovery path + // as every later failure, which also keeps Tenderdash's snapshot ladder moving instead + // of aborting state sync with an exception. + if let Err(e) = app + .platform() + .drive + .grove + .commit_session(session.state_sync_info, grove_version) + { + return reject_restored_snapshot(app, &format!("unable to commit the session: {}", e)); + } + + tracing::debug!("[state_sync] transfer complete, verifying grovedb"); + + // From here on the session is COMMITTED: grovedb durably holds the restored state + // while the platform state still describes the node from before the sync. Every + // failure below must therefore go through `reject_restored_snapshot`, which puts the + // node back to an empty, self-consistent slate and asks Tenderdash for another + // snapshot. Returning an error instead would leave the node holding a database its + // platform state knows nothing about, and the `info` handler panics on exactly that + // mismatch — a crash loop that no restart can clear. + let incorrect_hashes = + match app + .platform() + .drive + .grove + .verify_grovedb(None, true, false, grove_version) + { + Ok(incorrect_hashes) => incorrect_hashes, + Err(e) => { + return reject_restored_snapshot( + app, + &format!("unable to verify the restored grovedb: {}", e), + ); + } + }; + if !incorrect_hashes.is_empty() { + let paths: Vec = incorrect_hashes + .keys() + .take(5) + .map(|path| path.iter().map(hex::encode).collect::>().join("/")) + .collect(); + return reject_restored_snapshot( + app, + &format!( + "grovedb verification failed with {} incorrect hashes, first paths: [{}]", + incorrect_hashes.len(), + paths.join(", ") + ), + ); + } + + // Rebuild the in-memory platform state from the reduced platform state contained in + // the restored snapshot. This re-derives masternode lists and quorums from Core and + // must leave the grovedb root hash untouched; the equality check below proves it. + // + // This is also where a snapshot taken before the reduced platform state existed + // (pre-v15) is refused. Refusing earlier would be better, but grovedb does not expose + // the session's transaction, so the Misc tree cannot be probed before the commit — + // see the note on `reject_restored_snapshot`. + if let Err(e) = app + .platform() + .reconstruct_platform_state(&session.app_hash, platform_version) + { + return reject_restored_snapshot( + app, + &format!("unable to reconstruct the platform state: {}", e), + ); + } + + let drive_app_hash = match app + .platform() + .drive + .grove + .root_hash(None, grove_version) + .unwrap() + { + Ok(drive_app_hash) => drive_app_hash, + Err(e) => { + return reject_restored_snapshot( + app, + &format!("unable to get the restored app hash: {}", e), + ); + } + }; + + if drive_app_hash != session.app_hash { + tracing::error!( + state_sync_app_hash = hex::encode(session.app_hash), + drive_app_hash = hex::encode(drive_app_hash), + "[state_sync] restored grovedb root hash does not match the snapshot app hash", + ); + return reject_restored_snapshot( + app, + &format!( + "grovedb verification failed with incorrect app hash: {}", + hex::encode(drive_app_hash) + ), + ); + } + + // The restore is complete and the node is self-consistent again, so the marker that + // tells a restarting process to wipe can go. This is deliberately the LAST step, after + // `reconstruct_platform_state` has committed the platform state to aux storage, and + // deliberately best-effort: a successful restore must not be turned into an ABCI error + // by a `remove_file` hiccup. + clear_restore_sentinel_best_effort(&app.platform().config.db_path); + + tracing::info!( + height = session.snapshot.height, + app_hash = hex::encode(session.app_hash), + "state_sync completed", + ); + + Ok(proto::ResponseApplySnapshotChunk { + result: response_apply_snapshot_chunk::Result::CompleteSnapshot.into(), + refetch_chunks: vec![], + reject_senders: vec![], + next_chunks: vec![], + }) +} + +/// Puts the node back to an empty, self-consistent slate after a restore that was already +/// committed to grovedb turned out to be unusable, and asks Tenderdash to try a different +/// snapshot. +/// +/// Ideally an unusable snapshot would be detected BEFORE `commit_session`, by probing the +/// Misc tree through the session's still-open transaction. grovedb keeps that transaction +/// private (`MultiStateSyncSession::transaction`, no accessor), so there is no way to read +/// the restored state before it lands. Until grovedb exposes it, this is the containment: +/// undo the commit by wiping, and let Tenderdash pick another snapshot. +/// +/// The restore sentinel is deliberately LEFT IN PLACE. The database is empty, but the +/// in-memory platform state may still describe the chain the offer wiped, so the node is +/// not yet provably consistent. Everything that can happen next resolves it: another +/// `offer_snapshot` re-wipes and re-marks, a successful restore clears it, an `init_chain` +/// clears it, and a restart before any of those wipes and comes up empty. +/// +/// `REJECT_SNAPSHOT` rather than an error is what keeps Tenderdash walking its ladder: it +/// discards this snapshot, tries the next, and falls back to block sync when it runs out. +/// An ABCI exception here would abort state sync altogether. +fn reject_restored_snapshot<'a, 'db: 'a, A, C>( + app: &'a A, + reason: &str, +) -> Result +where + A: StateSyncApplication<'db, C> + 'db, + C: CoreRPCLike + 'db, +{ + tracing::error!( + reason, + "[state_sync] restored snapshot is unusable, wiping and asking for another one", + ); + + wipe_drive_for_restore(&app.platform().drive).map_err(|e| { + AbciError::StateSyncInternalError(format!( + "apply_snapshot_chunk unable to wipe after rejecting a snapshot ({}): {}", + reason, e + )) + })?; + + Ok(proto::ResponseApplySnapshotChunk { + result: response_apply_snapshot_chunk::Result::RejectSnapshot.into(), + refetch_chunks: vec![], + reject_senders: vec![], + next_chunks: vec![], + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::abci::app::FullAbciApplication; + use crate::abci::handler::offer_snapshot; + use crate::test::helpers::setup::TestPlatformBuilder; + + #[test] + fn apply_snapshot_chunk_without_session_is_rejected() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let app = FullAbciApplication::new(&platform); + + assert!(apply_snapshot_chunk( + &app, + proto::RequestApplySnapshotChunk { + chunk_id: vec![1u8; 32], + chunk: vec![], + sender: String::new(), + }, + ) + .is_err()); + } + + #[test] + fn apply_snapshot_chunk_caps_sizes_before_decoding() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let app = FullAbciApplication::new(&platform); + + assert!(apply_snapshot_chunk( + &app, + proto::RequestApplySnapshotChunk { + chunk_id: vec![0u8; MAX_STATE_SYNC_CHUNK_ID_SIZE + 1], + chunk: vec![], + sender: String::new(), + }, + ) + .is_err()); + + assert!(apply_snapshot_chunk( + &app, + proto::RequestApplySnapshotChunk { + chunk_id: vec![1u8; 32], + chunk: vec![0u8; MAX_STATE_SYNC_CHUNK_SIZE + 1], + sender: String::new(), + }, + ) + .is_err()); + } + + #[test] + fn apply_snapshot_chunk_asks_for_refetch_of_a_bad_chunk() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let app = FullAbciApplication::new(&platform); + + let target_app_hash = vec![7u8; 32]; + offer_snapshot( + &app, + proto::RequestOfferSnapshot { + snapshot: Some(proto::Snapshot { + height: 100, + version: 1, + hash: target_app_hash.clone(), + metadata: vec![], + }), + app_hash: target_app_hash.clone(), + }, + ) + .expect("should accept offer"); + + // Garbage bytes for the root chunk: grovedb rejects them, and the session must + // survive with a Retry + refetch of exactly that chunk, banning the sender. + let response = apply_snapshot_chunk( + &app, + proto::RequestApplySnapshotChunk { + chunk_id: target_app_hash.clone(), + chunk: vec![0xde, 0xad, 0xbe, 0xef], + sender: "peer-1".to_string(), + }, + ) + .expect("bad chunk should not error the session"); + + assert_eq!( + response.result, + i32::from(response_apply_snapshot_chunk::Result::Retry) + ); + assert_eq!(response.refetch_chunks, vec![target_app_hash]); + assert_eq!(response.reject_senders, vec!["peer-1".to_string()]); + assert!( + app.snapshot_fetching_session.read().unwrap().is_some(), + "the session must survive a bad chunk" + ); + } +} diff --git a/packages/rs-drive-abci/src/abci/handler/init_chain.rs b/packages/rs-drive-abci/src/abci/handler/init_chain.rs index 0573b05e6b3..5923aefd11f 100644 --- a/packages/rs-drive-abci/src/abci/handler/init_chain.rs +++ b/packages/rs-drive-abci/src/abci/handler/init_chain.rs @@ -1,5 +1,6 @@ use crate::abci::app::{BlockExecutionApplication, PlatformApplication, TransactionalApplication}; use crate::error::Error; +use crate::platform_types::snapshot::clear_restore_sentinel_best_effort; use crate::rpc::core::CoreRPCLike; use tenderdash_abci::proto::abci as proto; @@ -32,6 +33,16 @@ where let app_hash = hex::encode(&response.app_hash); + // Genesis has just been created, so the node is self-consistent again. If a state sync + // restore had been abandoned (every offered snapshot rejected, Tenderdash falling back + // to block sync), its marker is still on disk and would make the NEXT restart wipe this + // perfectly good chain. Clear it here — this is the block-sync arm of the same recovery + // that `Platform::open_with_client` performs for an interrupted restore. + // Best-effort: failing to remove a marker file must not turn a working genesis into a + // failed init_chain. The worst case is one unnecessary wipe-and-resync on a later + // restart, which is loud but safe. + clear_restore_sentinel_best_effort(&app.platform().config.db_path); + tracing::info!( app_hash, chain_id, diff --git a/packages/rs-drive-abci/src/abci/handler/list_snapshots.rs b/packages/rs-drive-abci/src/abci/handler/list_snapshots.rs new file mode 100644 index 00000000000..0b2752685e1 --- /dev/null +++ b/packages/rs-drive-abci/src/abci/handler/list_snapshots.rs @@ -0,0 +1,142 @@ +use crate::abci::app::PlatformApplication; +use crate::abci::AbciError; +use crate::error::Error; +use crate::platform_types::platform_state::PlatformStateV0Methods; +use crate::rpc::core::CoreRPCLike; +use tenderdash_abci::proto::abci as proto; + +/// Lists the state sync snapshots this node can serve. +/// +/// Snapshots are the rocksdb checkpoints Drive already keeps (`drive.checkpoints`). +/// Only checkpoints that contain the reduced platform state are offered: a checkpoint +/// taken before the protocol version that introduced it (v15) cannot be restored, since +/// a state-synced node would have no way to reconstruct its platform state. +pub fn list_snapshots( + app: &A, + _request: proto::RequestListSnapshots, +) -> Result +where + A: PlatformApplication, + C: CoreRPCLike, +{ + tracing::trace!("[state_sync] api list_snapshots called"); + + if !app.platform().config.abci.state_sync.snapshots_enabled { + return Ok(Default::default()); + } + + let platform_state = app.platform().state.load(); + let platform_version = platform_state.current_platform_version()?; + let grove_version = &platform_version.drive.grove_version; + + let checkpoints = app.platform().drive.checkpoints.load(); + + let mut snapshots = Vec::new(); + for (height, checkpoint_info) in checkpoints.iter() { + let checkpoint = &checkpoint_info.checkpoint; + + let restorable = checkpoint + .has_reduced_platform_state(grove_version) + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "list_snapshots unable to inspect checkpoint at height {}: {}", + height, e + )) + })?; + if !restorable { + continue; + } + + let root_hash = checkpoint + .grove_db + .root_hash(None, grove_version) + .unwrap() + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "list_snapshots unable to get root hash of checkpoint at height {}: {}", + height, e + )) + })?; + + snapshots.push(proto::Snapshot { + height: *height, + version: platform_version.drive_abci.state_sync.protocol_version as u32, + hash: root_hash.to_vec(), + metadata: Vec::new(), + }); + } + + Ok(proto::ResponseListSnapshots { snapshots }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::abci::app::FullAbciApplication; + use crate::config::PlatformConfig; + use crate::test::helpers::fast_forward_to_block::fast_forward_to_block; + use crate::test::helpers::setup::TestPlatformBuilder; + use dpp::version::PlatformVersion; + + fn config_with_snapshots_enabled() -> PlatformConfig { + let mut config = PlatformConfig::default_local(); + config.abci.state_sync.snapshots_enabled = true; + config + } + + #[test] + fn list_snapshots_returns_nothing_when_serving_is_disabled() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let app = FullAbciApplication::new(&platform); + + let response = list_snapshots(&app, Default::default()).expect("should list snapshots"); + assert!(response.snapshots.is_empty()); + } + + #[test] + fn list_snapshots_serves_only_checkpoints_with_reduced_platform_state() { + let platform = TestPlatformBuilder::new() + .with_config(config_with_snapshots_enabled()) + .build_with_mock_rpc() + .set_genesis_state(); + let platform_version = PlatformVersion::latest(); + let app = FullAbciApplication::new(&platform); + + // A checkpoint taken before the reduced platform state exists (pre-v15 + // activation) is unrestorable and must not be offered. + fast_forward_to_block(&platform, 1_000_000, 10, 42, 0, false); + platform + .create_grovedb_checkpoint(platform_version) + .expect("should create checkpoint"); + + let response = list_snapshots(&app, Default::default()).expect("should list snapshots"); + assert!( + response.snapshots.is_empty(), + "checkpoints without the reduced platform state must be filtered out" + ); + + // Once the reduced platform state is in the replicated state, new checkpoints + // are restorable and must be offered. + let reduced_platform_state = platform.state.load().to_reduced_platform_state(None, 42); + platform + .store_reduced_platform_state(&reduced_platform_state, None, platform_version) + .expect("should store reduced platform state"); + + fast_forward_to_block(&platform, 2_000_000, 20, 43, 0, false); + platform + .create_grovedb_checkpoint(platform_version) + .expect("should create checkpoint"); + + let response = list_snapshots(&app, Default::default()).expect("should list snapshots"); + assert_eq!(response.snapshots.len(), 1); + let snapshot = &response.snapshots[0]; + assert_eq!(snapshot.height, 20); + assert_eq!( + snapshot.version, + platform_version.drive_abci.state_sync.protocol_version as u32 + ); + assert_eq!(snapshot.hash.len(), 32); + } +} diff --git a/packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs b/packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs new file mode 100644 index 00000000000..ed6c1652c3a --- /dev/null +++ b/packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs @@ -0,0 +1,181 @@ +use crate::abci::app::{PlatformApplication, SnapshotManagerApplication}; +use crate::abci::AbciError; +use crate::error::Error; +use crate::platform_types::platform_state::PlatformStateV0Methods; +use crate::platform_types::snapshot::{ + MAX_STATE_SYNC_CHUNK_ID_SIZE, SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS, +}; +use crate::rpc::core::CoreRPCLike; +use std::sync::Arc; +use tenderdash_abci::proto::abci as proto; + +/// Serves one chunk of a state sync snapshot from the checkpoint registry. +/// +/// The served checkpoint is pinned in the snapshot manager so checkpoint pruning cannot +/// delete it from disk while a peer is still downloading it. +pub fn load_snapshot_chunk( + app: &A, + request: proto::RequestLoadSnapshotChunk, +) -> Result +where + A: PlatformApplication + SnapshotManagerApplication, + C: CoreRPCLike, +{ + tracing::trace!( + height = request.height, + version = request.version, + chunk_id = hex::encode(&request.chunk_id), + "[state_sync] api load_snapshot_chunk", + ); + + if !app.platform().config.abci.state_sync.snapshots_enabled { + return Err(AbciError::StateSyncBadRequest( + "load_snapshot_chunk snapshot serving is disabled".to_string(), + ) + .into()); + } + + // Cap peer-supplied sizes before anything decodes them (issue #3773) + if request.chunk_id.len() > MAX_STATE_SYNC_CHUNK_ID_SIZE { + return Err(AbciError::StateSyncBadRequest(format!( + "load_snapshot_chunk chunk id of {} bytes exceeds the {} byte limit", + request.chunk_id.len(), + MAX_STATE_SYNC_CHUNK_ID_SIZE + )) + .into()); + } + + let wire_version = u16::try_from(request.version) + .ok() + .filter(|version| SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS.contains(version)); + let Some(wire_version) = wire_version else { + return Err(AbciError::StateSyncBadRequest(format!( + "load_snapshot_chunk unsupported state sync protocol version {}, supported: {:?}", + request.version, SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS + )) + .into()); + }; + + let platform_state = app.platform().state.load(); + let platform_version = platform_state.current_platform_version()?; + let grove_version = &platform_version.drive.grove_version; + + // Resolve the checkpoint: from the registry, or — if pruning already dropped it — + // from the pins of transfers already in flight. + let checkpoint = app + .platform() + .drive + .checkpoints + .load() + .get(&request.height) + .map(|checkpoint_info| Arc::clone(&checkpoint_info.checkpoint)) + .or_else(|| app.snapshot_manager().pinned_checkpoint(request.height)) + .ok_or_else(|| { + AbciError::StateSyncBadRequest(format!( + "load_snapshot_chunk no snapshot at height {}", + request.height + )) + })?; + + // Pin (or refresh the pin of) the checkpoint for the duration of the transfer + app.snapshot_manager() + .pin_for_serving(request.height, Arc::clone(&checkpoint)); + + let chunk = checkpoint + .grove_db + .fetch_chunk(&request.chunk_id, None, wire_version, grove_version) + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "load_snapshot_chunk unable to fetch chunk: {}", + e + )) + })?; + + Ok(proto::ResponseLoadSnapshotChunk { chunk }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::abci::app::FullAbciApplication; + use crate::config::PlatformConfig; + use crate::test::helpers::fast_forward_to_block::fast_forward_to_block; + use crate::test::helpers::setup::TestPlatformBuilder; + use dpp::version::PlatformVersion; + + #[test] + fn load_snapshot_chunk_serves_root_chunk_and_rejects_bad_requests() { + let mut config = PlatformConfig::default_local(); + config.abci.state_sync.snapshots_enabled = true; + let platform = TestPlatformBuilder::new() + .with_config(config) + .build_with_mock_rpc() + .set_genesis_state(); + let platform_version = PlatformVersion::latest(); + let app = FullAbciApplication::new(&platform); + + let reduced_platform_state = platform.state.load().to_reduced_platform_state(None, 42); + platform + .store_reduced_platform_state(&reduced_platform_state, None, platform_version) + .expect("should store reduced platform state"); + fast_forward_to_block(&platform, 1_000_000, 10, 42, 0, false); + platform + .create_grovedb_checkpoint(platform_version) + .expect("should create checkpoint"); + + let root_hash = platform + .drive + .grove + .root_hash(None, &platform_version.drive.grove_version) + .unwrap() + .expect("should get root hash"); + + // The root chunk (chunk id == app hash) must be served + let response = load_snapshot_chunk( + &app, + proto::RequestLoadSnapshotChunk { + height: 10, + version: 1, + chunk_id: root_hash.to_vec(), + }, + ) + .expect("should load root chunk"); + assert!(!response.chunk.is_empty()); + + // The served checkpoint must now be pinned against pruning + assert!(app.snapshot_manager.pinned_checkpoint(10).is_some()); + + // Unknown height is rejected + assert!(load_snapshot_chunk( + &app, + proto::RequestLoadSnapshotChunk { + height: 999, + version: 1, + chunk_id: root_hash.to_vec(), + }, + ) + .is_err()); + + // Unsupported wire version is rejected + assert!(load_snapshot_chunk( + &app, + proto::RequestLoadSnapshotChunk { + height: 10, + version: 2, + chunk_id: root_hash.to_vec(), + }, + ) + .is_err()); + + // Oversized chunk id is rejected before any decoding + assert!(load_snapshot_chunk( + &app, + proto::RequestLoadSnapshotChunk { + height: 10, + version: 1, + chunk_id: vec![0u8; MAX_STATE_SYNC_CHUNK_ID_SIZE + 1], + }, + ) + .is_err()); + } +} diff --git a/packages/rs-drive-abci/src/abci/handler/mod.rs b/packages/rs-drive-abci/src/abci/handler/mod.rs index 8acd0737ebe..443758734a3 100644 --- a/packages/rs-drive-abci/src/abci/handler/mod.rs +++ b/packages/rs-drive-abci/src/abci/handler/mod.rs @@ -35,6 +35,7 @@ //! can only make changes that are backwards compatible. Otherwise new calls must be made instead. //! +mod apply_snapshot_chunk; mod check_tx; mod echo; pub mod error; @@ -42,16 +43,23 @@ mod extend_vote; mod finalize_block; mod info; mod init_chain; +mod list_snapshots; +mod load_snapshot_chunk; +mod offer_snapshot; mod prepare_proposal; mod process_proposal; mod verify_vote_extension; +pub use apply_snapshot_chunk::apply_snapshot_chunk; pub use check_tx::check_tx; pub use echo::echo; pub use extend_vote::extend_vote; pub use finalize_block::finalize_block; pub use info::info; pub use init_chain::init_chain; +pub use list_snapshots::list_snapshots; +pub use load_snapshot_chunk::load_snapshot_chunk; +pub use offer_snapshot::offer_snapshot; pub use prepare_proposal::prepare_proposal; pub use process_proposal::process_proposal; pub use verify_vote_extension::verify_vote_extension; diff --git a/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs b/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs new file mode 100644 index 00000000000..5231371d6f0 --- /dev/null +++ b/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs @@ -0,0 +1,206 @@ +use crate::abci::app::StateSyncApplication; +use crate::abci::AbciError; +use crate::error::Error; +use crate::platform_types::platform_state::PlatformStateV0Methods; +use crate::platform_types::snapshot::{ + wipe_drive_for_restore, write_restore_sentinel, SnapshotFetchingSession, + STATE_SYNC_SUBTREES_BATCH_SIZE, SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS, +}; +use crate::rpc::core::CoreRPCLike; +use tenderdash_abci::proto::abci as proto; +use tenderdash_abci::proto::abci::response_offer_snapshot; + +/// Handles a snapshot offered by Tenderdash during state sync. +/// +/// Accepting an offer wipes the local grovedb and opens a grovedb state sync session +/// targeting the light-client-verified app hash. A later offer for a higher height +/// replaces a session already in progress (also answered with Accept); an offer for a +/// lower or equal height than the session in progress is rejected. +pub fn offer_snapshot<'a, 'db: 'a, A, C: 'db>( + app: &'a A, + request: proto::RequestOfferSnapshot, +) -> Result +where + A: StateSyncApplication<'db, C> + 'db, + C: CoreRPCLike, +{ + let request_app_hash: [u8; 32] = request.app_hash.try_into().map_err(|_| { + AbciError::StateSyncBadRequest("offer_snapshot invalid app_hash length".to_string()) + })?; + let offered_snapshot = request.snapshot.ok_or(AbciError::StateSyncBadRequest( + "offer_snapshot empty snapshot in request".to_string(), + ))?; + + tracing::debug!( + height = offered_snapshot.height, + version = offered_snapshot.version, + "[state_sync] api offer_snapshot", + ); + + // The grovedb wire version of the whole transfer is the OFFERED snapshot's version, + // validated against the single supported set. Unsupported versions ask Tenderdash to + // reject every snapshot of this format and try others. + let wire_version = u16::try_from(offered_snapshot.version) + .ok() + .filter(|version| SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS.contains(version)); + let Some(wire_version) = wire_version else { + tracing::warn!( + height = offered_snapshot.height, + version = offered_snapshot.version, + supported = ?SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS, + "[state_sync] offer_snapshot rejecting unsupported snapshot version", + ); + return Ok(proto::ResponseOfferSnapshot { + result: response_offer_snapshot::Result::RejectFormat.into(), + }); + }; + + let platform_version = app.platform().state.load().current_platform_version()?; + + let mut session_write_guard = app.snapshot_fetching_session().write().map_err(|_| { + AbciError::StateSyncInternalError( + "offer_snapshot unable to lock session (poisoned)".to_string(), + ) + })?; + + if let Some(session) = session_write_guard.as_ref() { + // An offer at the same height is a legitimate snapshot restart (Tenderdash's + // RETRY_SNAPSHOT flow) and replaces the session; only strictly older offers are + // rejected. + if offered_snapshot.height < session.snapshot.height { + return Err(AbciError::StateSyncBadRequest(format!( + "offer_snapshot already syncing snapshot at height {}, offered height {} is older", + session.snapshot.height, offered_snapshot.height + )) + .into()); + } + tracing::warn!( + current_height = session.snapshot.height, + offered_height = offered_snapshot.height, + "[state_sync] offer_snapshot replacing session in progress", + ); + } + + // Mark the database as under restore BEFORE destroying it. From here until the + // restore completes, the node may be in a state that cannot serve consensus, and the + // only thing that can tell a restarted process so is this marker: without it, startup + // finds a database that disagrees with its platform state and cannot distinguish an + // interrupted restore from corruption. See `Platform::open_with_client`. + write_restore_sentinel( + &app.platform().config.db_path, + &request_app_hash, + offered_snapshot.height, + ) + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "offer_snapshot unable to record the restore sentinel: {}", + e + )) + })?; + + // Both the fresh-session and the replace-session paths wipe grovedb (dropping the + // caches derived from it), start a new grovedb sync session, and answer Accept. + wipe_drive_for_restore(&app.platform().drive).map_err(|e| { + AbciError::StateSyncInternalError(format!("offer_snapshot unable to wipe grovedb: {}", e)) + })?; + + let state_sync_info = app + .platform() + .drive + .grove + .start_snapshot_syncing( + request_app_hash, + STATE_SYNC_SUBTREES_BATCH_SIZE, + wire_version, + &platform_version.drive.grove_version, + ) + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "offer_snapshot unable to start snapshot syncing session: {}", + e + )) + })?; + + *session_write_guard = Some(SnapshotFetchingSession { + snapshot: offered_snapshot, + app_hash: request_app_hash, + wire_version, + state_sync_info, + }); + + Ok(proto::ResponseOfferSnapshot { + result: response_offer_snapshot::Result::Accept.into(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::abci::app::FullAbciApplication; + use crate::test::helpers::setup::TestPlatformBuilder; + + fn offer_at(height: u64, version: u32) -> proto::RequestOfferSnapshot { + proto::RequestOfferSnapshot { + snapshot: Some(proto::Snapshot { + height, + version, + hash: vec![7u8; 32], + metadata: vec![], + }), + app_hash: vec![7u8; 32], + } + } + + #[test] + fn offer_snapshot_rejects_unsupported_version_with_reject_format() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let app = FullAbciApplication::new(&platform); + + let response = offer_snapshot(&app, offer_at(100, 999)).expect("should not error"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::RejectFormat) + ); + assert!(app.snapshot_fetching_session.read().unwrap().is_none()); + } + + #[test] + fn offer_snapshot_accepts_fresh_and_replacing_offers() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let app = FullAbciApplication::new(&platform); + + // Fresh session is accepted + let response = offer_snapshot(&app, offer_at(100, 1)).expect("should accept fresh offer"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::Accept) + ); + + // A strictly lower height while syncing is rejected + assert!(offer_snapshot(&app, offer_at(50, 1)).is_err()); + + // A same-height re-offer is a snapshot restart (Tenderdash RETRY_SNAPSHOT): + // the session is replaced and the offer accepted + let response = offer_snapshot(&app, offer_at(100, 1)).expect("should accept restart"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::Accept) + ); + + // A newer snapshot replaces the session and MUST also answer Accept + // (the old prototype returned the default UNKNOWN result here) + let response = offer_snapshot(&app, offer_at(200, 1)).expect("should accept newer offer"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::Accept) + ); + let session_guard = app.snapshot_fetching_session.read().unwrap(); + let session = session_guard.as_ref().expect("session must exist"); + assert_eq!(session.snapshot.height, 200); + assert_eq!(session.wire_version, 1); + } +} diff --git a/packages/rs-drive-abci/src/execution/engine/consensus_params_update/mod.rs b/packages/rs-drive-abci/src/execution/engine/consensus_params_update/mod.rs index e86a9ddb3eb..e7f9df5b222 100644 --- a/packages/rs-drive-abci/src/execution/engine/consensus_params_update/mod.rs +++ b/packages/rs-drive-abci/src/execution/engine/consensus_params_update/mod.rs @@ -8,6 +8,7 @@ use tenderdash_abci::proto::types::ConsensusParams; mod v0; mod v1; +mod v2; pub(crate) fn consensus_params_update( network: Network, @@ -33,9 +34,15 @@ pub(crate) fn consensus_params_update( new_platform_version, epoch_info, )), + 2 => Ok(v2::consensus_params_update_v2( + network, + original_platform_version, + new_platform_version, + epoch_info, + )), version => Err(Error::Execution(ExecutionError::UnknownVersionMismatch { method: "consensus_params_update".to_string(), - known_versions: vec![0, 1], + known_versions: vec![0, 1, 2], received: version, })), } @@ -143,7 +150,7 @@ mod tests { received, })) => { assert_eq!(method, "consensus_params_update"); - assert_eq!(known_versions, vec![0, 1]); + assert_eq!(known_versions, vec![0, 1, 2]); assert_eq!(received, 99); } other => panic!("expected UnknownVersionMismatch error, got: {:?}", other), @@ -587,4 +594,72 @@ mod tests { assert!(result.is_none()); } } + + mod v2_evidence_params { + use super::*; + + /// Crossing to v15 (whose method table selects consensus_params_update v2) must + /// emit both the new app version and the evidence params from issue #2512. + #[test] + fn crossing_to_v15_emits_evidence_params() { + let platform_v14 = PlatformVersion::get(14).expect("v14 exists"); + let platform_v15 = PlatformVersion::get(15).expect("v15 exists"); + let epoch_info = epoch_change_to(10); + + let params = + consensus_params_update(Network::Devnet, platform_v14, platform_v15, &epoch_info) + .expect("should not error") + .expect("crossing to v15 must emit consensus params"); + + let version = params.version.expect("version params must be set"); + assert_eq!(version.app_version, 15); + + let evidence = params.evidence.expect("evidence params must be set"); + assert_eq!(evidence.max_age_num_blocks, 15_000); + assert_eq!( + evidence + .max_age_duration + .expect("max age duration must be set") + .seconds, + 20 * 24 * 60 * 60 + ); + assert_eq!(evidence.max_bytes, 1_048_576); + } + + /// Once the network is on v15, a block without a version change emits nothing: + /// the evidence params are a one-shot emission on the activation block. + #[test] + fn steady_state_v15_emits_nothing() { + let platform_v15 = PlatformVersion::get(15).expect("v15 exists"); + let epoch_info = mid_epoch(11); + + let result = + consensus_params_update(Network::Devnet, platform_v15, platform_v15, &epoch_info) + .expect("should not error"); + assert!(result.is_none()); + } + + /// A version change that does not cross the v15 boundary must not attach + /// evidence params even when dispatched through v2. + #[test] + fn non_crossing_version_change_has_no_evidence_params() { + let platform_v13 = PlatformVersion::get(13).expect("v13 exists"); + let platform_v14 = PlatformVersion::get(14).expect("v14 exists"); + let epoch_info = epoch_change_to(9); + + let params = v2::consensus_params_update_v2( + Network::Devnet, + platform_v13, + platform_v14, + &epoch_info, + ) + .expect("version change must emit consensus params"); + + assert!(params.version.is_some()); + assert!( + params.evidence.is_none(), + "evidence params are only for the v15 crossing" + ); + } + } } diff --git a/packages/rs-drive-abci/src/execution/engine/consensus_params_update/v2/mod.rs b/packages/rs-drive-abci/src/execution/engine/consensus_params_update/v2/mod.rs new file mode 100644 index 00000000000..8f1c4663a04 --- /dev/null +++ b/packages/rs-drive-abci/src/execution/engine/consensus_params_update/v2/mod.rs @@ -0,0 +1,60 @@ +use crate::execution::engine::consensus_params_update::v1::consensus_params_update_v1; +use crate::platform_types::epoch_info::EpochInfo; +use dpp::dashcore::Network; +use dpp::version::v15::PROTOCOL_VERSION_15; +use dpp::version::PlatformVersion; +use tenderdash_abci::proto::google::protobuf::Duration; +use tenderdash_abci::proto::types::{ConsensusParams, EvidenceParams}; + +/// Maximum evidence age in blocks, applied when the network crosses to protocol +/// version 15 (state sync). Value proposed in issue #2512 for nodes that bootstrap +/// from snapshots and do not hold full history. +/// +/// REVIEW BEFORE RELEASE: at ~6s blocks, 15_000 blocks is roughly one day, while +/// [`V15_EVIDENCE_MAX_AGE_DURATION_SECONDS`] below is 20 days. Evidence expires when +/// EITHER bound is exceeded, so the effective window is the smaller (~1 day) — the two +/// values from #2512 look inconsistent and need to be confirmed before this ships. +const V15_EVIDENCE_MAX_AGE_NUM_BLOCKS: i64 = 15_000; + +/// Maximum evidence age in time: 20 days, per issue #2512. See the review note on +/// [`V15_EVIDENCE_MAX_AGE_NUM_BLOCKS`]. +const V15_EVIDENCE_MAX_AGE_DURATION_SECONDS: i64 = 20 * 24 * 60 * 60; + +/// Maximum total evidence per block in bytes. Tenderdash's default (1 MiB); #2512 does +/// not change it, but the whole evidence section must be populated when it is emitted. +const V15_EVIDENCE_MAX_BYTES: i64 = 1_048_576; + +/// Same as v1, but the first block of protocol version 15 additionally emits evidence +/// params sized for a network whose nodes may have bootstrapped via state sync +/// (issue #2512). +#[inline(always)] +pub(super) fn consensus_params_update_v2( + network: Network, + original_platform_version: &PlatformVersion, + new_platform_version: &PlatformVersion, + epoch_info: &EpochInfo, +) -> Option { + let mut consensus_params = consensus_params_update_v1( + network, + original_platform_version, + new_platform_version, + epoch_info, + )?; + + // Crossing to v15 implies a protocol version change, so v1 always emits params on + // the activation block and we only need to attach the evidence section. + let is_crossing_to_v15 = original_platform_version.protocol_version < PROTOCOL_VERSION_15 + && new_platform_version.protocol_version >= PROTOCOL_VERSION_15; + if is_crossing_to_v15 { + consensus_params.evidence = Some(EvidenceParams { + max_age_num_blocks: V15_EVIDENCE_MAX_AGE_NUM_BLOCKS, + max_age_duration: Some(Duration { + seconds: V15_EVIDENCE_MAX_AGE_DURATION_SECONDS, + nanos: 0, + }), + max_bytes: V15_EVIDENCE_MAX_BYTES, + }); + } + + Some(consensus_params) +} diff --git a/packages/rs-drive-abci/src/execution/engine/initialization/init_chain/v0/mod.rs b/packages/rs-drive-abci/src/execution/engine/initialization/init_chain/v0/mod.rs index e34fee97d36..fd74c21372d 100644 --- a/packages/rs-drive-abci/src/execution/engine/initialization/init_chain/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/engine/initialization/init_chain/v0/mod.rs @@ -144,13 +144,28 @@ where self.config.execution.epoch_time_length_s, )?); + let mut consensus_params = consensus_params_update( + self.config.network, + first_platform_version, + platform_version, + &epoch_info, + )?; + + // `first_platform_version` above is a fiction for Tenderdash's benefit — it + // makes the update carry the real app version, since Tenderdash starts genesis + // assuming the first one. But it also makes a chain that STARTS on protocol + // v15+ look like it just crossed to v15, and the update then carries the + // evidence window override meant for chains upgrading with pre-state-sync + // genesis documents (#2512) — silently clobbering the evidence params of the + // genesis document being initialized right now. At genesis the operator's + // genesis document is authoritative, so the evidence section must not be + // emitted here; a `None` section leaves the genesis values in force. + if let Some(params) = consensus_params.as_mut() { + params.evidence = None; + } + Ok(ResponseInitChain { - consensus_params: consensus_params_update( - self.config.network, - first_platform_version, - platform_version, - &epoch_info, - )?, + consensus_params, app_hash: app_hash.to_vec(), validator_set_update: Some(validator_set), next_core_chain_lock_update: None, diff --git a/packages/rs-drive-abci/src/execution/engine/run_block_proposal/mod.rs b/packages/rs-drive-abci/src/execution/engine/run_block_proposal/mod.rs index 68d87275ace..f0063a452f9 100644 --- a/packages/rs-drive-abci/src/execution/engine/run_block_proposal/mod.rs +++ b/packages/rs-drive-abci/src/execution/engine/run_block_proposal/mod.rs @@ -15,6 +15,7 @@ use dpp::version::PlatformVersion; use drive::grovedb::Transaction; mod v0; +mod v1; impl Platform where @@ -154,9 +155,19 @@ Your software version: {}, latest supported protocol version: {}."#, block_platform_version, timer, ), + 1 => self.run_block_proposal_v1( + block_proposal, + known_from_us, + epoch_info, + transaction, + platform_state, + block_platform_state, + block_platform_version, + timer, + ), version => Err(Error::Execution(ExecutionError::UnknownVersionMismatch { method: "run_block_proposal".to_string(), - known_versions: vec![0], + known_versions: vec![0, 1], received: version, })), } diff --git a/packages/rs-drive-abci/src/execution/engine/run_block_proposal/v1/mod.rs b/packages/rs-drive-abci/src/execution/engine/run_block_proposal/v1/mod.rs new file mode 100644 index 00000000000..cdaf3180ae3 --- /dev/null +++ b/packages/rs-drive-abci/src/execution/engine/run_block_proposal/v1/mod.rs @@ -0,0 +1,510 @@ +use dpp::block::epoch::Epoch; + +use dpp::validation::ValidationResult; + +use dpp::version::PlatformVersion; +use drive::grovedb::Transaction; + +use crate::abci::AbciError; +use crate::error::execution::ExecutionError; + +use crate::error::Error; +use crate::execution::types::block_execution_context::v0::{ + BlockExecutionContextV0Getters, BlockExecutionContextV0MutableGetters, +}; +use crate::execution::types::block_execution_context::BlockExecutionContext; +use crate::execution::types::block_fees::v0::BlockFeesV0; +use crate::execution::types::block_state_info::v0::{ + BlockStateInfoV0Getters, BlockStateInfoV0Methods, BlockStateInfoV0Setters, +}; +use crate::execution::types::{block_execution_context, block_state_info}; +use crate::metrics::HistogramTiming; +use crate::platform_types::block_execution_outcome; +use crate::platform_types::block_proposal; +use crate::platform_types::epoch_info::v0::{EpochInfoV0Getters, EpochInfoV0Methods}; +use crate::platform_types::epoch_info::EpochInfo; +use crate::platform_types::platform::Platform; +use crate::platform_types::platform_state::PlatformState; +use crate::platform_types::platform_state::PlatformStateV0Methods; +use crate::platform_types::verify_chain_lock_result::v0::VerifyChainLockResult; +use crate::rpc::core::CoreRPCLike; +use dpp::reduced_platform_state::v0::ReducedBlockInfoV0; + +impl Platform +where + C: CoreRPCLike, +{ + /// Runs a block proposal, either from process proposal or prepare proposal. + /// + /// This function takes a `BlockProposal` and a `Transaction` as input and processes the block + /// proposal. It first validates the block proposal and then processes raw state transitions, + /// withdrawal transactions, and block fees. It also updates the validator set. + /// + /// v1 (protocol v15, state sync): identical to v0 except that + /// `validator_set_update` runs BEFORE the root hash is computed (it only mutates the + /// in-memory block platform state, never grovedb, so the move cannot change the root + /// hash or the rotation outcome), and the reduced platform state — including the + /// post-rotation next validator set — is then written into the replicated grovedb + /// state immediately before the root hash, so it is covered by this block's app hash + /// and a state-synced node can reconstruct the full platform state from it. + /// + /// # Arguments + /// + /// * `block_proposal` - The block proposal to be processed. + /// * `known_from_us` - Do we know that we made this block proposal?. + /// * `transaction` - The transaction associated with the block proposal. + /// + /// # Returns + /// + /// * `Result, Error>` - If the block proposal is + /// successfully processed, it returns a `ValidationResult` containing the `BlockExecutionOutcome`. + /// If the block proposal processing fails, it returns an `Error`. Consensus errors are returned + /// in the `ValidationResult`, while critical system errors are returned in the `Result`. + /// + /// # Errors + /// + /// This function may return an `Error` variant if there is a problem with processing the block + /// proposal, updating the core info, processing raw state transitions, or processing block fees. + /// + #[allow(clippy::too_many_arguments)] + pub(super) fn run_block_proposal_v1( + &self, + block_proposal: block_proposal::v0::BlockProposal, + known_from_us: bool, + epoch_info: EpochInfo, + transaction: &Transaction, + last_committed_platform_state: &PlatformState, + mut block_platform_state: PlatformState, + platform_version: &'static PlatformVersion, + timer: Option<&HistogramTiming>, + ) -> Result, Error> + { + tracing::trace!( + method = "run_block_proposal_v1", + ?block_proposal, + ?epoch_info, + "Running a block proposal for height: {}, round: {}", + block_proposal.height, + block_proposal.round, + ); + + // Run block proposal determines version by itself based on the previous + // state and block time. + // It should provide correct version on prepare proposal to block header + // and validate it on process proposal. + // If version set to 0 (default number value) it means we are on prepare proposal, + // so there is no need for validation. + if !known_from_us + && block_proposal.consensus_versions.app != platform_version.protocol_version as u64 + { + return Ok(ValidationResult::new_with_error( + AbciError::BadRequest(format!( + "received a block proposal with protocol version {}, expected: {}", + block_proposal.consensus_versions.app, platform_version.protocol_version + )) + .into(), + )); + } + + let last_block_time_ms = last_committed_platform_state.last_committed_block_time_ms(); + let last_block_height = last_committed_platform_state.last_committed_known_block_height_or( + self.config.abci.genesis_height.saturating_sub(1), + ); + let last_block_core_height = last_committed_platform_state + .last_committed_known_core_height_or(self.config.abci.genesis_core_height); + + // Init block execution context + let block_state_info = block_state_info::v0::BlockStateInfoV0::from_block_proposal( + &block_proposal, + last_block_time_ms, + ); + + // First let's check that this is the follower to a previous block + if !block_state_info.next_block_to(last_block_height, last_block_core_height)? { + // we are on the wrong height or round + return Ok(ValidationResult::new_with_error(AbciError::WrongBlockReceived(format!( + "received a block proposal for height: {} core height: {}, current height: {} core height: {}", + block_state_info.height, block_state_info.core_chain_locked_height, last_block_height, last_block_core_height + )).into())); + } + + // destructure the block proposal + let block_proposal::v0::BlockProposal { + core_chain_locked_height, + core_chain_lock_update, + proposed_app_version, + proposer_pro_tx_hash, + validator_set_quorum_hash, + raw_state_transitions, + .. + } = block_proposal; + + let block_info = block_state_info.to_block_info( + Epoch::new(epoch_info.current_epoch_index()) + .expect("current epoch index should be in range"), + ); + + if epoch_info.is_epoch_change_but_not_genesis() { + tracing::info!( + epoch_index = epoch_info.current_epoch_index(), + "epoch change occurring from epoch {} to epoch {}", + epoch_info + .previous_epoch_index() + .expect("must be set since we aren't on genesis"), + epoch_info.current_epoch_index(), + ); + } + + // Update block platform state with current and next epoch protocol versions + // if it was proposed + // This is happening only on epoch change + self.upgrade_protocol_version_on_epoch_change( + &block_info, + &epoch_info, + last_committed_platform_state, + &mut block_platform_state, + transaction, + platform_version, + )?; + + // If there is a core chain lock update, we should start by verifying it + if let Some(core_chain_lock_update) = core_chain_lock_update.as_ref() { + if !known_from_us { + let verification_result = self.verify_chain_lock( + block_state_info.round, // the round is to allow us to bypass local verification in case of chain stall + &block_platform_state, + core_chain_lock_update, + true, // if it's not known from us, then we should try submitting it + platform_version, + ); + + let VerifyChainLockResult { + chain_lock_signature_is_deserializable, + found_valid_locally, + found_valid_by_core, + core_is_synced, + } = match verification_result { + Ok(verification_result) => verification_result, + Err(Error::Execution(e)) => { + // This will happen only if an internal version error + return Err(Error::Execution(e)); + } + Err(e) => { + // This will happen only if a core rpc error + return Ok(ValidationResult::new_with_error( + AbciError::InvalidChainLock(e.to_string()).into(), + )); + } + }; + + if !chain_lock_signature_is_deserializable { + return Ok(ValidationResult::new_with_error( + AbciError::InvalidChainLock(format!( + "received a chain lock for height {} that has a signature that can not be deserialized {:?}", + block_info.height, core_chain_lock_update, + )) + .into(), + )); + } + + if let Some(found_valid_locally) = found_valid_locally { + // This means we are able to check if the chain lock is valid + if !found_valid_locally { + // The signature was not valid + return Ok(ValidationResult::new_with_error( + AbciError::InvalidChainLock(format!( + "received a chain lock for height {} that we figured out was invalid based on platform state {:?}", + block_info.height, core_chain_lock_update, + )) + .into(), + )); + } + } + + if let Some(found_valid_by_core) = found_valid_by_core { + // This means we asked core if the chain lock was valid + if !found_valid_by_core { + // Core said it wasn't valid + return Ok(ValidationResult::new_with_error( + AbciError::InvalidChainLock(format!( + "received a chain lock for height {} that is invalid based on a core request {:?}", + block_info.height, core_chain_lock_update, + )) + .into(), + )); + } + } + + if let Some(core_is_synced) = core_is_synced { + // Core is just not synced + if !core_is_synced { + // The submission was not accepted by core + return Ok(ValidationResult::new_with_error( + AbciError::ChainLockedBlockNotKnownByCore(format!( + "received a chain lock for height {} that we could not accept because core is not synced {:?}", + block_info.height, core_chain_lock_update, + )) + .into(), + )); + } + } + } + } + + // Update the masternode list and create masternode identities and also update the active quorums + self.update_core_info( + Some(last_committed_platform_state), + &mut block_platform_state, + core_chain_locked_height, + false, + &block_info, + transaction, + platform_version, + )?; + + // Update the validator proposed app version + // It should be called after protocol version upgrade + self.drive + .update_validator_proposed_app_version( + proposer_pro_tx_hash, + proposed_app_version as u32, + Some(transaction), + &platform_version.drive, + ) + .map_err(|e| { + Error::Execution(ExecutionError::UpdateValidatorProposedAppVersionError(e)) + })?; // This is a system error + + // Rebroadcast expired withdrawals if they exist + // We do that before we mark withdrawals as expired + // to rebroadcast them on the next block but not the same + // one + // TODO: It must be also only on core height change + self.rebroadcast_expired_withdrawal_documents( + &block_info, + last_committed_platform_state, + transaction, + platform_version, + )?; + + // Mark all previously broadcasted and chainlocked withdrawals as complete + // only when we are on a new core height + if block_state_info.core_chain_locked_height() != last_block_core_height { + self.update_broadcasted_withdrawal_statuses( + &block_info, + transaction, + platform_version, + )?; + } + + // Preparing withdrawal transactions for signing and broadcasting + // To process withdrawals we need to dequeue untiled transactions from the withdrawal transactions queue + // Untiled transactions then converted to unsigned transactions, appending current block information + // required for signature verification (core height and quorum hash) + // Then we save unsigned transaction bytes to block execution context + // to be signed (on extend_vote), verified (on verify_vote) and broadcasted (on finalize_block) + // Also, the dequeued untiled transaction added to the broadcasted transaction queue to for further + // resigning in case of failures. + let unsigned_withdrawal_transaction_bytes = self + .dequeue_and_build_unsigned_withdrawal_transactions( + validator_set_quorum_hash, + &block_info, + Some(transaction), + platform_version, + )?; + + // Run all dao platform events, such as vote tallying and distribution of contested documents + // This must be done before state transition processing + // Otherwise we would expect a proof after a successful vote that has since been cleaned up. + self.run_dao_platform_events( + &block_info, + last_committed_platform_state, + &block_platform_state, + Some(transaction), + platform_version, + )?; + + // Process transactions + let state_transitions_result = self.process_raw_state_transitions( + raw_state_transitions, + &block_platform_state, + &block_info, + transaction, + platform_version, + known_from_us, + timer, + )?; + + // Store the address balances to recent block storage + self.store_address_balances_to_recent_block_storage( + &state_transitions_result.address_balances_updated, + &block_info, + transaction, + platform_version, + )?; + + // Clean up expired compacted address balance entries + self.cleanup_recent_block_storage_address_balances( + &block_info, + transaction, + platform_version, + )?; + + // Record shielded pool anchor if the commitment tree changed this block. + // This stores block_height → anchor_bytes so shielded transactions can + // reference a recent anchor for spend authorization. + self.record_shielded_pool_anchor_if_changed( + block_proposal.height, + transaction, + platform_version, + )?; + + // Prune anchors older than the configured retention depth + self.prune_shielded_pool_anchors(block_proposal.height, transaction, platform_version)?; + + // Pool withdrawals into transactions queue + + // Takes queued withdrawals, creates untiled withdrawal transaction payload, saves them to queue + // Corresponding withdrawal documents are changed from queued to pooled + self.pool_withdrawals_into_transactions_queue( + &block_info, + last_committed_platform_state, + Some(transaction), + platform_version, + )?; + + // Cleans up the expired locks for withdrawal amounts + // to update daily withdrawal limit + // This is for example when we make a withdrawal for 30 Dash + // But we can only withdraw 1000 Dash a day + // after the withdrawal we should only be able to withdraw 970 Dash + // But 24 hours later that locked 30 comes back + self.clean_up_expired_locks_of_withdrawal_amounts( + &block_info, + transaction, + platform_version, + )?; + + // Create a new block execution context + + let mut block_execution_context: BlockExecutionContext = + block_execution_context::v0::BlockExecutionContextV0 { + block_state_info: block_state_info.into(), + epoch_info, + unsigned_withdrawal_transactions: unsigned_withdrawal_transaction_bytes, + block_address_balance_changes: std::collections::BTreeMap::new(), + block_platform_state, + proposer_results: None, + } + .into(); + + // while we have the state transitions executed, we now need to process the block fees + let block_fees_v0: BlockFeesV0 = state_transitions_result.aggregated_fees().clone().into(); + + // Process fees + let processed_block_fees = self.process_block_fees_and_validate_sum_trees( + &block_execution_context, + block_fees_v0.into(), + transaction, + platform_version, + )?; + + tracing::debug!(block_fees = ?processed_block_fees, "block fees are processed"); + + // Record the credits this block minted into Platform (asset locks funding state + // transitions, epoch Core rewards) as a credit inflow: the daily withdrawal limit adds + // inflows younger than its day-old base to the daily maximum, so it limits net outflow. + // A system event, so nobody pays fees for the write. + self.record_credit_inflows_for_withdrawals( + state_transitions_result + .credit_mints() + .saturating_add(processed_block_fees.credit_mints), + &block_info, + transaction, + platform_version, + )?; + + // Record the total credits in Platform if this block changed it: the daily withdrawal + // limit is a share of the total credits Platform held a day ago, read from this history. + // This runs after fees and epoch rewards, the last things in a block that can move the + // total, and before the app hash so the entry is part of this block's state. + self.record_total_credits_history_for_withdrawals( + &block_info, + transaction, + platform_version, + )?; + + // Unlike v0, the validator set update happens BEFORE the root hash is computed. + // It only mutates the in-memory block platform state (the rotated + // next_validator_set_quorum_hash) and never touches grovedb, so the rotation + // outcome and the root hash are unaffected by the move; it must come first so + // the reduced platform state written below carries the post-rotation state. + let validator_set_update = self.validator_set_update( + block_proposal.proposer_pro_tx_hash, + last_committed_platform_state, + &mut block_execution_context, + platform_version, + )?; + + // Write the reduced platform state into the replicated grovedb state, immediately + // before the root hash so it is covered by this block's app hash. A state-synced + // node reads it back to reconstruct the full platform state, which otherwise only + // exists in non-replicated aux storage. The app hash, block id hash and signature + // of this block are unknown at this point and are stored as `None`. + let reduced_platform_state = block_execution_context + .block_platform_state() + .to_reduced_platform_state( + Some(ReducedBlockInfoV0 { + basic_info: block_info, + app_hash: None, + quorum_hash: validator_set_quorum_hash.into(), + block_id_hash: None, + proposer_pro_tx_hash: proposer_pro_tx_hash.into(), + signature: None, + round: block_proposal.round, + }), + core_chain_locked_height, + ); + + self.store_reduced_platform_state( + &reduced_platform_state, + Some(transaction), + platform_version, + )?; + + let root_hash = self + .drive + .grove + .root_hash(Some(transaction), &platform_version.drive.grove_version) + .unwrap() + .map_err(|e| Error::Drive(drive::error::Error::from(e)))?; //GroveDb errors are system errors + + block_execution_context + .block_state_info_mut() + .set_app_hash(Some(root_hash)); + + if tracing::enabled!(tracing::Level::TRACE) { + tracing::trace!( + method = "run_block_proposal_v1", + app_hash = hex::encode(root_hash), + block_hash = hex::encode(block_proposal.block_hash.unwrap_or_default()), + platform_state_fingerprint = hex::encode( + block_execution_context + .block_platform_state() + .fingerprint()? + ), + "Block proposal executed successfully", + ); + } + + Ok(ValidationResult::new_with_data( + block_execution_outcome::v0::BlockExecutionOutcome { + app_hash: root_hash, + state_transitions_result, + validator_set_update, + platform_version, + block_execution_context, + }, + )) + } +} diff --git a/packages/rs-drive-abci/src/execution/platform_events/block_end/create_grovedb_checkpoint/v0/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/block_end/create_grovedb_checkpoint/v0/mod.rs index c6b1d43549e..632c68e4e58 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/block_end/create_grovedb_checkpoint/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/block_end/create_grovedb_checkpoint/v0/mod.rs @@ -37,13 +37,19 @@ where let block_height = platform_state.last_committed_block_height(); let block_time = platform_state.last_committed_block_time_ms().unwrap_or(0); - let keep_n = platform_version.drive_abci.checkpoints.num_checkpoints as usize; - - // Build the checkpoint path: db_path/checkpoints/ - let checkpoint_path = self - .config - .db_path - .join("checkpoints") + // When snapshot serving is enabled, the operator-provided state sync + // configuration overrides the platform-version-driven checkpoint retention. + let state_sync_config = &self.config.abci.state_sync; + let keep_n = if state_sync_config.snapshots_enabled { + state_sync_config.max_num_snapshots + } else { + platform_version.drive_abci.checkpoints.num_checkpoints as usize + }; + + // Build the checkpoint path: / + // (defaults to db_path/checkpoints) + let checkpoint_path = state_sync_config + .resolved_checkpoints_path(&self.config.db_path) .join(block_height.to_string()); // Create the parent checkpoints directory if it doesn't exist diff --git a/packages/rs-drive-abci/src/execution/platform_events/block_end/should_checkpoint/v0/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/block_end/should_checkpoint/v0/mod.rs index f16198bc266..dbb9e544f11 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/block_end/should_checkpoint/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/block_end/should_checkpoint/v0/mod.rs @@ -39,10 +39,22 @@ where return Ok(None); } - // How often we want a checkpoint - let checkpoint_interval_milliseconds = - platform_version.drive_abci.checkpoints.frequency_seconds as u64 * 1000; - let keep_n = platform_version.drive_abci.checkpoints.num_checkpoints as usize; + // How often we want a checkpoint. When snapshot serving is enabled, the + // operator-provided state sync configuration overrides the + // platform-version-driven checkpoint parameters. + let state_sync_config = &self.config.abci.state_sync; + let (frequency_seconds, keep_n) = if state_sync_config.snapshots_enabled { + ( + state_sync_config.snapshots_frequency_seconds as u64, + state_sync_config.max_num_snapshots, + ) + } else { + ( + platform_version.drive_abci.checkpoints.frequency_seconds as u64, + platform_version.drive_abci.checkpoints.num_checkpoints as usize, + ) + }; + let checkpoint_interval_milliseconds = frequency_seconds * 1000; // If disabled or misconfigured, do nothing. if checkpoint_interval_milliseconds == 0 || keep_n == 0 { diff --git a/packages/rs-drive-abci/src/execution/platform_events/block_end/update_checkpoints/v0/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/block_end/update_checkpoints/v0/mod.rs index f3b297ce7e0..a75d735c551 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/block_end/update_checkpoints/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/block_end/update_checkpoints/v0/mod.rs @@ -33,13 +33,19 @@ where return Ok(false); }; - let keep_n = platform_version.drive_abci.checkpoints.num_checkpoints as usize; + // When snapshot serving is enabled, the operator-provided state sync + // configuration overrides the platform-version-driven checkpoint retention. + let state_sync_config = &self.config.abci.state_sync; + let keep_n = if state_sync_config.snapshots_enabled { + state_sync_config.max_num_snapshots + } else { + platform_version.drive_abci.checkpoints.num_checkpoints as usize + }; - // Build the checkpoint path: db_path/checkpoints/ - let checkpoint_path = self - .config - .db_path - .join("checkpoints") + // Build the checkpoint path: / + // (defaults to db_path/checkpoints) + let checkpoint_path = state_sync_config + .resolved_checkpoints_path(&self.config.db_path) .join(block_height.to_string()); // Create the checkpoints directory if it doesn't exist diff --git a/packages/rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/mod.rs index 62f1cdfab8c..fcb2237a59a 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/mod.rs @@ -971,5 +971,132 @@ mod tests { "wrap-around should not trigger when last block was on different quorum" ); } + + /// run_block_proposal v1 (protocol v15) moves `validator_set_update` from AFTER + /// the root-hash computation (its v0 position) to BEFORE it, so the reduced + /// platform state written into the replicated state can carry the post-rotation + /// next validator set. The only observable differences between the two call + /// sites are (a) `block_state_info.app_hash` being set and (b) grovedb having + /// received additional writes in between. Rotation reads neither, and this test + /// proves it: for rotation-triggering and non-triggering scenarios alike, the + /// rotation outcome (returned update and resulting next validator set quorum + /// hash) is identical whether or not the app hash was set and grovedb was + /// written to before the call. + #[test] + fn v2_rotation_outcome_is_independent_of_root_hash_ordering() { + use crate::execution::types::block_execution_context::v0::BlockExecutionContextV0MutableGetters; + use crate::execution::types::block_state_info::v0::BlockStateInfoV0Setters; + use dpp::reduced_platform_state::v0::ReducedBlockInfoV0; + + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let platform_version = PlatformVersion::latest(); + + let mut rng = StdRng::seed_from_u64(57); + let qh1 = quorum_hash_from_seed(1); + let qh2 = quorum_hash_from_seed(2); + let vs1 = make_validator_set(qh1, &[10, 20, 30], &mut rng); + let vs2 = make_validator_set(qh2, &[40, 50, 60], &mut rng); + + let mut validator_sets = IndexMap::new(); + validator_sets.insert(qh1, vs1); + validator_sets.insert(qh2, vs2); + + // Scenarios: (proposer seed, last committed proposer seed, description) + // - proposer 20 after 10: mid-quorum, no rotation + // - proposer 30 after 20: last member, rotation to qh2 + // - proposer 10 after 20: wrap-around, rotation to qh2 + let scenarios: [(u8, u8, &str); 3] = [ + (20, 10, "no rotation"), + (30, 20, "rotation on last member"), + (10, 20, "rotation on wrap-around"), + ]; + + for (proposer_seed, last_proposer_seed, description) in scenarios { + let mut platform_state = platform.state.load().as_ref().clone(); + platform_state.set_current_validator_set_quorum_hash(qh1); + platform_state.set_validator_sets(validator_sets.clone()); + let mut last_proposer = [0u8; 32]; + last_proposer[31] = last_proposer_seed; + platform_state.set_last_committed_block_info(Some(make_extended_block_info( + *qh1.as_byte_array(), + last_proposer, + 5, + ))); + + let mut proposer = [0u8; 32]; + proposer[31] = proposer_seed; + + // v1 ordering: rotation runs before the root hash exists and before any + // reduced-state write. + let mut context_before_root_hash = + make_block_execution_context(platform_state.clone()); + let update_before = platform + .validator_set_update_v2( + proposer, + &platform_state, + &mut context_before_root_hash, + ) + .expect("should succeed before root hash"); + + // v0 ordering: by the time rotation runs, the app hash has been computed + // and set, and grovedb has received the block's writes (simulated here by + // a committed reduced-state write). + let reduced_platform_state = platform_state.to_reduced_platform_state( + Some(ReducedBlockInfoV0 { + basic_info: BlockInfo::default(), + app_hash: None, + quorum_hash: (*qh1.as_byte_array()).into(), + block_id_hash: None, + proposer_pro_tx_hash: proposer.into(), + signature: None, + round: 0, + }), + 1, + ); + platform + .store_reduced_platform_state(&reduced_platform_state, None, platform_version) + .expect("should store reduced platform state"); + let mut context_after_root_hash = + make_block_execution_context(platform_state.clone()); + context_after_root_hash + .block_state_info_mut() + .set_app_hash(Some([9u8; 32])); + let update_after = platform + .validator_set_update_v2( + proposer, + &platform_state, + &mut context_after_root_hash, + ) + .expect("should succeed after root hash"); + + assert_eq!( + update_before, update_after, + "validator set update must not depend on call ordering ({})", + description + ); + assert_eq!( + context_before_root_hash + .block_platform_state() + .next_validator_set_quorum_hash(), + context_after_root_hash + .block_platform_state() + .next_validator_set_quorum_hash(), + "next validator set quorum hash must not depend on call ordering ({})", + description + ); + assert_eq!( + context_before_root_hash + .block_platform_state() + .current_validator_set_quorum_hash(), + context_after_root_hash + .block_platform_state() + .current_validator_set_quorum_hash(), + "current validator set quorum hash must not depend on call ordering ({})", + description + ); + } + } } } diff --git a/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_core_info/v0/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_core_info/v0/mod.rs index 91564d2db1c..e4075cb173e 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_core_info/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_core_info/v0/mod.rs @@ -59,11 +59,18 @@ where platform_version, )?; + // `is_init_chain` doubles as `start_from_scratch`: on init chain and on state + // sync reconstruction the quorums must be built even if the (freshly + // constructed) block state happens to already report the requested core height. + // The flag's only effect inside update_quorum_info is to skip that + // same-core-height short-circuit; on the normal block path (`is_init_chain = + // false`) behavior is unchanged. The previous hardcoded `false` only worked + // because those flows start from a state whose derived core height is 0. self.update_quorum_info( platform_state, block_platform_state, core_block_height, - false, + is_init_chain, platform_version, ) } diff --git a/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_masternode_list/v0/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_masternode_list/v0/mod.rs index 3bc37499fa7..e7842b5fbb9 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_masternode_list/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_masternode_list/v0/mod.rs @@ -41,16 +41,20 @@ where transaction: &Transaction, platform_version: &PlatformVersion, ) -> Result<(), Error> { - if let Some(last_committed_block_info) = - block_platform_state.last_committed_block_info().as_ref() - { - if core_block_height == last_committed_block_info.basic_info().core_height { - tracing::debug!( - method = "update_masternode_list_v0", - "no update mnl at height {}", - core_block_height, - ); - return Ok(()); // no need to do anything + // On init chain and on state sync reconstruction the masternode list must be + // built from scratch even if the block state already reports this core height. + if !is_init_chain { + if let Some(last_committed_block_info) = + block_platform_state.last_committed_block_info().as_ref() + { + if core_block_height == last_committed_block_info.basic_info().core_height { + tracing::debug!( + method = "update_masternode_list_v0", + "no update mnl at height {}", + core_block_height, + ); + return Ok(()); // no need to do anything + } } } tracing::debug!( diff --git a/packages/rs-drive-abci/src/execution/platform_events/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/mod.rs index 1ac0715b9d2..32461d6a15e 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/mod.rs @@ -20,6 +20,8 @@ pub(in crate::execution) mod fee_pool_outwards_distribution; pub(in crate::execution) mod initialization; /// Protocol upgrade events pub(in crate::execution) mod protocol_upgrade; +/// State sync platform state reconstruction +pub(in crate::execution) mod state_sync; /// State transition processing pub(in crate::execution) mod state_transition_processing; mod tokens; diff --git a/packages/rs-drive-abci/src/execution/platform_events/protocol_upgrade/perform_events_on_first_block_of_protocol_change/v0/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/protocol_upgrade/perform_events_on_first_block_of_protocol_change/v0/mod.rs index 4f66d59dce8..ab03451de62 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/protocol_upgrade/perform_events_on_first_block_of_protocol_change/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/protocol_upgrade/perform_events_on_first_block_of_protocol_change/v0/mod.rs @@ -4,10 +4,12 @@ use crate::platform_types::platform::Platform; use crate::platform_types::platform_state::PlatformState; use crate::platform_types::platform_state::PlatformStateV0Methods; use dpp::block::block_info::BlockInfo; +use dpp::block::extended_block_info::v0::ExtendedBlockInfoV0Getters; use dpp::dashcore::hashes::Hash; use dpp::data_contracts::SystemDataContract; use dpp::fee::Credits; use dpp::platform_value::Identifier; +use dpp::reduced_platform_state::v0::ReducedBlockInfoV0; use dpp::serialization::PlatformDeserializable; use dpp::system_data_contracts::load_system_data_contract; use dpp::version::PlatformVersion; @@ -119,6 +121,10 @@ impl Platform { self.transition_to_version_14(block_info, transaction, platform_version)?; } + if previous_protocol_version < 15 && platform_version.protocol_version >= 15 { + self.transition_to_version_15(platform_state, transaction, platform_version)?; + } + Ok(()) } @@ -738,6 +744,44 @@ impl Platform { Ok(()) } + + /// When transitioning to version 15 we write the initial reduced platform state (built + /// from the last committed platform state) under `Misc/reduced_saved_state`, so the key + /// exists in the replicated state from the fork block onward. `run_block_proposal` v1 + /// overwrites it later in this same block with the state of the block being processed; + /// this initial write guarantees no v15 block ever commits without the key, which is + /// what makes every snapshot taken at or after activation restorable via state sync. + fn transition_to_version_15( + &self, + platform_state: &PlatformState, + transaction: &Transaction, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + let last_committed_block_info = + platform_state + .last_committed_block_info() + .as_ref() + .map(|extended_block_info| ReducedBlockInfoV0 { + basic_info: *extended_block_info.basic_info(), + app_hash: Some((*extended_block_info.app_hash()).into()), + quorum_hash: (*extended_block_info.quorum_hash()).into(), + block_id_hash: Some((*extended_block_info.block_id_hash()).into()), + proposer_pro_tx_hash: (*extended_block_info.proposer_pro_tx_hash()).into(), + signature: Some(*extended_block_info.signature()), + round: extended_block_info.round(), + }); + + let reduced_platform_state = platform_state.to_reduced_platform_state( + last_committed_block_info, + platform_state.last_committed_core_height(), + ); + + self.store_reduced_platform_state( + &reduced_platform_state, + Some(transaction), + platform_version, + ) + } } #[cfg(test)] @@ -2642,4 +2686,55 @@ mod tests { diffs.join("\n"), ); } + + #[test] + fn test_transition_to_version_15_writes_initial_reduced_platform_state() { + use dpp::reduced_platform_state::ReducedPlatformState; + + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let platform_version = PlatformVersion::latest(); + + let transaction = platform.drive.grove.start_transaction(); + + let platform_state = platform.state.load(); + + // Before the transition, the replicated state must not carry the reduced state key. + let pre_transition = platform + .fetch_reduced_platform_state(Some(&transaction), platform_version) + .expect("fetching an absent reduced platform state should not error"); + assert!( + pre_transition.is_none(), + "reduced platform state must not exist before transition_to_version_15" + ); + + let result = + platform.transition_to_version_15(&platform_state, &transaction, platform_version); + assert!(result.is_ok(), "transition failed: {:?}", result.err()); + + let reduced = platform + .fetch_reduced_platform_state(Some(&transaction), platform_version) + .expect("expected to fetch reduced platform state") + .expect("reduced platform state must exist after transition_to_version_15"); + + let ReducedPlatformState::V0(reduced) = reduced; + assert_eq!( + reduced.current_protocol_version_in_consensus, + platform_state.current_protocol_version_in_consensus() + ); + assert_eq!( + reduced.next_epoch_protocol_version, + platform_state.next_epoch_protocol_version() + ); + assert_eq!( + reduced.quorum_positions.len(), + platform_state.validator_sets().len(), + "quorum positions must mirror the validator set order" + ); + assert_eq!( + reduced.proposed_core_chain_locked_height, + platform_state.last_committed_core_height() + ); + } } diff --git a/packages/rs-drive-abci/src/execution/platform_events/state_sync/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/state_sync/mod.rs new file mode 100644 index 00000000000..f48dae65527 --- /dev/null +++ b/packages/rs-drive-abci/src/execution/platform_events/state_sync/mod.rs @@ -0,0 +1,3 @@ +//! State sync events: reconstruction of the platform state after a snapshot restore. + +mod reconstruct_platform_state; diff --git a/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs new file mode 100644 index 00000000000..311a96276c3 --- /dev/null +++ b/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs @@ -0,0 +1,285 @@ +use crate::abci::AbciError; +use crate::error::Error; +use crate::platform_types::platform::Platform; +use crate::platform_types::platform_state::{PlatformState, PlatformStateV0Methods}; +use crate::platform_types::signature_verification_quorum_set::SignatureVerificationQuorumSet; +use crate::platform_types::validator_set::ValidatorSet; +use crate::rpc::core::CoreRPCLike; +use dpp::block::extended_block_info::v0::{ExtendedBlockInfoV0, ExtendedBlockInfoV0Getters}; +use dpp::block::extended_block_info::ExtendedBlockInfo; +use dpp::dashcore::hashes::Hash; +use dpp::dashcore::QuorumHash; +use dpp::fee::default_costs::CachedEpochIndexFeeVersions; +use dpp::platform_value::Bytes32; +use dpp::reduced_platform_state::ReducedPlatformState; +use dpp::version::fee::FeeVersion; +use dpp::version::PlatformVersion; +use indexmap::IndexMap; +use std::collections::BTreeMap; + +impl Platform +where + C: CoreRPCLike, +{ + /// Reconstructs the full in-memory platform state after a state sync snapshot + /// restore, and persists it to aux storage so it survives restarts. + /// + /// ## Expected state + /// + /// The restored grovedb contains the reduced platform state that + /// `run_block_proposal` v1 wrote while processing the snapshot block, i.e. the + /// state after the whole block including `validator_set_update`, immediately + /// before the root hash was computed. Reconstruction: + /// + /// 1. restores the scalar fields (protocol versions, quorum hashes, fee versions) + /// directly from the reduced state; + /// 2. re-derives the masternode lists, masternode identities and quorums from Core + /// via `update_core_info` with `start_from_scratch = true` — the identity writes + /// are re-derivations of data already present in the restored state, so the + /// grovedb root hash MUST NOT change (the caller's root-hash equality check is + /// the proof of that idempotence); + /// 3. restores the validator set order recorded by the source (`quorum_positions`), + /// which cannot be recovered from Core RPC; + /// 4. advances the state to the snapshot block via `update_state_cache`, which + /// performs the same next-into-current validator set rotation the source node + /// performed when it finalized that block, persists the state to aux storage and + /// publishes it, so the `info` handler reports the snapshot height and app hash + /// after both this restore and any later restart. + pub fn reconstruct_platform_state( + &self, + app_hash: &[u8; 32], + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + let reduced_platform_state = self + .fetch_reduced_platform_state(None, platform_version)? + .ok_or_else(|| { + AbciError::StateSyncInternalError( + "reconstruct_platform_state restored snapshot does not contain a reduced \ + platform state (was it taken before the v15 activation height?)" + .to_string(), + ) + })?; + let ReducedPlatformState::V0(saved) = reduced_platform_state; + + // Everything below runs with the platform version the snapshot's chain was + // actually on, which may lag the version this binary considers latest. + let state_platform_version = + PlatformVersion::get(saved.current_protocol_version_in_consensus)?; + + // Restore the fee versions of previous epochs faithfully, by version number + let previous_fee_versions: CachedEpochIndexFeeVersions = saved + .previous_fee_versions + .iter() + .map(|(epoch_index, fee_version_number)| { + Ok((*epoch_index, FeeVersion::get(*fee_version_number)?)) + }) + .collect::>()?; + + let mut platform_state = PlatformState { + genesis_block_info: None, + last_committed_block_info: None, + current_protocol_version_in_consensus: saved.current_protocol_version_in_consensus, + next_epoch_protocol_version: saved.next_epoch_protocol_version, + current_validator_set_quorum_hash: QuorumHash::from_byte_array( + saved.current_validator_set_quorum_hash.to_buffer(), + ), + next_validator_set_quorum_hash: saved + .next_validator_set_quorum_hash + .map(|quorum_hash| QuorumHash::from_byte_array(quorum_hash.to_buffer())), + validator_sets: Default::default(), + chain_lock_validating_quorums: SignatureVerificationQuorumSet::new( + &self.config.chain_lock, + state_platform_version, + )?, + instant_lock_validating_quorums: SignatureVerificationQuorumSet::new( + &self.config.instant_lock, + state_platform_version, + )?, + full_masternode_list: Default::default(), + hpmn_masternode_list: Default::default(), + previous_fee_versions, + }; + + let saved_block_info = + saved + .last_committed_block_info + .ok_or(AbciError::StateSyncInternalError( + "reconstruct_platform_state reduced platform state has no last committed \ + block info" + .to_string(), + ))?; + + // The reduced state is written before the block's root hash exists, so its app + // hash is normally None and the snapshot app hash fills it in. If it does carry + // one, it must agree with the snapshot. + if let Some(saved_app_hash) = saved_block_info.app_hash { + if saved_app_hash.to_buffer() != *app_hash { + return Err(AbciError::StateSyncInternalError(format!( + "reconstruct_platform_state reduced platform state app hash {} does not \ + match snapshot app hash {}", + hex::encode(saved_app_hash.to_buffer()), + hex::encode(app_hash), + )) + .into()); + } + } + + let current_block_info: ExtendedBlockInfo = ExtendedBlockInfoV0 { + basic_info: saved_block_info.basic_info, + app_hash: *app_hash, + quorum_hash: saved_block_info.quorum_hash.to_buffer(), + // Not known during proposal processing, and not needed for consensus after + // a restore; restored as zeroes. + block_id_hash: saved_block_info + .block_id_hash + .map(|hash| hash.to_buffer()) + .unwrap_or_default(), + proposer_pro_tx_hash: saved_block_info.proposer_pro_tx_hash.to_buffer(), + // Same: unknown at store time, restored as zeroes when absent. + signature: saved_block_info.signature.unwrap_or([0u8; 96]), + round: saved_block_info.round, + } + .into(); + + // Re-derive masternode lists, masternode identities and quorums from Core, from + // scratch, at the core height the snapshot block ran with. The identity writes + // must be byte-identical to what is already in the restored state. + let transaction = self.drive.grove.start_transaction(); + self.update_core_info( + None, + &mut platform_state, + saved.proposed_core_chain_locked_height, + true, + current_block_info.basic_info(), + &transaction, + state_platform_version, + )?; + + // Core RPC returns quorums in an order that need not match the incremental + // order the source node maintained; restore the recorded order. + sort_validator_sets_by_saved_positions( + platform_state.validator_sets_mut(), + &saved.quorum_positions, + ); + + let block_height = platform_state.last_committed_block_height(); + + // Commit the re-derivation BEFORE the in-memory state is published: if this + // commit fails, nothing has been published and the error propagates with the + // node's observable state unchanged. (Publishing first, as normal block + // finalization does, would leave the info handler reporting a snapshot height + // that grovedb never persisted.) + self.drive + .grove + .commit_transaction(transaction) + .unwrap() + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "reconstruct_platform_state unable to commit transaction: {}", + e + )) + })?; + + // Advance the state to the snapshot block: rotates next-into-current exactly as + // the source did on finalization, persists to aux storage and publishes the + // state for the info handler. Aux writes are not part of the root hash, so + // committing them separately cannot change the app hash the caller verifies. + let aux_transaction = self.drive.grove.start_transaction(); + self.update_state_cache( + current_block_info, + platform_state, + &aux_transaction, + state_platform_version, + )?; + self.drive + .grove + .commit_transaction(aux_transaction) + .unwrap() + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "reconstruct_platform_state unable to commit aux transaction: {}", + e + )) + })?; + + tracing::debug!( + block_height, + app_hash = hex::encode(app_hash), + "[state_sync] platform state reconstructed", + ); + + Ok(()) + } +} + +/// Sorts the validator sets into the order recorded in the reduced platform state. +/// +/// Validator sets not present in the recorded order (which should not happen when the +/// reduced state and Core agree on the quorum list) sort last, preserving their +/// relative order. +fn sort_validator_sets_by_saved_positions( + validator_sets: &mut IndexMap, + quorum_positions: &[Bytes32], +) { + let lookup_table: BTreeMap<&[u8], usize> = quorum_positions + .iter() + .enumerate() + .map(|(position, quorum_hash)| (quorum_hash.as_slice(), position)) + .collect(); + + validator_sets.sort_by(|a_hash, _, b_hash, _| { + let a_position = lookup_table + .get(a_hash.as_byte_array().as_slice()) + .unwrap_or(&usize::MAX); + let b_position = lookup_table + .get(b_hash.as_byte_array().as_slice()) + .unwrap_or(&usize::MAX); + + a_position.cmp(b_position) + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn quorum_hash(seed: u8) -> QuorumHash { + let mut bytes = [0u8; 32]; + bytes[31] = seed; + QuorumHash::from_byte_array(bytes) + } + + #[test] + fn should_sort_validator_sets_into_saved_positions() { + use dpp::bls_signatures::{Bls12381G2Impl, SecretKey}; + use dpp::core_types::validator_set::v0::ValidatorSetV0; + use rand::rngs::StdRng; + use rand::SeedableRng; + + let mut rng = StdRng::seed_from_u64(7); + let mut validator_sets: IndexMap = IndexMap::new(); + for seed in [1u8, 2, 3] { + validator_sets.insert( + quorum_hash(seed), + ValidatorSet::V0(ValidatorSetV0 { + quorum_hash: quorum_hash(seed), + quorum_index: None, + core_height: 100, + members: Default::default(), + threshold_public_key: SecretKey::::random(&mut rng) + .public_key(), + }), + ); + } + + let saved_positions: Vec = [3u8, 1, 2] + .into_iter() + .map(|seed| quorum_hash(seed).to_byte_array().into()) + .collect(); + + sort_validator_sets_by_saved_positions(&mut validator_sets, &saved_positions); + + let order: Vec = validator_sets.keys().copied().collect(); + assert_eq!(order, vec![quorum_hash(3), quorum_hash(1), quorum_hash(2)]); + } +} diff --git a/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/mod.rs b/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/mod.rs new file mode 100644 index 00000000000..c7d52ae3e10 --- /dev/null +++ b/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/mod.rs @@ -0,0 +1,34 @@ +mod v0; + +use crate::error::execution::ExecutionError; +use crate::error::Error; +use crate::platform_types::platform::Platform; +use dpp::reduced_platform_state::ReducedPlatformState; +use dpp::version::PlatformVersion; +use drive::query::TransactionArg; + +impl Platform { + /// Fetch the reduced platform state from the replicated grovedb state. + /// + /// Returns `Ok(None)` when the reduced state is absent (a snapshot taken before the + /// protocol version that introduced it). + pub fn fetch_reduced_platform_state( + &self, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result, Error> { + match platform_version + .drive_abci + .methods + .platform_state_storage + .fetch_reduced_platform_state + { + 0 => self.fetch_reduced_platform_state_v0(transaction, platform_version), + version => Err(Error::Execution(ExecutionError::UnknownVersionMismatch { + method: "fetch_reduced_platform_state".to_string(), + known_versions: vec![0], + received: version, + })), + } + } +} diff --git a/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/v0/mod.rs b/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/v0/mod.rs new file mode 100644 index 00000000000..29ac4392488 --- /dev/null +++ b/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/v0/mod.rs @@ -0,0 +1,23 @@ +use crate::error::Error; +use crate::platform_types::platform::Platform; +use dpp::reduced_platform_state::ReducedPlatformState; +use dpp::serialization::PlatformDeserializableFromVersionedStructure; +use dpp::version::PlatformVersion; +use drive::query::TransactionArg; + +impl Platform { + pub(super) fn fetch_reduced_platform_state_v0( + &self, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result, Error> { + self.drive + .fetch_reduced_platform_state_bytes(transaction, platform_version) + .map_err(Error::Drive)? + .map(|bytes| { + ReducedPlatformState::versioned_deserialize(&bytes, platform_version) + .map_err(Error::Protocol) + }) + .transpose() + } +} diff --git a/packages/rs-drive-abci/src/execution/storage/mod.rs b/packages/rs-drive-abci/src/execution/storage/mod.rs index 92c2b2417dc..017babf8c28 100644 --- a/packages/rs-drive-abci/src/execution/storage/mod.rs +++ b/packages/rs-drive-abci/src/execution/storage/mod.rs @@ -1,2 +1,4 @@ pub mod fetch_platform_state; +mod fetch_reduced_platform_state; mod store_platform_state; +mod store_reduced_platform_state; diff --git a/packages/rs-drive-abci/src/execution/storage/store_reduced_platform_state/mod.rs b/packages/rs-drive-abci/src/execution/storage/store_reduced_platform_state/mod.rs new file mode 100644 index 00000000000..5b037ca8d39 --- /dev/null +++ b/packages/rs-drive-abci/src/execution/storage/store_reduced_platform_state/mod.rs @@ -0,0 +1,32 @@ +mod v0; + +use crate::error::execution::ExecutionError; +use crate::error::Error; +use crate::platform_types::platform::Platform; +use dpp::reduced_platform_state::ReducedPlatformState; +use dpp::version::PlatformVersion; +use drive::query::TransactionArg; + +impl Platform { + /// Store the reduced platform state in the replicated grovedb state + pub fn store_reduced_platform_state( + &self, + state: &ReducedPlatformState, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + match platform_version + .drive_abci + .methods + .platform_state_storage + .store_reduced_platform_state + { + 0 => self.store_reduced_platform_state_v0(state, transaction, platform_version), + version => Err(Error::Execution(ExecutionError::UnknownVersionMismatch { + method: "store_reduced_platform_state".to_string(), + known_versions: vec![0], + received: version, + })), + } + } +} diff --git a/packages/rs-drive-abci/src/execution/storage/store_reduced_platform_state/v0/mod.rs b/packages/rs-drive-abci/src/execution/storage/store_reduced_platform_state/v0/mod.rs new file mode 100644 index 00000000000..aa4db11a82e --- /dev/null +++ b/packages/rs-drive-abci/src/execution/storage/store_reduced_platform_state/v0/mod.rs @@ -0,0 +1,23 @@ +use crate::error::Error; +use crate::platform_types::platform::Platform; +use dpp::reduced_platform_state::ReducedPlatformState; +use dpp::serialization::PlatformSerializable; +use dpp::version::PlatformVersion; +use drive::query::TransactionArg; + +impl Platform { + pub(super) fn store_reduced_platform_state_v0( + &self, + state: &ReducedPlatformState, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + self.drive + .store_reduced_platform_state_bytes( + &state.serialize_to_bytes()?, + transaction, + platform_version, + ) + .map_err(Error::Drive) + } +} diff --git a/packages/rs-drive-abci/src/platform_types/mod.rs b/packages/rs-drive-abci/src/platform_types/mod.rs index 0f3b33981de..7f59de23585 100644 --- a/packages/rs-drive-abci/src/platform_types/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/mod.rs @@ -22,6 +22,8 @@ pub mod platform_state; pub mod required_identity_public_key_set; /// Signature verification quorums for Core pub mod signature_verification_quorum_set; +/// ABCI state sync snapshot types +pub mod snapshot; /// The state transition execution result as part of the block execution outcome pub mod state_transitions_processing_result; /// The validator module diff --git a/packages/rs-drive-abci/src/platform_types/platform/mod.rs b/packages/rs-drive-abci/src/platform_types/platform/mod.rs index 972e530a1ce..a59abe9d7e8 100644 --- a/packages/rs-drive-abci/src/platform_types/platform/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/platform/mod.rs @@ -10,6 +10,9 @@ use std::fmt::{Debug, Formatter}; use crate::platform_types::check_tx_proof_verifier::CheckTxProofVerifier; use crate::platform_types::platform_state::{PlatformState, PlatformStateV0Methods}; +use crate::platform_types::snapshot::{ + clear_restore_sentinel, restore_sentinel_exists, wipe_drive_for_restore, +}; use arc_swap::ArcSwap; use dpp::prelude::BlockHeight; use dpp::serialization::PlatformDeserializableFromVersionedStructure; @@ -147,6 +150,38 @@ impl Platform { let (drive, current_platform_version) = Drive::open(&config.db_path, Some(config.drive.clone())).map_err(Error::Drive)?; + // A state sync restore that never finished leaves grovedb holding state the + // platform state knows nothing about. That is not recoverable by restarting — + // the `info` handler panics on the mismatch, so the node would crash-loop — and it + // cannot be told apart from corruption without a marker. `offer_snapshot` writes + // one before it wipes; if it is still here, the restore did not finish. + // + // Recovery is to become an empty node: wipe, drop the caches derived from what was + // wiped, and come up as if freshly installed, so Tenderdash can offer another + // snapshot or fall back to block sync. Clearing the marker afterwards is safe + // precisely because an empty database with no saved state is self-consistent. + let current_platform_version = if restore_sentinel_exists(&config.db_path) { + tracing::warn!( + db_path = ?config.db_path, + "[state_sync] an unfinished state sync restore was found on startup; wiping \ + and coming up empty so the node can sync again", + ); + + wipe_drive_for_restore(&drive).map_err(Error::Drive)?; + clear_restore_sentinel(&config.db_path).map_err(|e| { + Error::Drive(drive::error::Error::IOErrorWithInfoString( + e.into(), + "trying to clear the state sync restore sentinel".to_owned(), + )) + })?; + + // The wipe removed the stored protocol version along with everything else, so + // the saved-state branch below must not be taken. + None + } else { + current_platform_version + }; + if let Some(platform_version) = current_platform_version { let Some(execution_state) = Platform::::fetch_platform_state(&drive, None, platform_version)? diff --git a/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs b/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs index 81f438fe51a..b6ea5973ea2 100644 --- a/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs @@ -26,6 +26,8 @@ use dpp::block::block_info::BlockInfo; use dpp::dashcore::hashes::Hash; use dpp::dashcore_rpc::json::MasternodeListItem; use dpp::fee::default_costs::CachedEpochIndexFeeVersions; +use dpp::reduced_platform_state::v0::{ReducedBlockInfoV0, ReducedPlatformStateV0}; +use dpp::reduced_platform_state::ReducedPlatformState; use dpp::util::hash::hash_double; use std::collections::BTreeMap; use std::fmt::{Debug, Formatter}; @@ -122,6 +124,44 @@ impl PlatformState { pub fn fingerprint(&self) -> Result<[u8; 32], Error> { Ok(hash_double(self.serialize_to_bytes()?)) } + + /// Builds the reduced platform state that is written into the replicated grovedb + /// state each block so a state-synced node can reconstruct the full platform state. + /// + /// `last_committed_block_info` describes the block currently being processed (it + /// becomes the last committed block once the block finalizes); fields that are not + /// known during proposal processing (app hash, block id hash, signature) are `None`. + /// `quorum_positions` records the order of the validator sets, which is not + /// otherwise recoverable from Core RPC during reconstruction. + pub fn to_reduced_platform_state( + &self, + last_committed_block_info: Option, + proposed_core_chain_locked_height: u32, + ) -> ReducedPlatformState { + ReducedPlatformState::V0(ReducedPlatformStateV0 { + last_committed_block_info, + current_protocol_version_in_consensus: self.current_protocol_version_in_consensus, + next_epoch_protocol_version: self.next_epoch_protocol_version, + current_validator_set_quorum_hash: self + .current_validator_set_quorum_hash + .to_byte_array() + .into(), + next_validator_set_quorum_hash: self + .next_validator_set_quorum_hash + .map(|quorum_hash| quorum_hash.to_byte_array().into()), + previous_fee_versions: self + .previous_fee_versions + .iter() + .map(|(epoch_index, fee_version)| (*epoch_index, fee_version.fee_version_number)) + .collect(), + quorum_positions: self + .validator_sets + .keys() + .map(|quorum_hash| quorum_hash.to_byte_array().into()) + .collect(), + proposed_core_chain_locked_height, + }) + } /// The default state at init chain pub fn default_with_protocol_versions( current_protocol_version_in_consensus: ProtocolVersion, diff --git a/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs b/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs new file mode 100644 index 00000000000..6290c334630 --- /dev/null +++ b/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs @@ -0,0 +1,251 @@ +//! ABCI state sync snapshot types. +//! +//! Snapshots are served directly from the rocksdb checkpoints Drive already creates +//! (`drive.checkpoints`, populated by `create_grovedb_checkpoint` after each qualifying +//! block is committed); there is no separate snapshot store. + +use drive::drive::{Checkpoint, Drive}; +use drive::grovedb::replication::MultiStateSyncSession; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; +use tenderdash_abci::proto::abci; + +/// Name of the marker file that records "a state sync restore is in progress". +/// +/// ## Why a plain file, and not aux storage +/// +/// The obvious home would be grovedb's aux column family, which is not part of the +/// provable tree. It cannot be used here: `GroveDb::wipe()` clears the aux column family +/// along with `default`, `roots` and `meta` +/// (`grovedb/storage/src/rocksdb_storage/storage.rs`, `wipe()` iterates all four). Since +/// wiping is exactly what both the offer path and the recovery path do, a sentinel living +/// in aux would be destroyed by the very operations it exists to survive, and its +/// lifetime would depend on subtle ordering between the write and the wipe. +/// +/// A file next to the database has none of those problems: it is outside everything +/// grovedb touches, it survives any wipe, it costs one `stat` at startup, and an operator +/// can see it. It is deliberately NOT in the provable tree either — it is node-local +/// recovery bookkeeping and must never affect the app hash. +pub const RESTORE_IN_PROGRESS_FILE_NAME: &str = "state_sync_restore_in_progress"; + +/// Path of the restore sentinel for a given database directory. +pub fn restore_sentinel_path(db_path: &Path) -> PathBuf { + db_path.join(RESTORE_IN_PROGRESS_FILE_NAME) +} + +/// Records that a state sync restore has started and the database is therefore allowed to +/// be inconsistent until it finishes. +/// +/// Written BEFORE the wipe, so the window in which the database has been destroyed but +/// nothing marks it as such is empty. The contents are for operators only; the code cares +/// solely about the file's presence. +pub fn write_restore_sentinel( + db_path: &Path, + app_hash: &[u8; 32], + height: u64, +) -> std::io::Result<()> { + std::fs::create_dir_all(db_path)?; + std::fs::write( + restore_sentinel_path(db_path), + format!( + "state sync restore in progress\nheight: {}\napp_hash: {}\n", + height, + hex::encode(app_hash) + ), + ) +} + +/// Clears the restore sentinel at a point where the node is already self-consistent — +/// after a completed restore, or after a genesis initialization — WITHOUT being able to +/// fail the operation that got it there. +/// +/// Propagating an I/O error from here would turn a fully successful restore (or a working +/// genesis) into a hard ABCI error over nothing but a `remove_file` hiccup. The cost of +/// failing to remove it is bounded and safe in the other direction: the next startup sees +/// a sentinel, wipes, and re-syncs. Loud, but never a wedge. +pub fn clear_restore_sentinel_best_effort(db_path: &Path) { + if let Err(error) = clear_restore_sentinel(db_path) { + tracing::error!( + ?error, + path = ?restore_sentinel_path(db_path), + "[state_sync] could not clear the state sync restore sentinel; the node is \ + consistent, but the next restart will wipe and re-sync unnecessarily. Remove \ + the file by hand to avoid that.", + ); + } +} + +/// Clears the restore sentinel. Only ever called once the node is in a self-consistent +/// state: after a restore has fully completed, after startup recovery has wiped, or after +/// a genesis initialization. +pub fn clear_restore_sentinel(db_path: &Path) -> std::io::Result<()> { + match std::fs::remove_file(restore_sentinel_path(db_path)) { + Ok(()) => Ok(()), + // Absent is the normal case on every path that clears defensively. + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } +} + +/// Whether a restore was in progress when this node last stopped. +pub fn restore_sentinel_exists(db_path: &Path) -> bool { + restore_sentinel_path(db_path).exists() +} + +/// Drops every Drive cache that was derived from grovedb. +/// +/// A wipe destroys the state these caches were built from. Left in place they would be +/// silently merged into whatever replaces it: `ProtocolVersionsCache` in particular keeps +/// a `loaded` flag, so `load_if_needed` would never re-read the new version counters and +/// the next block would write vote counts derived from the wiped chain — an immediate app +/// hash fork. Resetting the counter wholesale (rather than `clear_global_cache`) is +/// deliberate: it clears that flag too, so the cache reloads on first use. +/// +/// `system_data_contracts` is deliberately NOT cleared — those are compiled-in, +/// version-keyed contracts that never come from grovedb. +/// +/// The checkpoint registry goes too, and it is not merely a cache: `Drive::open` populates +/// `drive.checkpoints` before any wipe can run, and `list_snapshots` serves whatever is in +/// it to peers. Left alone, a node that wiped and re-synced would keep offering snapshots +/// of the chain it just discarded. The entries are marked for deletion first so their +/// directories are removed when the last `Arc` drops, rather than leaking on disk. +pub fn reset_drive_caches_after_wipe(drive: &Drive) { + *drive.cache.protocol_versions_counter.write() = Default::default(); + drive.cache.data_contracts.clear(); + *drive.cache.genesis_time_ms.write() = None; + + let checkpoints = drive.checkpoints.load(); + for checkpoint_info in checkpoints.values() { + checkpoint_info.checkpoint.mark_for_deletion(); + } + drive.checkpoints.store(Arc::new(BTreeMap::new())); +} + +/// Wipes grovedb and drops the caches derived from it, leaving the node an empty but +/// entirely self-consistent slate. +/// +/// This is the single place both the offer path and the crash-recovery path go through, +/// so the two can never drift apart. +pub fn wipe_drive_for_restore(drive: &Drive) -> Result<(), drive::error::Error> { + drive.grove.wipe()?; + reset_drive_caches_after_wipe(drive); + Ok(()) +} + +/// The grovedb state sync protocol versions this node can serve and consume. +/// +/// Exactly one protocol version exists: state sync never shipped, so grovedb updates +/// its replication protocol in place and stays at version 1. This single supported-set +/// constant and the offered-snapshot validation against it exist so that any future +/// incompatible protocol change fails fast on both the serving and consuming side +/// instead of producing a corrupt restore. (`drive_abci.state_sync.protocol_version` +/// is the version stamped on snapshots this node offers.) +pub const SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS: &[u16] = &[1]; + +/// Maximum accepted size (in bytes) of a single snapshot chunk, enforced before any +/// grovedb decode of peer-supplied data (issue #3773). +pub const MAX_STATE_SYNC_CHUNK_SIZE: usize = 16 * 1024 * 1024; + +/// Maximum accepted size (in bytes) of a chunk id, enforced before any grovedb decode +/// of peer-supplied data (issue #3773). Chunk ids are packed vectors of 32-byte subtree +/// prefixes plus short traversal instructions, so well-formed ids stay far below this. +pub const MAX_STATE_SYNC_CHUNK_ID_SIZE: usize = 64 * 1024; + +/// Maximum number of subtrees processed in a single batch of a grovedb state sync +/// session on the consuming side. +pub const STATE_SYNC_SUBTREES_BATCH_SIZE: usize = 64; + +/// A state sync transfer in progress on the consuming side. +pub struct SnapshotFetchingSession<'db> { + /// The snapshot being restored + pub snapshot: abci::Snapshot, + /// The light-client-verified app hash for the snapshot height, from Tenderdash + pub app_hash: [u8; 32], + /// The grovedb state sync wire protocol version this transfer speaks — taken from + /// the offered snapshot's `version`, validated against + /// [`SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS`], and used for every chunk of the + /// transfer. + pub wire_version: u16, + /// The grovedb state sync session + pub state_sync_info: Pin>>, +} + +/// How long a served checkpoint stays pinned after the last chunk request for it. +/// +/// A state-syncing peer requests chunks continuously; if none arrived for this long the +/// transfer is considered abandoned and the pin is released, allowing a checkpoint that +/// pruning already marked for deletion to be removed from disk. +const SERVING_PIN_INACTIVITY_TTL: Duration = Duration::from_secs(600); + +/// Keeps checkpoints that are actively being served to state-syncing peers alive. +/// +/// Checkpoint pruning marks old checkpoints for deletion and drops them from the +/// registry; the directory is removed when the last `Arc` drops. Holding an +/// `Arc` clone here for every checkpoint a peer is currently downloading extends that +/// refcount, so a checkpoint cannot be deleted mid-transfer. Pins are released after +/// [`SERVING_PIN_INACTIVITY_TTL`] of inactivity. +#[derive(Default)] +pub struct SnapshotManager { + /// Height -> (pinned checkpoint, instant of the most recent chunk request) + serving_pins: RwLock, Instant)>>, +} + +impl SnapshotManager { + /// Creates a new snapshot manager with no active pins + pub fn new() -> Self { + Self::default() + } + + /// Pins a checkpoint that is being served (or refreshes the pin of one that already + /// is), and drops pins whose transfers have been inactive for longer than the TTL. + pub fn pin_for_serving(&self, height: u64, checkpoint: Arc) { + let now = Instant::now(); + let mut pins = self + .serving_pins + .write() + .expect("serving pins lock poisoned"); + pins.retain(|_, (_, last_served)| { + now.saturating_duration_since(*last_served) < SERVING_PIN_INACTIVITY_TTL + }); + pins.insert(height, (checkpoint, now)); + } + + /// Returns a pinned checkpoint for the given height, if the pin is still held. + /// + /// Used to keep serving a snapshot whose checkpoint pruning has already dropped + /// from the registry. + pub fn pinned_checkpoint(&self, height: u64) -> Option> { + self.serving_pins + .read() + .expect("serving pins lock poisoned") + .get(&height) + .map(|(checkpoint, _)| Arc::clone(checkpoint)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn supported_wire_versions_include_the_version_platform_versions_stamp() { + use dpp::version::PlatformVersion; + // Every platform version stamps its state_sync.protocol_version on the + // snapshots it offers; the supported set must accept what we serve. + for platform_version in dpp::version::ALL_VERSIONS + .map(PlatformVersion::get) + .filter_map(Result::ok) + { + assert!( + SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS + .contains(&platform_version.drive_abci.state_sync.protocol_version), + "platform version {} stamps unsupported state sync wire version {}", + platform_version.protocol_version, + platform_version.drive_abci.state_sync.protocol_version + ); + } + } +} diff --git a/packages/rs-drive-abci/src/utils/mod.rs b/packages/rs-drive-abci/src/utils/mod.rs index b7292f50cff..9f9361d0195 100644 --- a/packages/rs-drive-abci/src/utils/mod.rs +++ b/packages/rs-drive-abci/src/utils/mod.rs @@ -2,5 +2,6 @@ mod serialization; mod spawn; pub use serialization::from_opt_str_or_number; +pub use serialization::from_str_or_native; pub use serialization::from_str_or_number; pub use spawn::spawn_blocking_task_with_name_if_supported; diff --git a/packages/rs-drive-abci/src/utils/serialization.rs b/packages/rs-drive-abci/src/utils/serialization.rs index 8259ff1dce3..cc5b965d49f 100644 --- a/packages/rs-drive-abci/src/utils/serialization.rs +++ b/packages/rs-drive-abci/src/utils/serialization.rs @@ -13,6 +13,29 @@ where s.parse::().map_err(Error::custom) } +/// Deserialize a value from a string (as provided by envy, where every value is a +/// string) or from its native representation (as in JSON round trips). +pub fn from_str_or_native<'de, D, T>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, + T: serde::Deserialize<'de> + std::str::FromStr, + ::Err: std::fmt::Display, +{ + use serde::de::Error; + + #[derive(Deserialize)] + #[serde(untagged)] + enum NativeOrString { + Native(T), + String(String), + } + + match NativeOrString::::deserialize(deserializer)? { + NativeOrString::Native(value) => Ok(value), + NativeOrString::String(s) => s.parse::().map_err(Error::custom), + } +} + /// Deserialize a value from an optional string or a number pub fn from_opt_str_or_number<'de, D, T>(deserializer: D) -> Result, D::Error> where diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/mod.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/mod.rs index dcd6ddd7ea4..06d0291b8df 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/test_cases/mod.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/mod.rs @@ -10,6 +10,8 @@ mod identity_transfer_tests; mod required_since_update_tests; // TODO: re-enable once OperationType shielded variants are implemented // mod shielded_tests; +mod state_sync_sentinel_tests; +pub(crate) mod state_sync_tests; mod token_tests; mod top_up_tests; mod update_identities_tests; diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs new file mode 100644 index 00000000000..b2753a574e2 --- /dev/null +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs @@ -0,0 +1,580 @@ +//! State sync QA: the restore sentinel and the never-wedge guarantee. +//! +//! A state sync restore destroys the node's database before it rebuilds it, and the +//! rebuild is not atomic with the platform state that has to describe it. Two things can +//! therefore leave a node holding a database its platform state knows nothing about: +//! +//! * the process dies between `commit_session` and `reconstruct_platform_state`; +//! * the restored snapshot turns out to be unusable (a pre-v15 snapshot with no reduced +//! platform state, or one that fails verification) — which any peer can cause. +//! +//! Both used to wedge the node permanently, because the `info` handler panics on an +//! app-hash mismatch and restarting reloads exactly the state that causes the panic. The +//! fix is a sentinel file written next to the database before the wipe, cleared only when +//! the node is self-consistent again, plus a rejection path that wipes back to a clean +//! slate instead of returning an error. +//! +//! # These tests do not need the patched grovedb +//! +//! Everything here holds at BOTH grovedb pins. Nothing asserts that a restore SUCCEEDS — +//! the tests that do live in `state_sync_equivalence_tests` and need dashpay/grovedb#840, +//! because Dash Platform state always contains sum trees. What is asserted here is that a +//! restore which does not succeed leaves a recoverable node, and at the unpinned revision +//! the sum-tree defect simply supplies the failure for free: the transfer commits, the +//! post-restore verification fails, and the same rejection path runs. + +#[cfg(test)] +mod tests { + use crate::execution::run_chain_for_strategy; + use crate::strategy::{ChainExecutionOutcome, NetworkStrategy}; + use crate::test_cases::state_sync_tests::tests::{ + install_reconstruction_core_mocks, sync_snapshot, SnapshotSyncOutcome, + }; + use dpp::version::PlatformVersion; + use drive_abci::abci::app::FullAbciApplication; + use drive_abci::config::{ + ChainLockConfig, ExecutionConfig, InstantLockConfig, PlatformConfig, PlatformTestConfig, + ValidatorSetConfig, + }; + use drive_abci::platform_types::platform::Platform; + use drive_abci::platform_types::platform_state::PlatformStateV0Methods; + use drive_abci::platform_types::snapshot::{ + restore_sentinel_exists, write_restore_sentinel, RESTORE_IN_PROGRESS_FILE_NAME, + }; + use drive_abci::rpc::core::MockCoreRPCLike; + use drive_abci::test::helpers::setup::{TempPlatform, TestPlatformBuilder}; + use strategy_tests::frequency::Frequency; + use strategy_tests::{IdentityInsertInfo, StartAddresses, StartIdentities, Strategy}; + use tenderdash_abci::proto::abci as proto; + use tenderdash_abci::proto::abci::response_offer_snapshot; + use tenderdash_abci::Application; + + const SOURCE_CHAIN_BLOCKS: u64 = 6; + const SOURCE_CHAIN_SEED: u64 = 15; + + fn sentinel_platform_config() -> PlatformConfig { + let mut testing_configs = PlatformTestConfig::default_minimal_verifications(); + testing_configs.disable_checkpoints = false; + testing_configs.store_platform_state = true; + + let mut config = PlatformConfig { + validator_set: ValidatorSetConfig::default_100_67(), + chain_lock: ChainLockConfig::default_100_67(), + instant_lock: InstantLockConfig::default_100_67(), + execution: ExecutionConfig { + verify_sum_trees: true, + ..ExecutionConfig::default() + }, + block_spacing_ms: 3000, + testing_configs, + ..Default::default() + }; + config.abci.state_sync.snapshots_enabled = true; + config.abci.state_sync.snapshots_frequency_seconds = 1; + config.abci.state_sync.max_num_snapshots = 3; + config + } + + fn sentinel_strategy() -> NetworkStrategy { + NetworkStrategy { + strategy: Strategy { + start_contracts: vec![], + operations: vec![], + start_identities: StartIdentities::default(), + start_addresses: StartAddresses::default(), + identity_inserts: IdentityInsertInfo { + frequency: Frequency { + times_per_block_range: 1..3, + chance_per_block: None, + }, + ..Default::default() + }, + identity_contract_nonce_gaps: None, + signer: None, + }, + total_hpmns: 100, + extra_normal_mns: 0, + validator_quorum_count: 24, + chain_lock_quorum_count: 24, + upgrading_info: None, + proposer_strategy: Default::default(), + rotate_quorums: false, + failure_testing: None, + query_testing: None, + verify_state_transition_results: false, + ..Default::default() + } + } + + fn root_hash(platform: &Platform) -> [u8; 32] { + platform + .drive + .grove + .root_hash(None, &PlatformVersion::latest().drive.grove_version) + .unwrap() + .expect("root hash") + } + + fn info_request() -> proto::RequestInfo { + proto::RequestInfo { + version: tenderdash_abci::proto::meta::TENDERDASH_VERSION.to_string(), + block_version: 0, + p2p_version: 0, + abci_version: tenderdash_abci::proto::meta::ABCI_VERSION.to_string(), + } + } + + /// Models process death: drops the `Platform` (releasing grovedb's lock and every + /// in-memory session, cache and platform state) and re-opens the SAME directory, which + /// is what a restarted drive-abci does. Only what was durably written survives. + fn restart( + target: TempPlatform, + config: &PlatformConfig, + ) -> TempPlatform { + let TempPlatform { + platform, tempdir, .. + } = target; + drop(platform); + TempPlatform::open_with_tempdir(tempdir, config.clone()) + } + + /// Calling `info` must not panic. The handler panics on an app-hash mismatch between + /// the platform state and grovedb, which is the exact shape of the wedge, so "did it + /// panic" is the property under test rather than the returned value. + fn info_does_not_panic(platform: &TempPlatform) -> bool { + let app = FullAbciApplication::new(platform); + let previous_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let result = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| app.info(info_request()))); + std::panic::set_hook(previous_hook); + result.is_ok() + } + + /// `offer_snapshot` must record the sentinel BEFORE it wipes, so there is no window in + /// which the database has been destroyed and nothing says so. + #[tokio::test] + async fn offer_snapshot_records_the_restore_sentinel_before_wiping() { + let config = sentinel_platform_config(); + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let outcome = run_chain_for_strategy( + &mut platform.platform, + SOURCE_CHAIN_BLOCKS, + sentinel_strategy(), + config.clone(), + SOURCE_CHAIN_SEED, + &mut None, + &mut None, + ) + .await; + let app = outcome.abci_app; + let db_path = app.platform.config.db_path.clone(); + + assert!( + !restore_sentinel_exists(&db_path), + "a node that never state-synced must not carry the sentinel" + ); + + let response = app + .offer_snapshot(proto::RequestOfferSnapshot { + snapshot: Some(proto::Snapshot { + height: 1000, + version: 1, + hash: vec![7u8; 32], + metadata: vec![], + }), + app_hash: vec![7u8; 32], + }) + .expect("the offer must be accepted"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::Accept) + ); + + assert!( + restore_sentinel_exists(&db_path), + "accepting an offer wipes the database, so it must first record that a restore \ + is in progress" + ); + // The sentinel is a plain file NEXT TO the database, not aux storage: `wipe()` + // clears the aux column family too, so a sentinel stored there would be destroyed + // by the very wipe it exists to survive. + assert!( + db_path.join(RESTORE_IN_PROGRESS_FILE_NAME).is_file(), + "the sentinel must live outside everything grovedb wipes" + ); + } + + /// A wipe must also drop the CHECKPOINT REGISTRY, not just the value caches. + /// + /// `drive.checkpoints` is populated by `Drive::open` and is what `list_snapshots` + /// serves to peers. It is not a cache that merely goes stale: left in place across a + /// wipe, a node that discarded a chain would keep advertising snapshots of it, and a + /// peer state-syncing from those would restore a chain this node no longer has and + /// cannot vouch for. + #[tokio::test] + async fn a_wiped_node_stops_serving_snapshots_of_the_discarded_chain() { + let config = sentinel_platform_config(); + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let outcome = run_chain_for_strategy( + &mut platform.platform, + SOURCE_CHAIN_BLOCKS, + sentinel_strategy(), + config.clone(), + SOURCE_CHAIN_SEED, + &mut None, + &mut None, + ) + .await; + let app = outcome.abci_app; + + assert!( + !app.list_snapshots(Default::default()) + .expect("should list snapshots") + .snapshots + .is_empty(), + "sanity: the chain must have produced servable snapshots to begin with" + ); + + // Accepting an offer wipes the database out from under those checkpoints. + app.offer_snapshot(proto::RequestOfferSnapshot { + snapshot: Some(proto::Snapshot { + height: 1000, + version: 1, + hash: vec![7u8; 32], + metadata: vec![], + }), + app_hash: vec![7u8; 32], + }) + .expect("the offer must be accepted"); + + assert!( + app.list_snapshots(Default::default()) + .expect("should list snapshots") + .snapshots + .is_empty(), + "after a wipe the node must stop advertising snapshots of the chain it just \ + discarded — otherwise a peer would state-sync from state this node no longer has" + ); + assert!( + app.platform.drive.checkpoints.load().is_empty(), + "the checkpoint registry itself must be cleared, not just filtered at serve time" + ); + } + + /// A rejected offer must not record a sentinel — nothing was wiped, so nothing needs + /// recovering. Without this, any peer could make a healthy node wipe itself on the next + /// restart just by offering a snapshot in a format it cannot speak. + #[tokio::test] + async fn a_rejected_offer_records_no_sentinel() { + let config = sentinel_platform_config(); + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let outcome = run_chain_for_strategy( + &mut platform.platform, + SOURCE_CHAIN_BLOCKS, + sentinel_strategy(), + config.clone(), + SOURCE_CHAIN_SEED, + &mut None, + &mut None, + ) + .await; + let app = outcome.abci_app; + let db_path = app.platform.config.db_path.clone(); + + let response = app + .offer_snapshot(proto::RequestOfferSnapshot { + snapshot: Some(proto::Snapshot { + height: 1000, + version: u32::MAX, + hash: vec![7u8; 32], + metadata: vec![], + }), + app_hash: vec![7u8; 32], + }) + .expect("an unsupported version must be answered, not error"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::RejectFormat) + ); + assert!( + !restore_sentinel_exists(&db_path), + "a rejected offer wipes nothing and must leave no sentinel behind" + ); + } + + /// The startup recovery itself: a node whose sentinel is still present comes up EMPTY + /// rather than crash-looping. + /// + /// The database here is deliberately a healthy, fully populated chain — the strongest + /// form of "grovedb holds state the platform state will not describe". Recovery must + /// throw it away, because there is no way to tell how far an interrupted restore got. + #[tokio::test] + async fn a_node_restarting_mid_restore_wipes_and_comes_up_empty() { + let config = sentinel_platform_config(); + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let outcome = run_chain_for_strategy( + &mut platform.platform, + SOURCE_CHAIN_BLOCKS, + sentinel_strategy(), + config.clone(), + SOURCE_CHAIN_SEED, + &mut None, + &mut None, + ) + .await; + let db_path = outcome.abci_app.platform.config.db_path.clone(); + assert_eq!( + outcome + .abci_app + .platform + .state + .load() + .last_committed_block_height(), + SOURCE_CHAIN_BLOCKS + ); + drop(outcome); + + // A restore was in progress when the process died. + write_restore_sentinel(&db_path, &[9u8; 32], 1000).expect("write sentinel"); + + let restarted = restart(platform, &config); + + assert_eq!( + restarted.state.load().last_committed_block_height(), + 0, + "an unfinished restore must not leave the node claiming a height it cannot back up" + ); + assert_eq!( + root_hash(&restarted.platform), + [0u8; 32], + "the database must have been wiped to an empty, self-consistent state" + ); + assert!( + !restore_sentinel_exists(&db_path), + "once the node is empty it is self-consistent again, so the sentinel is cleared" + ); + assert!( + info_does_not_panic(&restarted), + "THE WHOLE POINT: the info handshake must succeed, so the node can be offered \ + another snapshot or fall back to block sync instead of crash-looping" + ); + } + + /// The complement, and the regression that matters most: a node WITHOUT a sentinel + /// must never be wiped. If startup recovery ever fires unconditionally it would + /// silently destroy every node's chain on restart. + #[tokio::test] + async fn a_normal_restart_keeps_its_state() { + let config = sentinel_platform_config(); + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let outcome = run_chain_for_strategy( + &mut platform.platform, + SOURCE_CHAIN_BLOCKS, + sentinel_strategy(), + config.clone(), + SOURCE_CHAIN_SEED, + &mut None, + &mut None, + ) + .await; + let db_path = outcome.abci_app.platform.config.db_path.clone(); + let healthy_root_hash = root_hash(outcome.abci_app.platform); + drop(outcome); + + assert!(!restore_sentinel_exists(&db_path)); + let restarted = restart(platform, &config); + + assert_eq!( + restarted.state.load().last_committed_block_height(), + SOURCE_CHAIN_BLOCKS, + "a normal restart must come back at the tip" + ); + assert_eq!( + root_hash(&restarted.platform), + healthy_root_hash, + "a normal restart must not touch the database" + ); + assert!(info_does_not_panic(&restarted)); + } + + /// The block-sync arm of the recovery. If every offered snapshot is rejected, + /// Tenderdash gives up on state sync and block-syncs from genesis. `init_chain` is + /// where the node becomes self-consistent again, so it must clear a sentinel left over + /// from the abandoned restore — otherwise the NEXT restart would wipe a perfectly good + /// chain. + #[tokio::test] + async fn init_chain_clears_a_sentinel_left_by_an_abandoned_restore() { + let config = sentinel_platform_config(); + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let db_path = platform.platform.config.db_path.clone(); + + // An abandoned restore: the marker is present and the database is empty. + write_restore_sentinel(&db_path, &[9u8; 32], 1000).expect("write sentinel"); + + // Block sync from genesis, which begins with init_chain. + let outcome = run_chain_for_strategy( + &mut platform.platform, + SOURCE_CHAIN_BLOCKS, + sentinel_strategy(), + config.clone(), + SOURCE_CHAIN_SEED, + &mut None, + &mut None, + ) + .await; + + assert_eq!( + outcome + .abci_app + .platform + .state + .load() + .last_committed_block_height(), + SOURCE_CHAIN_BLOCKS, + "the node must block-sync normally after an abandoned restore" + ); + assert!( + !restore_sentinel_exists(&db_path), + "init_chain makes the node self-consistent, so it must clear the sentinel — \ + otherwise the next restart would wipe this chain" + ); + drop(outcome); + + // And prove it: a restart keeps the block-synced chain. + let restarted = restart(platform, &config); + assert_eq!( + restarted.state.load().last_committed_block_height(), + SOURCE_CHAIN_BLOCKS, + "the chain built after an abandoned restore must survive a restart" + ); + } + + /// End to end for the remotely-triggerable case: a peer offers a snapshot this node + /// cannot use, and the node must end up able to sync rather than wedged. + /// + /// The snapshot here is a pre-v15 one (a v14 chain's checkpoint, which carries no + /// reduced platform state). Nothing stops a peer from advertising it: `proto::Snapshot` + /// carries a height, a wire version and a hash, and no protocol version at all. + /// + /// This test is pin-agnostic on purpose. With grovedb #840 the refusal comes from the + /// missing reduced platform state; at the unpatched revision the sum-tree defect makes + /// the post-restore verification fail first. Either way the snapshot is refused AFTER + /// the session was committed, which is precisely the path that has to leave the node + /// recoverable. + #[tokio::test] + async fn an_unusable_snapshot_leaves_the_node_able_to_sync() { + let config = sentinel_platform_config(); + + let mut source_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .with_initial_protocol_version(14) + .build_with_mock_rpc(); + let ChainExecutionOutcome { + abci_app: source_app, + proposers, + validator_quorums, + .. + } = run_chain_for_strategy( + &mut source_platform.platform, + SOURCE_CHAIN_BLOCKS, + sentinel_strategy(), + config.clone(), + SOURCE_CHAIN_SEED, + &mut None, + &mut None, + ) + .await; + + let (height, checkpoint) = { + let checkpoints = source_app.platform.drive.checkpoints.load(); + let (height, info) = checkpoints + .last_key_value() + .expect("at least one checkpoint"); + (*height, std::sync::Arc::clone(&info.checkpoint)) + }; + let platform_version = PlatformVersion::latest(); + let checkpoint_root = checkpoint + .grove_db + .root_hash(None, &platform_version.drive.grove_version) + .unwrap() + .expect("checkpoint root hash"); + let forged_snapshot = proto::Snapshot { + height, + version: platform_version.drive_abci.state_sync.protocol_version as u32, + hash: checkpoint_root.to_vec(), + metadata: vec![], + }; + + let mut target_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + install_reconstruction_core_mocks( + &mut target_platform.platform, + proposers + .iter() + .map(|proposer| proposer.masternode.clone()) + .collect(), + &validator_quorums, + ); + let db_path = target_platform.platform.config.db_path.clone(); + + { + let target_app = FullAbciApplication::new(&target_platform); + let outcome = sync_snapshot(&source_app, &target_app, &forged_snapshot, false) + .expect("an unusable snapshot must be answered, not errored"); + assert_eq!( + outcome, + SnapshotSyncOutcome::Rejected, + "an unusable snapshot must be answered with REJECT_SNAPSHOT so Tenderdash \ + tries the next one instead of aborting state sync" + ); + + // The node wiped itself back to a clean slate rather than keeping state it + // cannot use... + assert_ne!( + root_hash(&target_platform.platform).to_vec(), + forged_snapshot.hash, + "the refused snapshot must not be left on disk" + ); + assert_eq!( + root_hash(&target_platform.platform), + [0u8; 32], + "the refusal must leave an empty database" + ); + // ...and the sentinel stays, because the in-memory platform state may still + // describe the chain the offer wiped. Whatever happens next resolves it. + assert!( + restore_sentinel_exists(&db_path), + "the node is empty but not yet provably consistent, so the marker stays \ + until a restore succeeds, an init_chain runs, or a restart wipes" + ); + } + + // A restart is the worst case, and it recovers. + let restarted = restart(target_platform, &config); + assert_eq!(restarted.state.load().last_committed_block_height(), 0); + assert_eq!(root_hash(&restarted.platform), [0u8; 32]); + assert!( + !restore_sentinel_exists(&db_path), + "startup recovery leaves the node self-consistent and clears the marker" + ); + assert!( + info_does_not_panic(&restarted), + "THE FIX: a peer offering an unusable snapshot must not be able to wedge this \ + node. Before the fix, info panicked here and drive-abci crash-looped." + ); + } +} diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs new file mode 100644 index 00000000000..aced9fb5d56 --- /dev/null +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs @@ -0,0 +1,800 @@ +//! Two-instance ABCI state sync integration tests: a source chain serves snapshots from +//! its checkpoint registry and a fresh target restores one chunk by chunk, then +//! reconstructs its platform state. +//! +//! KNOWN LIMITATION at the pinned grovedb revision (6c882c3): state sync does not +//! faithfully restore SumTree subtrees — the copied node hashes reproduce the source +//! root hash, but re-opening a restored sum tree recomputes a different root (latent +//! corruption), which the strict `verify_grovedb` call in `apply_snapshot_chunk` +//! correctly refuses. See `tests/sum_tree_sync_probe.rs` for the minimal upstream +//! reproducer. The full happy-path test below is therefore `#[ignore]`d until the +//! grovedb pin includes the sum-tree restore fix (dashpay/grovedb#840), and an active +//! test pins today's refusal behavior instead. + +#[cfg(test)] +pub(crate) mod tests { + use crate::execution::run_chain_for_strategy; + use crate::strategy::{ChainExecutionOutcome, NetworkStrategy}; + use dpp::dashcore::hashes::Hash; + use dpp::dashcore::{BlockHash, QuorumHash}; + use dpp::dashcore_rpc::dashcore_rpc_json::{ + ExtendedQuorumDetails, MasternodeListDiff, MasternodeListItem, QuorumInfoResult, + }; + use dpp::dashcore_rpc::json::{ExtendedQuorumListResult, QuorumType}; + use dpp::version::PlatformVersion; + use drive_abci::abci::app::FullAbciApplication; + use drive_abci::config::{ + ChainLockConfig, ExecutionConfig, InstantLockConfig, PlatformConfig, PlatformTestConfig, + ValidatorSetConfig, + }; + use drive_abci::mimic::test_quorum::TestQuorumInfo; + use drive_abci::platform_types::platform::Platform; + use drive_abci::platform_types::platform_state::PlatformStateV0Methods; + use drive_abci::rpc::core::MockCoreRPCLike; + use drive_abci::test::helpers::setup::TestPlatformBuilder; + use std::collections::{BTreeMap, HashMap, VecDeque}; + use strategy_tests::frequency::Frequency; + use strategy_tests::{IdentityInsertInfo, StartAddresses, StartIdentities, Strategy}; + use tenderdash_abci::proto::abci as proto; + use tenderdash_abci::proto::abci::{response_apply_snapshot_chunk, response_offer_snapshot}; + use tenderdash_abci::Application; + + /// A quiet chain with a trickle of identity inserts, no masternode churn and no + /// quorum rotation, so the target's from-scratch Core re-derivation sees exactly + /// the same masternodes and quorums the source chain ran with. + fn state_sync_network_strategy() -> NetworkStrategy { + NetworkStrategy { + strategy: Strategy { + start_contracts: vec![], + operations: vec![], + start_identities: StartIdentities::default(), + start_addresses: StartAddresses::default(), + identity_inserts: IdentityInsertInfo { + frequency: Frequency { + times_per_block_range: 1..2, + chance_per_block: None, + }, + ..Default::default() + }, + identity_contract_nonce_gaps: None, + signer: None, + }, + total_hpmns: 100, + extra_normal_mns: 0, + validator_quorum_count: 24, + chain_lock_quorum_count: 24, + upgrading_info: None, + proposer_strategy: Default::default(), + rotate_quorums: false, + failure_testing: None, + query_testing: None, + verify_state_transition_results: false, + ..Default::default() + } + } + + /// Snapshot serving on with a 1s frequency (every 3s block crosses the boundary, + /// so every block after the first creates a checkpoint), keeping 3 checkpoints. + fn state_sync_platform_config() -> PlatformConfig { + let mut testing_configs = PlatformTestConfig::default_minimal_verifications(); + testing_configs.disable_checkpoints = false; + testing_configs.store_platform_state = true; + + let mut config = PlatformConfig { + validator_set: ValidatorSetConfig::default_100_67(), + chain_lock: ChainLockConfig::default_100_67(), + instant_lock: InstantLockConfig::default_100_67(), + execution: ExecutionConfig { + verify_sum_trees: true, + ..ExecutionConfig::default() + }, + block_spacing_ms: 3000, + testing_configs, + ..Default::default() + }; + config.abci.state_sync.snapshots_enabled = true; + config.abci.state_sync.snapshots_frequency_seconds = 1; + config.abci.state_sync.max_num_snapshots = 3; + config + } + + /// Installs on a fresh target the Core RPC answers its platform state + /// reconstruction will ask for: the full masternode list (the target requests it + /// from scratch, base height None) and the same quorums the source ran with. + pub(crate) fn install_reconstruction_core_mocks( + platform: &mut Platform, + masternodes: Vec, + validator_quorums: &BTreeMap, + ) { + platform + .core_rpc + .expect_get_protx_diff_with_masternodes() + .returning(move |base_block, block| { + assert!( + base_block.is_none(), + "state reconstruction must request the full masternode list from scratch" + ); + Ok(MasternodeListDiff { + base_height: 0, + block_height: block, + added_mns: masternodes.clone(), + removed_mns: vec![], + updated_mns: vec![], + }) + }); + + let quorum_details: Vec<(QuorumHash, ExtendedQuorumDetails)> = validator_quorums + .keys() + .map(|quorum_hash| { + ( + *quorum_hash, + ExtendedQuorumDetails { + creation_height: 0, + quorum_index: None, + mined_block_hash: BlockHash::all_zeros(), + num_valid_members: 0, + health_ratio: 0.0, + }, + ) + }) + .collect(); + platform + .core_rpc + .expect_get_quorum_listextended() + .returning(move |_| { + Ok(ExtendedQuorumListResult { + quorums_by_type: HashMap::from([( + QuorumType::Llmq100_67, + quorum_details.clone().into_iter().collect(), + )]), + }) + }); + + let quorum_infos: HashMap = validator_quorums + .iter() + .map(|(quorum_hash, test_quorum_info)| (*quorum_hash, test_quorum_info.into())) + .collect(); + platform.core_rpc.expect_get_quorum_info().returning( + move |_, quorum_hash: &QuorumHash, _| { + Ok(quorum_infos + .get::(quorum_hash) + .unwrap_or_else(|| { + panic!("expected to get quorum {}", hex::encode(quorum_hash)) + }) + .clone()) + }, + ); + } + + /// Drives the chunk transfer loop between a serving app and a restoring app, + /// modeled on grovedb's run_sync driver: start from the root chunk (id == app + /// hash) and keep requesting whatever the target asks for next. + /// + /// When `tamper_with_first_chunk` is set, the first served chunk is corrupted to + /// prove the target answers RETRY with a refetch of exactly that chunk (banning + /// the sender) instead of killing the session. At the current grovedb revision the + /// refetched chunk cannot be re-applied within the session (grovedb removes a + /// chunk id from its pending set before processing), so the target then answers + /// RETRY_SNAPSHOT; the driver handles that the way Tenderdash would, by + /// re-offering the same snapshot and restarting the transfer. + /// How a snapshot transfer ended. + /// + /// `Rejected` is not an error: the target restored the snapshot, found it unusable, + /// wiped itself back to a clean slate and asked Tenderdash for a different one. The + /// driver reports it so tests can tell a clean refusal from a transport failure. + #[derive(Debug, PartialEq, Eq)] + pub(crate) enum SnapshotSyncOutcome { + /// The target restored and accepted the snapshot. + Completed, + /// The target answered REJECT_SNAPSHOT; Tenderdash would move on to the next one. + Rejected, + } + + pub(crate) fn sync_snapshot( + source_app: &FullAbciApplication, + target_app: &FullAbciApplication, + snapshot: &proto::Snapshot, + tamper_with_first_chunk: bool, + ) -> Result { + let mut tamper_next = tamper_with_first_chunk; + let mut restarts = 0usize; + + 'snapshot_attempt: loop { + let offer_response = target_app.offer_snapshot(proto::RequestOfferSnapshot { + snapshot: Some(snapshot.clone()), + app_hash: snapshot.hash.clone(), + })?; + assert_eq!( + offer_response.result, + i32::from(response_offer_snapshot::Result::Accept), + "target must accept the offered snapshot" + ); + + let mut chunk_queue: VecDeque> = VecDeque::from([snapshot.hash.clone()]); + + while let Some(chunk_id) = chunk_queue.pop_front() { + let chunk = source_app + .load_snapshot_chunk(proto::RequestLoadSnapshotChunk { + height: snapshot.height, + version: snapshot.version, + chunk_id: chunk_id.clone(), + })? + .chunk; + + if tamper_next { + tamper_next = false; + let mut tampered = chunk.clone(); + let last = tampered.len() - 1; + tampered[last] ^= 0xff; + + let response = + target_app.apply_snapshot_chunk(proto::RequestApplySnapshotChunk { + chunk_id: chunk_id.clone(), + chunk: tampered, + sender: "malicious-peer".to_string(), + })?; + assert_eq!( + response.result, + i32::from(response_apply_snapshot_chunk::Result::Retry), + "a tampered chunk must be answered with a retry, not kill the session" + ); + assert_eq!( + response.refetch_chunks, + vec![chunk_id.clone()], + "the tampered chunk must be refetched" + ); + assert_eq!(response.reject_senders, vec!["malicious-peer".to_string()]); + assert!( + target_app + .snapshot_fetching_session + .read() + .unwrap() + .is_some(), + "the session must survive a tampered chunk" + ); + } + + let response = + target_app.apply_snapshot_chunk(proto::RequestApplySnapshotChunk { + chunk_id, + chunk, + sender: "honest-peer".to_string(), + })?; + + match response.result { + result + if result == i32::from(response_apply_snapshot_chunk::Result::Accept) => + { + chunk_queue.extend(response.next_chunks); + } + result + if result + == i32::from( + response_apply_snapshot_chunk::Result::CompleteSnapshot, + ) => + { + assert!( + chunk_queue.is_empty(), + "transfer completed with chunks still queued" + ); + return Ok(SnapshotSyncOutcome::Completed); + } + result + if result + == i32::from(response_apply_snapshot_chunk::Result::RejectSnapshot) => + { + // The target restored the snapshot, found it unusable and wiped + // itself back to a clean slate. Tenderdash would try the next + // snapshot; there is nothing more for this driver to do. + assert!( + target_app + .snapshot_fetching_session + .read() + .unwrap() + .is_none(), + "a rejected snapshot must not leave a session open" + ); + return Ok(SnapshotSyncOutcome::Rejected); + } + result + if result + == i32::from(response_apply_snapshot_chunk::Result::RetrySnapshot) => + { + restarts += 1; + assert!(restarts <= 2, "too many snapshot restarts"); + continue 'snapshot_attempt; + } + other => panic!("unexpected apply_snapshot_chunk result {}", other), + } + } + + panic!("chunk transfer ran out of chunks without completing"); + } + } + + struct SourceChain<'a> { + source_app: FullAbciApplication<'a, MockCoreRPCLike>, + proposers: Vec, + validator_quorums: BTreeMap, + snapshot: proto::Snapshot, + } + + /// Runs the source chain past several checkpoints and picks its newest offered + /// snapshot. + async fn run_source_chain<'a>( + source_platform: &'a mut drive_abci::test::helpers::setup::TempPlatform, + config: &PlatformConfig, + ) -> SourceChain<'a> { + let ChainExecutionOutcome { + abci_app: source_app, + proposers, + validator_quorums, + .. + } = run_chain_for_strategy( + source_platform, + 15, + state_sync_network_strategy(), + config.clone(), + 15, + &mut None, + &mut None, + ) + .await; + + let snapshots = source_app + .list_snapshots(Default::default()) + .expect("source should list snapshots") + .snapshots; + assert!( + !snapshots.is_empty(), + "the source chain must have produced at least one restorable snapshot" + ); + let snapshot = snapshots + .iter() + .max_by_key(|snapshot| snapshot.height) + .expect("at least one snapshot") + .clone(); + + SourceChain { + source_app, + proposers: proposers + .iter() + .map(|proposer| proposer.masternode.clone()) + .collect(), + validator_quorums, + snapshot, + } + } + + /// End to end: run a source chain past several checkpoints, serve its newest + /// snapshot, restore it chunk by chunk on a fresh target (with one tampered chunk + /// along the way to prove refetch/restart recovery), reconstruct the target + /// platform state, and verify the target matches the source checkpoint exactly. + // QA BRANCH: un-ignored because the workspace root Cargo.toml carries a TEMPORARY + // patch redirecting grovedb to the sum-tree restore fix (dashpay/grovedb#840). Restore + // the `#[ignore]` if that patch is dropped without bumping the real grovedb pin. + #[tokio::test] + async fn run_state_sync_between_two_platforms() { + let config = state_sync_platform_config(); + let mut source_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let source = run_source_chain(&mut source_platform, &config).await; + let snapshot = &source.snapshot; + + // The platform state the source had at exactly the snapshot height + let source_platform_state = source + .source_app + .platform + .checkpoint_platform_states + .load() + .get(&snapshot.height) + .expect("source must cache the platform state of its checkpoint") + .clone(); + + // A fresh target node, knowing nothing but Core RPC + let mut target_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + install_reconstruction_core_mocks( + &mut target_platform.platform, + source.proposers.clone(), + &source.validator_quorums, + ); + let target_app = FullAbciApplication::new(&target_platform); + + assert_eq!( + sync_snapshot(&source.source_app, &target_app, snapshot, true) + .expect("state sync must not error"), + SnapshotSyncOutcome::Completed, + "state sync must complete" + ); + + let platform_version = PlatformVersion::latest(); + let grove_version = &platform_version.drive.grove_version; + + // Grove roots agree between source checkpoint and target + let target_root_hash = target_platform + .drive + .grove + .root_hash(None, grove_version) + .unwrap() + .expect("target root hash"); + assert_eq!(target_root_hash.to_vec(), snapshot.hash); + + // The restored grovedb is internally consistent + let verification_issues = target_platform + .drive + .grove + .verify_grovedb(None, true, false, grove_version) + .expect("expected to verify grovedb"); + assert!( + verification_issues.is_empty(), + "restored grovedb must verify cleanly: {:?}", + verification_issues + ); + + // The reconstructed platform state matches the source's state at the snapshot + // height, except for the fields that are not replicated (block signature and + // block id hash restore as zeroes) + let target_state = target_platform.state.load(); + assert_eq!( + target_state.current_protocol_version_in_consensus(), + source_platform_state.current_protocol_version_in_consensus() + ); + assert_eq!( + target_state.next_epoch_protocol_version(), + source_platform_state.next_epoch_protocol_version() + ); + assert_eq!( + target_state.last_committed_block_height(), + snapshot.height, + "target must be at the snapshot height" + ); + assert_eq!( + target_state.last_committed_block_app_hash(), + source_platform_state.last_committed_block_app_hash() + ); + assert_eq!( + target_state.current_validator_set_quorum_hash(), + source_platform_state.current_validator_set_quorum_hash() + ); + assert_eq!( + target_state.next_validator_set_quorum_hash(), + source_platform_state.next_validator_set_quorum_hash() + ); + assert_eq!( + target_state.validator_sets().keys().collect::>(), + source_platform_state + .validator_sets() + .keys() + .collect::>(), + "validator set order must be restored from the recorded quorum positions" + ); + assert_eq!( + target_state.validator_sets(), + source_platform_state.validator_sets(), + "validator sets must match" + ); + assert_eq!( + target_state.full_masternode_list(), + source_platform_state.full_masternode_list() + ); + assert_eq!( + target_state.hpmn_masternode_list(), + source_platform_state.hpmn_masternode_list() + ); + assert_eq!( + target_state.previous_fee_versions(), + source_platform_state.previous_fee_versions(), + "fee versions of previous epochs must be restored faithfully" + ); + + // The target's info handler must pass its own app-hash consistency check and + // report the snapshot height and hash to Tenderdash's post-sync verifyApp + let info = target_app + .info(proto::RequestInfo { + version: tenderdash_abci::proto::meta::TENDERDASH_VERSION.to_string(), + block_version: 0, + p2p_version: 0, + abci_version: tenderdash_abci::proto::meta::ABCI_VERSION.to_string(), + }) + .expect("target info handler must succeed"); + assert_eq!(info.last_block_height as u64, snapshot.height); + assert_eq!(info.last_block_app_hash, snapshot.hash); + } + + /// Pins today's behavior at the pinned grovedb revision: the transfer itself + /// completes (including recovery from a tampered chunk via RETRY and a snapshot + /// restart), but the strict post-restore verification detects that grovedb did not + /// faithfully restore the sum trees and refuses the snapshot instead of accepting + /// latent corruption. When this test starts failing because the sync SUCCEEDS, + /// grovedb has been fixed: un-ignore `run_state_sync_between_two_platforms` and + /// drop this pin. + // QA BRANCH ONLY — this test asserts the grovedb sum-tree restore DEFECT is present. + // The workspace root Cargo.toml carries a TEMPORARY patch redirecting grovedb to the + // fix (dashpay/grovedb#840), so the sync now SUCCEEDS and this defect-present pin + // fails by design. Un-ignore (and delete it, as its own doc comment instructs) in the + // same change that drops the root Cargo.toml patch section. + #[tokio::test] + #[ignore = "QA branch: the root Cargo.toml patch to dashpay/grovedb#840 fixes this defect, \ + so this defect-present pin fails by design — see the comment above"] + async fn state_sync_transfer_detects_sum_tree_restore_defect() { + let config = state_sync_platform_config(); + let mut source_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let source = run_source_chain(&mut source_platform, &config).await; + + let mut target_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + install_reconstruction_core_mocks( + &mut target_platform.platform, + source.proposers.clone(), + &source.validator_quorums, + ); + let target_app = FullAbciApplication::new(&target_platform); + + let outcome = sync_snapshot(&source.source_app, &target_app, &source.snapshot, true) + .expect("a refused snapshot is answered, not errored"); + assert_eq!( + outcome, + SnapshotSyncOutcome::Rejected, + "at grovedb rev 6c882c3 the restored sum trees must fail verification — if this \ + now completes, grovedb is fixed: un-ignore run_state_sync_between_two_platforms \ + and remove this pin" + ); + + // The target refused the snapshot: it never advanced past genesis, and — since the + // refusal happens after the session was already committed — it wiped itself back to + // a clean slate rather than keeping the unusable state. + assert_eq!( + target_platform.state.load().last_committed_block_height(), + 0 + ); + assert_ne!( + target_platform + .drive + .grove + .root_hash(None, &PlatformVersion::latest().drive.grove_version) + .unwrap() + .expect("target root hash") + .to_vec(), + source.snapshot.hash, + "a refused snapshot must not be left on disk" + ); + } + + /// Exercises the platform state reconstruction end to end without going through + /// the (currently defective, see above) grovedb chunk restore: the source chain's + /// own grovedb IS a faithfully "restored" snapshot of itself, so reconstructing + /// on it must (a) not change the grovedb root hash — the proof that re-deriving + /// masternode identities from Core is byte-idempotent — and (b) reproduce the + /// source's in-memory platform state from the reduced platform state alone. + #[tokio::test] + async fn platform_state_reconstruction_is_idempotent_and_matches_source_state() { + let config = state_sync_platform_config(); + let mut source_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let source = run_source_chain(&mut source_platform, &config).await; + let platform = source.source_app.platform; + + let platform_version = PlatformVersion::latest(); + let grove_version = &platform_version.drive.grove_version; + + let original_state = platform.state.load().clone(); + let tip_app_hash = platform + .drive + .grove + .root_hash(None, grove_version) + .unwrap() + .expect("source root hash"); + assert_eq!( + original_state.last_committed_block_app_hash(), + Some(tip_app_hash), + "sanity: chain tip state matches grove root" + ); + + // The run_chain mocks already answer the from-scratch masternode/quorum + // requests reconstruction makes, exactly as they did for the chain itself. + platform + .reconstruct_platform_state(&tip_app_hash, platform_version) + .expect("platform state reconstruction must succeed"); + + // (a) idempotence: re-deriving masternode identities wrote nothing new + let root_hash_after = platform + .drive + .grove + .root_hash(None, grove_version) + .unwrap() + .expect("source root hash after reconstruction"); + assert_eq!( + root_hash_after, tip_app_hash, + "reconstruction must not change the grovedb root hash" + ); + + // (b) the reconstructed state matches the original, except the fields the + // reduced state cannot carry (block signature / block id hash) + let reconstructed_state = platform.state.load(); + assert_eq!( + reconstructed_state.current_protocol_version_in_consensus(), + original_state.current_protocol_version_in_consensus() + ); + assert_eq!( + reconstructed_state.next_epoch_protocol_version(), + original_state.next_epoch_protocol_version() + ); + assert_eq!( + reconstructed_state.last_committed_block_height(), + original_state.last_committed_block_height() + ); + assert_eq!( + reconstructed_state.last_committed_block_app_hash(), + original_state.last_committed_block_app_hash() + ); + assert_eq!( + reconstructed_state.last_committed_core_height(), + original_state.last_committed_core_height() + ); + assert_eq!( + reconstructed_state.current_validator_set_quorum_hash(), + original_state.current_validator_set_quorum_hash() + ); + assert_eq!( + reconstructed_state.next_validator_set_quorum_hash(), + original_state.next_validator_set_quorum_hash() + ); + assert_eq!( + reconstructed_state + .validator_sets() + .keys() + .collect::>(), + original_state.validator_sets().keys().collect::>(), + "validator set order must be restored from the recorded quorum positions" + ); + assert_eq!( + reconstructed_state.validator_sets(), + original_state.validator_sets() + ); + assert_eq!( + reconstructed_state.full_masternode_list(), + original_state.full_masternode_list() + ); + assert_eq!( + reconstructed_state.hpmn_masternode_list(), + original_state.hpmn_masternode_list() + ); + assert_eq!( + reconstructed_state.previous_fee_versions(), + original_state.previous_fee_versions() + ); + + // The info handler accepts the reconstructed state (it panics on an app-hash + // mismatch between the in-memory state and the grove root) + let info = source + .source_app + .info(proto::RequestInfo { + version: tenderdash_abci::proto::meta::TENDERDASH_VERSION.to_string(), + block_version: 0, + p2p_version: 0, + abci_version: tenderdash_abci::proto::meta::ABCI_VERSION.to_string(), + }) + .expect("info handler must accept the reconstructed state"); + assert_eq!( + info.last_block_height as u64, + original_state.last_committed_block_height() + ); + assert_eq!(info.last_block_app_hash, tip_app_hash.to_vec()); + } + + /// A snapshot from a chain that never wrote the reduced platform state (pre-v15) + /// is not offered by the source, and a target driven at it anyway refuses to + /// restore it. + #[tokio::test] + async fn pre_v15_snapshot_is_not_served_and_cannot_be_restored() { + let config = state_sync_platform_config(); + + let mut source_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .with_initial_protocol_version(14) + .build_with_mock_rpc(); + + let ChainExecutionOutcome { + abci_app: source_app, + proposers, + validator_quorums, + .. + } = run_chain_for_strategy( + &mut source_platform, + 15, + state_sync_network_strategy(), + config.clone(), + 15, + &mut None, + &mut None, + ) + .await; + + // The v14 chain created checkpoints, but none carries the reduced platform + // state, so none may be offered. + assert!( + !source_app.platform.drive.checkpoints.load().is_empty(), + "the source must have created checkpoints" + ); + let snapshots = source_app + .list_snapshots(Default::default()) + .expect("source should list snapshots") + .snapshots; + assert!( + snapshots.is_empty(), + "pre-v15 checkpoints are unrestorable and must not be offered" + ); + + // Even if a peer maliciously offers such a snapshot, the target must refuse to + // restore it. (At the current grovedb revision the refusal comes from the + // post-restore verification; once grovedb faithfully restores sum trees it + // comes from the missing reduced platform state at the reconstruction step. + // Either way the snapshot must not be accepted.) + let (height, checkpoint) = { + let checkpoints = source_app.platform.drive.checkpoints.load(); + let (height, info) = checkpoints + .last_key_value() + .expect("at least one checkpoint"); + (*height, std::sync::Arc::clone(&info.checkpoint)) + }; + let platform_version = PlatformVersion::latest(); + let checkpoint_root = checkpoint + .grove_db + .root_hash(None, &platform_version.drive.grove_version) + .unwrap() + .expect("checkpoint root hash"); + let forged_snapshot = proto::Snapshot { + height, + version: 1, + hash: checkpoint_root.to_vec(), + metadata: vec![], + }; + + let mut target_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + install_reconstruction_core_mocks( + &mut target_platform.platform, + proposers + .iter() + .map(|proposer| proposer.masternode.clone()) + .collect(), + &validator_quorums, + ); + let target_app = FullAbciApplication::new(&target_platform); + + let outcome = sync_snapshot(&source_app, &target_app, &forged_snapshot, false) + .expect("a refused snapshot is answered, not errored"); + assert_eq!( + outcome, + SnapshotSyncOutcome::Rejected, + "a snapshot without the reduced platform state must be refused" + ); + + // The target holds no usable platform state: it never advanced past genesis + assert_eq!( + target_platform.state.load().last_committed_block_height(), + 0 + ); + // ...and it did not keep the state it could not use: the refusal wipes back to a + // clean slate so Tenderdash can offer another snapshot or fall back to block sync. + assert_ne!( + target_platform + .drive + .grove + .root_hash(None, &platform_version.drive.grove_version) + .unwrap() + .expect("target root hash") + .to_vec(), + forged_snapshot.hash, + "a refused snapshot must not be left on disk" + ); + } +} diff --git a/packages/rs-drive-abci/tests/sum_tree_sync_probe.rs b/packages/rs-drive-abci/tests/sum_tree_sync_probe.rs new file mode 100644 index 00000000000..9db5d67992b --- /dev/null +++ b/packages/rs-drive-abci/tests/sum_tree_sync_probe.rs @@ -0,0 +1,135 @@ +//! Minimal reproducer / tripwire for a grovedb state sync limitation at the pinned +//! revision (6c882c3): the replication protocol at this revision does not faithfully +//! restore SumTree subtrees. The chunk transfer copies the source's node hashes, so the restored +//! database reproduces the source ROOT hash — but re-opening the restored sum tree +//! and recomputing its root yields a different hash, i.e. the corruption is latent +//! and `verify_grovedb` detects it. +//! +//! This is why `apply_snapshot_chunk` runs the strict `verify_grovedb` check after +//! committing a state sync session, and why the full two-instance state sync +//! integration test (`run_state_sync_between_two_platforms`) is `#[ignore]`d. +//! +//! WHEN THIS TEST STARTS FAILING because no verification issues are reported, the +//! grovedb pin has been fixed: delete this tripwire and un-ignore the full +//! integration test. + +use drive::grovedb::{Element, GroveDb}; +use drive::grovedb_path::SubtreePath; +use platform_version::version::PlatformVersion; +use std::collections::VecDeque; + +#[test] +// QA BRANCH ONLY — paired with the TEMPORARY `[patch."https://github.com/dashpay/grovedb"]` +// section in the workspace root Cargo.toml, which redirects grovedb to the fix in +// dashpay/grovedb#840. This tripwire asserts the DEFECT is present, so with the patch +// applied it fails by design (it reports zero verification issues instead of the sum +// tree). Ignoring it keeps the QA suite green while the fix is patched in. +// +// UN-IGNORE THIS (and delete it, per the module docs) in the same change that drops the +// root Cargo.toml patch section — i.e. when the real grovedb pin is bumped. +#[ignore = "QA branch: the root Cargo.toml patch to dashpay/grovedb#840 fixes this defect, \ + so this defect-present tripwire fails by design — see the comment above"] +fn sum_tree_state_sync_restore_is_latently_corrupt_at_pinned_grovedb() { + let grove_version = &PlatformVersion::latest().drive.grove_version; + let source_dir = tempfile::tempdir().unwrap(); + let source = GroveDb::open(source_dir.path()).unwrap(); + + let root: SubtreePath<[u8; 0]> = SubtreePath::empty(); + + source + .insert( + root.clone(), + b"s", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .unwrap(); + let sum_path: &[&[u8]] = &[b"s"]; + for (key, value) in [(b"a", 5i64), (b"b", 7i64)] { + source + .insert( + sum_path, + key, + Element::new_sum_item(value), + None, + None, + grove_version, + ) + .unwrap() + .unwrap(); + } + // A normal tree with an item, for contrast: it restores cleanly. + source + .insert( + root.clone(), + b"n", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .unwrap(); + let normal_path: &[&[u8]] = &[b"n"]; + source + .insert( + normal_path, + b"k", + Element::new_item(b"v".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .unwrap(); + + let app_hash = source.root_hash(None, grove_version).unwrap().unwrap(); + + let target_dir = tempfile::tempdir().unwrap(); + let target = GroveDb::open(target_dir.path()).unwrap(); + let mut session = target + .start_snapshot_syncing(app_hash, 64, 1, grove_version) + .unwrap(); + + let mut queue: VecDeque> = VecDeque::from([app_hash.to_vec()]); + while let Some(chunk_id) = queue.pop_front() { + let chunk = source + .fetch_chunk(&chunk_id, None, 1, grove_version) + .unwrap(); + let next = session + .apply_chunk(&chunk_id, &chunk, 1, grove_version) + .unwrap(); + queue.extend(next); + if session.is_sync_completed() { + break; + } + } + assert!(session.is_sync_completed()); + target.commit_session(session, grove_version).unwrap(); + + // The copied node hashes reproduce the source root hash exactly... + let target_root = target.root_hash(None, grove_version).unwrap().unwrap(); + assert_eq!( + target_root, app_hash, + "restored root hash must match the source" + ); + + // ...but recomputing the restored sum tree exposes the latent corruption. + let issues = target + .verify_grovedb(None, true, false, grove_version) + .unwrap(); + let paths: Vec = issues + .keys() + .map(|path| path.iter().map(hex::encode).collect::>().join("/")) + .collect(); + assert_eq!( + paths, + vec!["73".to_string()], // hex of b"s", the sum tree + "expected exactly the sum tree to fail verification — if no issues are \ + reported, grovedb has been fixed: delete this tripwire and un-ignore \ + run_state_sync_between_two_platforms" + ); +} diff --git a/packages/rs-drive/src/drive/mod.rs b/packages/rs-drive/src/drive/mod.rs index 98eeafa0c24..367ad56a397 100644 --- a/packages/rs-drive/src/drive/mod.rs +++ b/packages/rs-drive/src/drive/mod.rs @@ -106,6 +106,26 @@ impl Checkpoint { } } + /// Returns true if this checkpoint contains the reduced platform state + /// (`Misc/reduced_saved_state`), which a state-syncing node needs to reconstruct the + /// platform state. Checkpoints taken before the protocol version that introduced the + /// reduced state lack the key and cannot be offered as state sync snapshots. + pub fn has_reduced_platform_state( + &self, + grove_version: &grovedb_version::version::GroveVersion, + ) -> Result { + self.grove_db + .get_raw_optional( + (&crate::drive::system::misc_path()).into(), + crate::drive::platform_state::REDUCED_PLATFORM_STATE_KEY, + None, + grove_version, + ) + .unwrap() + .map(|maybe_element| maybe_element.is_some()) + .map_err(Error::from) + } + /// Marks this checkpoint for deletion when it is dropped. pub fn mark_for_deletion(&self) { self.marked_for_deletion diff --git a/packages/rs-drive/src/drive/platform_state/fetch_reduced_platform_state_bytes/mod.rs b/packages/rs-drive/src/drive/platform_state/fetch_reduced_platform_state_bytes/mod.rs new file mode 100644 index 00000000000..2045c905235 --- /dev/null +++ b/packages/rs-drive/src/drive/platform_state/fetch_reduced_platform_state_bytes/mod.rs @@ -0,0 +1,33 @@ +mod v0; + +use crate::drive::Drive; +use crate::error::drive::DriveError; +use crate::error::Error; +use dpp::version::PlatformVersion; +use grovedb::TransactionArg; + +impl Drive { + /// Fetch the reduced platform state from the replicated grovedb state (Misc tree). + /// + /// Returns `Ok(None)` when the key is absent (for example before the protocol + /// version that introduced the reduced state activated). + pub fn fetch_reduced_platform_state_bytes( + &self, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result>, Error> { + match platform_version + .drive + .methods + .platform_state + .fetch_reduced_platform_state_bytes + { + 0 => self.fetch_reduced_platform_state_bytes_v0(transaction, platform_version), + version => Err(Error::Drive(DriveError::UnknownVersionMismatch { + method: "fetch_reduced_platform_state_bytes".to_string(), + known_versions: vec![0], + received: version, + })), + } + } +} diff --git a/packages/rs-drive/src/drive/platform_state/fetch_reduced_platform_state_bytes/v0/mod.rs b/packages/rs-drive/src/drive/platform_state/fetch_reduced_platform_state_bytes/v0/mod.rs new file mode 100644 index 00000000000..7df67f7c084 --- /dev/null +++ b/packages/rs-drive/src/drive/platform_state/fetch_reduced_platform_state_bytes/v0/mod.rs @@ -0,0 +1,24 @@ +use crate::drive::platform_state::REDUCED_PLATFORM_STATE_KEY; +use crate::drive::system::misc_path; +use crate::drive::Drive; +use crate::error::Error; +use crate::util::grove_operations::DirectQueryType; +use grovedb::TransactionArg; +use platform_version::version::PlatformVersion; + +impl Drive { + pub(super) fn fetch_reduced_platform_state_bytes_v0( + &self, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result>, Error> { + self.grove_get_raw_optional_item( + (&misc_path()).into(), + REDUCED_PLATFORM_STATE_KEY, + DirectQueryType::StatefulDirectQuery, + transaction, + &mut vec![], + &platform_version.drive, + ) + } +} diff --git a/packages/rs-drive/src/drive/platform_state/mod.rs b/packages/rs-drive/src/drive/platform_state/mod.rs index d6a0ce16c49..9b100fc239d 100644 --- a/packages/rs-drive/src/drive/platform_state/mod.rs +++ b/packages/rs-drive/src/drive/platform_state/mod.rs @@ -1,4 +1,64 @@ mod fetch_platform_state_bytes; +mod fetch_reduced_platform_state_bytes; mod store_platform_state_bytes; +mod store_reduced_platform_state_bytes; const PLATFORM_STATE_KEY: &[u8; 11] = b"saved_state"; +pub(crate) const REDUCED_PLATFORM_STATE_KEY: &[u8; 19] = b"reduced_saved_state"; + +#[cfg(test)] +mod tests { + use crate::util::test_helpers::setup::setup_drive_with_initial_state_structure; + use platform_version::version::PlatformVersion; + + #[test] + fn should_return_none_when_reduced_platform_state_is_absent() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + + let fetched = drive + .fetch_reduced_platform_state_bytes(None, platform_version) + .expect("fetching an absent reduced platform state should not error"); + + assert_eq!(fetched, None); + } + + #[test] + fn should_roundtrip_reduced_platform_state_bytes() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + + let state_bytes = vec![1u8, 2, 3, 4, 5]; + + drive + .store_reduced_platform_state_bytes(&state_bytes, None, platform_version) + .expect("should store reduced platform state"); + + let fetched = drive + .fetch_reduced_platform_state_bytes(None, platform_version) + .expect("should fetch reduced platform state"); + + assert_eq!(fetched, Some(state_bytes)); + } + + #[test] + fn should_overwrite_reduced_platform_state_bytes() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + + drive + .store_reduced_platform_state_bytes(&[1u8, 2, 3], None, platform_version) + .expect("should store reduced platform state"); + + let updated_bytes = vec![9u8, 8, 7]; + drive + .store_reduced_platform_state_bytes(&updated_bytes, None, platform_version) + .expect("should overwrite reduced platform state"); + + let fetched = drive + .fetch_reduced_platform_state_bytes(None, platform_version) + .expect("should fetch reduced platform state"); + + assert_eq!(fetched, Some(updated_bytes)); + } +} diff --git a/packages/rs-drive/src/drive/platform_state/store_reduced_platform_state_bytes/mod.rs b/packages/rs-drive/src/drive/platform_state/store_reduced_platform_state_bytes/mod.rs new file mode 100644 index 00000000000..0346e197218 --- /dev/null +++ b/packages/rs-drive/src/drive/platform_state/store_reduced_platform_state_bytes/mod.rs @@ -0,0 +1,35 @@ +mod v0; + +use crate::drive::Drive; +use crate::error::drive::DriveError; +use crate::error::Error; +use dpp::version::PlatformVersion; +use grovedb::TransactionArg; + +impl Drive { + /// Store the reduced platform state in the replicated grovedb state (Misc tree) + pub fn store_reduced_platform_state_bytes( + &self, + state_bytes: &[u8], + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + match platform_version + .drive + .methods + .platform_state + .store_reduced_platform_state_bytes + { + 0 => self.store_reduced_platform_state_bytes_v0( + state_bytes, + transaction, + platform_version, + ), + version => Err(Error::Drive(DriveError::UnknownVersionMismatch { + method: "store_reduced_platform_state_bytes".to_string(), + known_versions: vec![0], + received: version, + })), + } + } +} diff --git a/packages/rs-drive/src/drive/platform_state/store_reduced_platform_state_bytes/v0/mod.rs b/packages/rs-drive/src/drive/platform_state/store_reduced_platform_state_bytes/v0/mod.rs new file mode 100644 index 00000000000..55d61b2e08a --- /dev/null +++ b/packages/rs-drive/src/drive/platform_state/store_reduced_platform_state_bytes/v0/mod.rs @@ -0,0 +1,28 @@ +use crate::drive::platform_state::REDUCED_PLATFORM_STATE_KEY; +use crate::drive::system::misc_path; +use crate::drive::Drive; +use crate::error::Error; +use grovedb::{Element, TransactionArg}; +use platform_version::version::PlatformVersion; + +impl Drive { + pub(super) fn store_reduced_platform_state_bytes_v0( + &self, + reduced_state_bytes: &[u8], + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + self.grove + .insert( + &misc_path(), + REDUCED_PLATFORM_STATE_KEY, + Element::Item(reduced_state_bytes.to_vec(), None), + None, + transaction, + &platform_version.drive.grove_version, + ) + .unwrap() + .map_err(Error::from)?; + Ok(()) + } +} diff --git a/packages/rs-drive/src/error/drive.rs b/packages/rs-drive/src/error/drive.rs index 3412c9df1aa..a64d5c94adb 100644 --- a/packages/rs-drive/src/error/drive.rs +++ b/packages/rs-drive/src/error/drive.rs @@ -218,4 +218,8 @@ pub enum DriveError { /// Checkpoint not found for specified block height #[error("checkpoint not found for block height: {0}")] CheckpointNotFound(u64), + + /// Snapshot error + #[error("snapshot error: {0}")] + Snapshot(String), } diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/mod.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/mod.rs index a1bf5fdd754..cfd94cf9969 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/mod.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/mod.rs @@ -2,6 +2,7 @@ use versioned_feature_core::{FeatureVersion, OptionalFeatureVersion}; pub mod v1; pub mod v10; +pub mod v11; pub mod v2; pub mod v3; pub mod v4; @@ -36,6 +37,8 @@ pub struct DriveAbciMethodVersions { pub struct DriveAbciPlatformStateStorageMethodVersions { pub fetch_platform_state: FeatureVersion, pub store_platform_state: FeatureVersion, + pub fetch_reduced_platform_state: FeatureVersion, + pub store_reduced_platform_state: FeatureVersion, } #[derive(Clone, Copy, Debug, Default)] diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v1.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v1.rs index 9798d693037..b03335c73d7 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v1.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v1.rs @@ -131,5 +131,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V1: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v10.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v10.rs index 38be834a186..a88ace56cab 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v10.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v10.rs @@ -140,5 +140,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V10: DriveAbciMethodVersions = DriveAbciMet platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v11.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v11.rs new file mode 100644 index 00000000000..8ea89d7169d --- /dev/null +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v11.rs @@ -0,0 +1,146 @@ +use crate::version::drive_abci_versions::drive_abci_method_versions::{ + DriveAbciBlockEndMethodVersions, DriveAbciBlockFeeProcessingMethodVersions, + DriveAbciBlockStartMethodVersions, DriveAbciCoreBasedUpdatesMethodVersions, + DriveAbciCoreChainLockMethodVersionsAndConstants, DriveAbciCoreInstantSendLockMethodVersions, + DriveAbciEngineMethodVersions, DriveAbciEpochMethodVersions, + DriveAbciFeePoolInwardsDistributionMethodVersions, + DriveAbciFeePoolOutwardsDistributionMethodVersions, + DriveAbciIdentityCreditWithdrawalMethodVersions, DriveAbciInitializationMethodVersions, + DriveAbciMasternodeIdentitiesUpdatesMethodVersions, DriveAbciMethodVersions, + DriveAbciPlatformStateStorageMethodVersions, DriveAbciProtocolUpgradeMethodVersions, + DriveAbciStateTransitionProcessingMethodVersions, DriveAbciTokensProcessingMethodVersions, + DriveAbciVotingMethodVersions, +}; + +/// Drive ABCI method versions 11. Introduced in protocol v15 for state sync: +/// `run_block_proposal` 0 -> 1 (the reduced platform state is written into the replicated +/// state each block, and `validator_set_update` moves above the root-hash computation so the +/// stored reduced state is sufficient to reconstruct the post-rotation state), and +/// `consensus_params_update` 1 -> 2 (emits evidence params when crossing to v15). +/// Everything else matches `DRIVE_ABCI_METHOD_VERSIONS_V10`. +pub const DRIVE_ABCI_METHOD_VERSIONS_V11: DriveAbciMethodVersions = DriveAbciMethodVersions { + engine: DriveAbciEngineMethodVersions { + init_chain: 0, + check_tx: 0, + run_block_proposal: 1, + finalize_block_proposal: 0, + consensus_params_update: 2, + }, + initialization: DriveAbciInitializationMethodVersions { + initial_core_height_and_time: 0, + create_genesis_state: 1, + }, + core_based_updates: DriveAbciCoreBasedUpdatesMethodVersions { + update_core_info: 0, + update_masternode_list: 0, + update_quorum_info: 0, + masternode_updates: DriveAbciMasternodeIdentitiesUpdatesMethodVersions { + get_voter_identity_key: 0, + get_operator_identity_keys: 0, + get_owner_identity_withdrawal_key: 0, + get_owner_identity_owner_key: 0, + get_voter_identifier_from_masternode_list_item: 0, + get_operator_identifier_from_masternode_list_item: 0, + create_operator_identity: 0, + create_owner_identity: 1, + create_voter_identity: 0, + disable_identity_keys: 0, + update_masternode_identities: 0, + update_operator_identity: 0, + update_owner_withdrawal_address: 1, + update_voter_identity: 0, + }, + }, + protocol_upgrade: DriveAbciProtocolUpgradeMethodVersions { + check_for_desired_protocol_upgrade: 1, + upgrade_protocol_version_on_epoch_change: 0, + perform_events_on_first_block_of_protocol_change: Some(1), + protocol_version_upgrade_percentage_needed: 67, + }, + block_fee_processing: DriveAbciBlockFeeProcessingMethodVersions { + add_process_epoch_change_operations: 0, + process_block_fees_and_validate_sum_trees: 1, + }, + tokens_processing: DriveAbciTokensProcessingMethodVersions { + validate_token_aggregated_balance: 0, + }, + core_chain_lock: DriveAbciCoreChainLockMethodVersionsAndConstants { + choose_quorum: 0, + verify_chain_lock: 0, + verify_chain_lock_locally: 0, + verify_chain_lock_through_core: 0, + make_sure_core_is_synced_to_chain_lock: 0, + recent_block_count_amount: 2, + }, + core_instant_send_lock: DriveAbciCoreInstantSendLockMethodVersions { + verify_recent_signature_locally: 0, + }, + fee_pool_inwards_distribution: DriveAbciFeePoolInwardsDistributionMethodVersions { + add_distribute_block_fees_into_pools_operations: 0, + add_distribute_storage_fee_to_epochs_operations: 0, + }, + fee_pool_outwards_distribution: DriveAbciFeePoolOutwardsDistributionMethodVersions { + add_distribute_fees_from_oldest_unpaid_epoch_pool_to_proposers_operations: 1, + add_epoch_pool_to_proposers_payout_operations: 0, + find_oldest_epoch_needing_payment: 0, + fetch_reward_shares_list_for_masternode: 0, + }, + withdrawals: DriveAbciIdentityCreditWithdrawalMethodVersions { + build_untied_withdrawal_transactions_from_documents: 0, + dequeue_and_build_unsigned_withdrawal_transactions: 0, + fetch_transactions_block_inclusion_status: 0, + pool_withdrawals_into_transactions_queue: 1, + update_broadcasted_withdrawal_statuses: 0, + rebroadcast_expired_withdrawal_documents: 1, + append_signatures_and_broadcast_withdrawal_transactions: 0, + cleanup_expired_locks_of_withdrawal_amounts: 1, // changed in v14: also prunes expired entries of the credit inflows sum tree + record_credit_inflows_for_withdrawals: Some(0), // new in v14: the block's credit mints recorded as an inflow for the net daily withdrawal limit + record_total_credits_history_for_withdrawals: Some(0), // changed in v14: per-block total credits history for the day-lagged daily withdrawal limit + }, + voting: DriveAbciVotingMethodVersions { + keep_record_of_finished_contested_resource_vote_poll: 0, + clean_up_after_vote_poll_end: 0, + clean_up_after_contested_resources_vote_poll_end: 1, + check_for_ended_vote_polls: 0, + tally_votes_for_contested_document_resource_vote_poll: 0, + award_document_to_winner: 0, + delay_vote_poll: 0, + run_dao_platform_events: 0, + remove_votes_for_removed_masternodes: 0, + }, + state_transition_processing: DriveAbciStateTransitionProcessingMethodVersions { + execute_event: 0, + process_raw_state_transitions: 0, + // unchanged from V9: v1 since v13 (records the balance effects of paid-INVALID / + // unsuccessful-paid transitions) + process_validation_result: 1, + decode_raw_state_transitions: 0, + validate_fees_of_event: 0, + store_address_balances_to_recent_block_storage: Some(0), + cleanup_recent_block_storage_address_balances: Some(0), + // unchanged from V9: v1 since v13 (records shielded-spend transparent credits) + record_added_balance_outputs: 1, + }, + epoch: DriveAbciEpochMethodVersions { + gather_epoch_info: 0, + get_genesis_time: 0, + }, + block_start: DriveAbciBlockStartMethodVersions { + clear_drive_block_cache: 0, + }, + block_end: DriveAbciBlockEndMethodVersions { + update_state_cache: 0, + update_drive_cache: 0, + validator_set_update: 2, + should_checkpoint: Some(0), + update_checkpoints: Some(0), + record_shielded_pool_anchor: Some(0), + prune_shielded_pool_anchors: Some(0), + }, + platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { + fetch_platform_state: 0, + store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, + }, +}; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v2.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v2.rs index c3177e006f7..e781dbca981 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v2.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v2.rs @@ -132,5 +132,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V2: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v3.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v3.rs index 06fe75413e5..e9a0fb51daa 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v3.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v3.rs @@ -131,5 +131,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V3: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v4.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v4.rs index 843e71c6d40..b5c18a3c727 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v4.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v4.rs @@ -131,5 +131,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V4: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v5.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v5.rs index bed7af26bf2..84d1c011f38 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v5.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v5.rs @@ -135,5 +135,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V5: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v6.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v6.rs index df56f534d95..f696c3d7dba 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v6.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v6.rs @@ -133,5 +133,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V6: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v7.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v7.rs index da44e9b81a9..6a10f182858 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v7.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v7.rs @@ -142,5 +142,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V7: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v8.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v8.rs index d9a461cc62f..0629a7808c7 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v8.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v8.rs @@ -142,5 +142,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V8: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v9.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v9.rs index 434f216c461..f3d4d857fd0 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v9.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v9.rs @@ -160,5 +160,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V9: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/mod.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/mod.rs new file mode 100644 index 00000000000..f0158bc6ad9 --- /dev/null +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/mod.rs @@ -0,0 +1,14 @@ +pub mod v1; + +use versioned_feature_core::FeatureVersion; + +/// Versions for ABCI state sync (snapshot serving and consumption). +#[derive(Clone, Debug, Default)] +pub struct DriveAbciStateSyncVersions { + /// The grovedb state sync protocol version used for snapshots this node creates + /// and serves. Exactly one version exists (grovedb updates its replication + /// protocol in place and stays at version 1); snapshots offered by peers are + /// validated against the supported set in `drive-abci`'s snapshot module so any + /// future incompatible protocol change fails fast on both sides. + pub protocol_version: FeatureVersion, +} diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/v1.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/v1.rs new file mode 100644 index 00000000000..f5a3d13e884 --- /dev/null +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/v1.rs @@ -0,0 +1,6 @@ +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::DriveAbciStateSyncVersions; + +pub const DRIVE_ABCI_STATE_SYNC_VERSIONS_V1: DriveAbciStateSyncVersions = + DriveAbciStateSyncVersions { + protocol_version: 1, + }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/mod.rs b/packages/rs-platform-version/src/version/drive_abci_versions/mod.rs index 6df817b3dfd..9adde49dbe5 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/mod.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/mod.rs @@ -1,6 +1,7 @@ pub mod drive_abci_checkpoint_parameters; pub mod drive_abci_method_versions; pub mod drive_abci_query_versions; +pub mod drive_abci_state_sync_versions; pub mod drive_abci_structure_versions; pub mod drive_abci_validation_versions; pub mod drive_abci_withdrawal_constants; @@ -8,6 +9,7 @@ pub mod drive_abci_withdrawal_constants; use drive_abci_checkpoint_parameters::DriveAbciCheckpointParameters; use drive_abci_method_versions::DriveAbciMethodVersions; use drive_abci_query_versions::DriveAbciQueryVersions; +use drive_abci_state_sync_versions::DriveAbciStateSyncVersions; use drive_abci_structure_versions::DriveAbciStructureVersions; use drive_abci_validation_versions::DriveAbciValidationVersions; use drive_abci_withdrawal_constants::DriveAbciWithdrawalConstants; @@ -20,4 +22,5 @@ pub struct DriveAbciVersion { pub withdrawal_constants: DriveAbciWithdrawalConstants, pub query: DriveAbciQueryVersions, pub checkpoints: DriveAbciCheckpointParameters, + pub state_sync: DriveAbciStateSyncVersions, } diff --git a/packages/rs-platform-version/src/version/drive_versions/mod.rs b/packages/rs-platform-version/src/version/drive_versions/mod.rs index ca7c22c6e3f..c768c5a36a1 100644 --- a/packages/rs-platform-version/src/version/drive_versions/mod.rs +++ b/packages/rs-platform-version/src/version/drive_versions/mod.rs @@ -78,6 +78,8 @@ pub struct DriveMethodVersions { pub struct DrivePlatformStateMethodVersions { pub fetch_platform_state_bytes: FeatureVersion, pub store_platform_state_bytes: FeatureVersion, + pub fetch_reduced_platform_state_bytes: FeatureVersion, + pub store_reduced_platform_state_bytes: FeatureVersion, } #[derive(Clone, Debug, Default)] diff --git a/packages/rs-platform-version/src/version/drive_versions/v1.rs b/packages/rs-platform-version/src/version/drive_versions/v1.rs index 6e87d8fe420..f7dc6e68748 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v1.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v1.rs @@ -90,6 +90,8 @@ pub const DRIVE_VERSION_V1: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v2.rs b/packages/rs-platform-version/src/version/drive_versions/v2.rs index 0fe4f8f235e..02a4ab14d14 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v2.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v2.rs @@ -90,6 +90,8 @@ pub const DRIVE_VERSION_V2: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v3.rs b/packages/rs-platform-version/src/version/drive_versions/v3.rs index a542fe99e85..13d5a29cc94 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v3.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v3.rs @@ -90,6 +90,8 @@ pub const DRIVE_VERSION_V3: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v4.rs b/packages/rs-platform-version/src/version/drive_versions/v4.rs index 4481d8b90ac..d2c09c3a2fd 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v4.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v4.rs @@ -90,6 +90,8 @@ pub const DRIVE_VERSION_V4: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v5.rs b/packages/rs-platform-version/src/version/drive_versions/v5.rs index bfbce3d74b1..6cd9ffdefe8 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v5.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v5.rs @@ -92,6 +92,8 @@ pub const DRIVE_VERSION_V5: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v6.rs b/packages/rs-platform-version/src/version/drive_versions/v6.rs index 304cbdb70c7..7b265daeafa 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v6.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v6.rs @@ -94,6 +94,8 @@ pub const DRIVE_VERSION_V6: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v7.rs b/packages/rs-platform-version/src/version/drive_versions/v7.rs index 05d8ad2d05e..3761c8fbd6d 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v7.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v7.rs @@ -92,6 +92,8 @@ pub const DRIVE_VERSION_V7: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v8.rs b/packages/rs-platform-version/src/version/drive_versions/v8.rs index 7f421173191..f48deed6d37 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v8.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v8.rs @@ -92,6 +92,8 @@ pub const DRIVE_VERSION_V8: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v9.rs b/packages/rs-platform-version/src/version/drive_versions/v9.rs index fade08c521d..a9cd31b2da7 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v9.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v9.rs @@ -106,6 +106,8 @@ pub const DRIVE_VERSION_V9: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/fee/mod.rs b/packages/rs-platform-version/src/version/fee/mod.rs index 8b603e2ddba..1df42b6beb0 100644 --- a/packages/rs-platform-version/src/version/fee/mod.rs +++ b/packages/rs-platform-version/src/version/fee/mod.rs @@ -29,6 +29,16 @@ pub mod vote_resolution_fund_fees; pub type FeeVersionNumber = u32; +/// The fee schedules [`FeeVersion::get`] can resolve, indexed by `fee_version_number - 1`. +/// +/// # This list is INCOMPLETE, and that is a known defect +/// +/// `FEE_VERSION2` — what protocol versions 9 and later actually run with — is missing, and +/// declares `fee_version_number: 1`, colliding with `FEE_VERSION1`. Since the fee version +/// NUMBER is the only thing persisted (`PlatformStateForSavingV1` and +/// `ReducedPlatformStateV0` both store `epoch index -> number`), every node that restarts +/// or state-syncs rehydrates previous epochs' fees as `FEE_VERSION1`. See the doc comment +/// on [`v2::FEE_VERSION2`] for why that is currently latent and what fixing it requires. pub const FEE_VERSIONS: &[FeeVersion] = &[FEE_VERSION1]; #[derive(Clone, Debug, Encode, Decode, Default, PartialEq, Eq)] @@ -116,3 +126,61 @@ impl From for FeeVersion { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::version::fee::v2::FEE_VERSION2; + + /// Every `FeeVersion` constant must carry a distinct `fee_version_number`, and + /// `FEE_VERSIONS` must contain all of them, because the number is the ONLY thing + /// persisted: `PlatformStateForSavingV1` and `ReducedPlatformStateV0` both store + /// `(epoch index -> fee version number)` and rehydrate through `FeeVersion::get`. A + /// number that does not resolve back to the constant it came from silently substitutes + /// a different fee schedule on any node that restarts or state-syncs. + /// + /// This test FAILS today, which is why it is ignored: `FEE_VERSION2` declares + /// `fee_version_number: 1`, the same as `FEE_VERSION1`, and is absent from + /// `FEE_VERSIONS`, so `FeeVersion::get(1)` returns `FEE_VERSION1` even for the epochs + /// that ran on `FEE_VERSION2`. See the doc comment on `FEE_VERSION2`. + /// + /// Un-ignore it as part of giving `FEE_VERSION2` its own number and adding it to + /// `FEE_VERSIONS`. That is protocol-visible and needs a migration, which is why the + /// defect is pinned here rather than fixed in place. + #[test] + #[ignore = "known defect: FEE_VERSION2 reuses fee_version_number 1 and is absent from \ + FEE_VERSIONS; fixing it is protocol-visible - see the FEE_VERSION2 docs"] + fn fee_version_numbers_are_unique_and_resolvable() { + let all_fee_versions = [&FEE_VERSION1, &FEE_VERSION2]; + + for fee_version in all_fee_versions { + let resolved = FeeVersion::get(fee_version.fee_version_number).unwrap_or_else(|_| { + panic!( + "fee version number {} does not resolve through FEE_VERSIONS", + fee_version.fee_version_number + ) + }); + assert_eq!( + resolved, fee_version, + "FeeVersion::get({}) returned a DIFFERENT fee schedule than the constant \ + declaring that number. Every number-only round trip - a node restarting, a \ + node state-syncing - would substitute this wrong schedule.", + fee_version.fee_version_number + ); + } + + let mut numbers: Vec = all_fee_versions + .iter() + .map(|fee_version| fee_version.fee_version_number) + .collect(); + numbers.sort_unstable(); + let mut deduped = numbers.clone(); + deduped.dedup(); + assert_eq!( + deduped.len(), + numbers.len(), + "two FeeVersion constants share a fee_version_number: {:?}", + numbers + ); + } +} diff --git a/packages/rs-platform-version/src/version/fee/v2.rs b/packages/rs-platform-version/src/version/fee/v2.rs index fe82ac5534f..24f09b06a49 100644 --- a/packages/rs-platform-version/src/version/fee/v2.rs +++ b/packages/rs-platform-version/src/version/fee/v2.rs @@ -9,7 +9,45 @@ use crate::version::fee::vote_resolution_fund_fees::v1::VOTE_RESOLUTION_FUND_FEE use crate::version::fee::FeeVersion; /// Introduced in protocol version 9 (2.0) +/// +/// # WARNING: `fee_version_number` collides with [`FEE_VERSION1`], and this one is not +/// reachable by number +/// +/// [`FeeVersion::get`] resolves a number through [`FEE_VERSIONS`], which contains only +/// `FEE_VERSION1`. This constant declares the SAME `fee_version_number: 1`, so +/// `FeeVersion::get(1)` can only ever return `FEE_VERSION1` — never this one, even though +/// this is what protocol versions 9 and later actually run with, and the two differ in +/// `data_contract_registration`. +/// +/// That makes every number-only round trip of a fee version silently lossy. Two exist: +/// +/// * `PlatformStateForSavingV1` stores `previous_fee_versions` as +/// `(epoch index -> fee version number)`, so a node that RESTARTS rehydrates previous +/// epochs' fees as `FEE_VERSION1`; +/// * `ReducedPlatformStateV0` does the same, so a node that STATE-SYNCS gets the same +/// substitution without even restarting. +/// +/// It is latent rather than a live consensus fork only because `previous_fee_versions` is +/// consulted solely to price storage refunds (`rs-drive/src/fees/op.rs`), and +/// `FEE_VERSION1` and `FEE_VERSION2` have IDENTICAL `storage` fees. It becomes a fork the +/// moment a future `FeeVersion` changes a storage or processing fee without also taking a +/// distinct number. +/// +/// ## The rule +/// +/// **Every `FeeVersion` constant must have a unique `fee_version_number`, and must be +/// listed in [`FEE_VERSIONS`] at the index its number implies.** Fixing this constant to +/// `fee_version_number: 2` and adding it to `FEE_VERSIONS` is protocol-visible (it changes +/// what a restarted or state-synced node computes for old epochs), so it needs a +/// versioned migration rather than an in-place edit — which is why this is documented here +/// instead of changed. `fee_version_numbers_are_unique` in `super` is the enforcement, and +/// is `#[ignore]`d until then. +/// +/// [`FEE_VERSION1`]: crate::version::fee::v1::FEE_VERSION1 +/// [`FEE_VERSIONS`]: crate::version::fee::FEE_VERSIONS +/// [`FeeVersion::get`]: crate::version::fee::FeeVersion::get pub const FEE_VERSION2: FeeVersion = FeeVersion { + // BUG: must be 2. See the doc comment above — changing it is protocol-visible. fee_version_number: 1, uses_version_fee_multiplier_permille: Some(1000), //No action storage: FEE_STORAGE_VERSION1, diff --git a/packages/rs-platform-version/src/version/mocks/v2_test.rs b/packages/rs-platform-version/src/version/mocks/v2_test.rs index 76579f3a0d7..838b110151d 100644 --- a/packages/rs-platform-version/src/version/mocks/v2_test.rs +++ b/packages/rs-platform-version/src/version/mocks/v2_test.rs @@ -23,6 +23,7 @@ use crate::version::drive_abci_versions::drive_abci_query_versions::{ DriveAbciQueryShieldedVersions, DriveAbciQuerySystemVersions, DriveAbciQueryTokenVersions, DriveAbciQueryValidatorVersions, DriveAbciQueryVersions, DriveAbciQueryVotingVersions, }; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v1::DRIVE_ABCI_VALIDATION_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v1::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V1; @@ -128,6 +129,8 @@ pub const TEST_PLATFORM_V2: PlatformVersion = PlatformVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { @@ -478,6 +481,7 @@ pub const TEST_PLATFORM_V2: PlatformVersion = PlatformVersion { }, }, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/mocks/v3_test.rs b/packages/rs-platform-version/src/version/mocks/v3_test.rs index e5cfd15b1bb..d6325c633c0 100644 --- a/packages/rs-platform-version/src/version/mocks/v3_test.rs +++ b/packages/rs-platform-version/src/version/mocks/v3_test.rs @@ -29,6 +29,7 @@ use crate::version::drive_abci_versions::drive_abci_method_versions::{ DriveAbciVotingMethodVersions, }; use crate::version::drive_abci_versions::drive_abci_query_versions::v1::DRIVE_ABCI_QUERY_VERSIONS_V1; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v3::DRIVE_ABCI_VALIDATION_VERSIONS_V3; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -166,12 +167,15 @@ pub const TEST_PLATFORM_V3: PlatformVersion = PlatformVersion { platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }, validation_and_processing: DRIVE_ABCI_VALIDATION_VERSIONS_V3, withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V1, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/mod.rs b/packages/rs-platform-version/src/version/mod.rs index 1b1635efb42..ae5fd0887e1 100644 --- a/packages/rs-platform-version/src/version/mod.rs +++ b/packages/rs-platform-version/src/version/mod.rs @@ -1,6 +1,6 @@ mod protocol_version; -use crate::version::v14::PROTOCOL_VERSION_14; +use crate::version::v15::PROTOCOL_VERSION_15; pub use protocol_version::*; use std::ops::RangeInclusive; @@ -20,6 +20,7 @@ pub mod v11; pub mod v12; pub mod v13; pub mod v14; +pub mod v15; pub mod v2; pub mod v3; pub mod v4; @@ -33,5 +34,5 @@ pub type ProtocolVersion = u32; pub const ALL_VERSIONS: RangeInclusive = 1..=LATEST_VERSION; -pub const LATEST_VERSION: ProtocolVersion = PROTOCOL_VERSION_14; +pub const LATEST_VERSION: ProtocolVersion = PROTOCOL_VERSION_15; pub const INITIAL_PROTOCOL_VERSION: ProtocolVersion = 1; diff --git a/packages/rs-platform-version/src/version/protocol_version.rs b/packages/rs-platform-version/src/version/protocol_version.rs index 0eded570c10..2ba05cc8366 100644 --- a/packages/rs-platform-version/src/version/protocol_version.rs +++ b/packages/rs-platform-version/src/version/protocol_version.rs @@ -22,6 +22,7 @@ use crate::version::v11::PLATFORM_V11; use crate::version::v12::PLATFORM_V12; use crate::version::v13::PLATFORM_V13; use crate::version::v14::PLATFORM_V14; +use crate::version::v15::PLATFORM_V15; use crate::version::v2::PLATFORM_V2; use crate::version::v3::PLATFORM_V3; use crate::version::v4::PLATFORM_V4; @@ -61,6 +62,7 @@ pub const PLATFORM_VERSIONS: &[PlatformVersion] = &[ PLATFORM_V12, PLATFORM_V13, PLATFORM_V14, + PLATFORM_V15, ]; #[cfg(feature = "mock-versions")] @@ -69,7 +71,7 @@ pub static PLATFORM_TEST_VERSIONS: OnceLock> = OnceLock::ne #[cfg(feature = "mock-versions")] const DEFAULT_PLATFORM_TEST_VERSIONS: &[PlatformVersion] = &[TEST_PLATFORM_V2, TEST_PLATFORM_V3]; -pub const LATEST_PLATFORM_VERSION: &PlatformVersion = &PLATFORM_V14; +pub const LATEST_PLATFORM_VERSION: &PlatformVersion = &PLATFORM_V15; pub const DESIRED_PLATFORM_VERSION: &PlatformVersion = LATEST_PLATFORM_VERSION; diff --git a/packages/rs-platform-version/src/version/v1.rs b/packages/rs-platform-version/src/version/v1.rs index b3c54787c77..9a81628e63c 100644 --- a/packages/rs-platform-version/src/version/v1.rs +++ b/packages/rs-platform-version/src/version/v1.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v1::DRIVE_ABCI_METHOD_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v1::DRIVE_ABCI_VALIDATION_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v1::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V1; @@ -40,6 +41,7 @@ pub const PLATFORM_V1: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V1, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v10.rs b/packages/rs-platform-version/src/version/v10.rs index f04b14d341a..66eabb8d84e 100644 --- a/packages/rs-platform-version/src/version/v10.rs +++ b/packages/rs-platform-version/src/version/v10.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v6::DRIVE_ABCI_METHOD_VERSIONS_V6; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v6::DRIVE_ABCI_VALIDATION_VERSIONS_V6; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -41,6 +42,7 @@ pub const PLATFORM_V10: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v11.rs b/packages/rs-platform-version/src/version/v11.rs index eb039ad49cf..414326d77a3 100644 --- a/packages/rs-platform-version/src/version/v11.rs +++ b/packages/rs-platform-version/src/version/v11.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v7::DRIVE_ABCI_METHOD_VERSIONS_V7; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v7::DRIVE_ABCI_VALIDATION_VERSIONS_V7; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -41,6 +42,7 @@ pub const PLATFORM_V11: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v12.rs b/packages/rs-platform-version/src/version/v12.rs index 2d334b1fd7d..cac54b6cb49 100644 --- a/packages/rs-platform-version/src/version/v12.rs +++ b/packages/rs-platform-version/src/version/v12.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v8::DRIVE_ABCI_METHOD_VERSIONS_V8; use crate::version::drive_abci_versions::drive_abci_query_versions::v1::DRIVE_ABCI_QUERY_VERSIONS_V1; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v8::DRIVE_ABCI_VALIDATION_VERSIONS_V8; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -44,6 +45,7 @@ pub const PLATFORM_V12: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V1, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v13.rs b/packages/rs-platform-version/src/version/v13.rs index b6249ba91fc..776ad91a2e1 100644 --- a/packages/rs-platform-version/src/version/v13.rs +++ b/packages/rs-platform-version/src/version/v13.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v9::DRIVE_ABCI_METHOD_VERSIONS_V9; use crate::version::drive_abci_versions::drive_abci_query_versions::v1::DRIVE_ABCI_QUERY_VERSIONS_V1; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v9::DRIVE_ABCI_VALIDATION_VERSIONS_V9; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -70,6 +71,7 @@ pub const PLATFORM_V13: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V1, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v14.rs b/packages/rs-platform-version/src/version/v14.rs index 12c9d8fe713..070f72c4827 100644 --- a/packages/rs-platform-version/src/version/v14.rs +++ b/packages/rs-platform-version/src/version/v14.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v10::DRIVE_ABCI_METHOD_VERSIONS_V10; use crate::version::drive_abci_versions::drive_abci_query_versions::v3::DRIVE_ABCI_QUERY_VERSIONS_V3; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v10::DRIVE_ABCI_VALIDATION_VERSIONS_V10; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v3::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V3; @@ -202,6 +203,7 @@ pub const PLATFORM_V14: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V3, // changed: prune bound for the total credits history query: DRIVE_ABCI_QUERY_VERSIONS_V3, // changed: ranked + boolean-HAVING routing gate; the v1 handler also resolves IN_TIME_RANGE from committed block time checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v15.rs b/packages/rs-platform-version/src/version/v15.rs new file mode 100644 index 00000000000..8b84f8d8757 --- /dev/null +++ b/packages/rs-platform-version/src/version/v15.rs @@ -0,0 +1,130 @@ +use crate::version::consensus_versions::ConsensusVersions; +use crate::version::dpp_versions::dpp_asset_lock_versions::v1::DPP_ASSET_LOCK_VERSIONS_V1; +use crate::version::dpp_versions::dpp_contract_versions::v6::CONTRACT_VERSIONS_V6; +use crate::version::dpp_versions::dpp_costs_versions::v1::DPP_COSTS_VERSIONS_V1; +use crate::version::dpp_versions::dpp_document_versions::v4::DOCUMENT_VERSIONS_V4; +use crate::version::dpp_versions::dpp_factory_versions::v1::DPP_FACTORY_VERSIONS_V1; +use crate::version::dpp_versions::dpp_identity_versions::v1::IDENTITY_VERSIONS_V1; +use crate::version::dpp_versions::dpp_method_versions::v3::DPP_METHOD_VERSIONS_V3; +use crate::version::dpp_versions::dpp_state_transition_conversion_versions::v2::STATE_TRANSITION_CONVERSION_VERSIONS_V2; +use crate::version::dpp_versions::dpp_state_transition_method_versions::v1::STATE_TRANSITION_METHOD_VERSIONS_V1; +use crate::version::dpp_versions::dpp_state_transition_serialization_versions::v3::STATE_TRANSITION_SERIALIZATION_VERSIONS_V3; +use crate::version::dpp_versions::dpp_state_transition_versions::v3::STATE_TRANSITION_VERSIONS_V3; +use crate::version::dpp_versions::dpp_token_versions::v2::TOKEN_VERSIONS_V2; +use crate::version::dpp_versions::dpp_validation_versions::v5::DPP_VALIDATION_VERSIONS_V5; +use crate::version::dpp_versions::dpp_voting_versions::v2::VOTING_VERSION_V2; +use crate::version::dpp_versions::DPPVersion; +use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; +use crate::version::drive_abci_versions::drive_abci_method_versions::v11::DRIVE_ABCI_METHOD_VERSIONS_V11; +use crate::version::drive_abci_versions::drive_abci_query_versions::v3::DRIVE_ABCI_QUERY_VERSIONS_V3; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; +use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; +use crate::version::drive_abci_versions::drive_abci_validation_versions::v10::DRIVE_ABCI_VALIDATION_VERSIONS_V10; +use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v3::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V3; +use crate::version::drive_abci_versions::DriveAbciVersion; +use crate::version::drive_versions::v9::DRIVE_VERSION_V9; +use crate::version::fee::v2::FEE_VERSION2; +use crate::version::protocol_version::PlatformVersion; +use crate::version::system_data_contract_versions::v3::SYSTEM_DATA_CONTRACT_VERSIONS_V3; +use crate::version::system_limits::v4::SYSTEM_LIMITS_V4; +use crate::version::ProtocolVersion; + +pub const PROTOCOL_VERSION_15: ProtocolVersion = 15; + +/// v15 enables ABCI state sync: a fresh node can bootstrap from a peer's grovedb +/// snapshot instead of replaying the chain. +/// +/// The consensus changes gate on `DRIVE_ABCI_METHOD_VERSIONS_V11`: +/// +/// * `run_block_proposal` 0 -> 1: every block writes a reduced platform state +/// (`Misc/reduced_saved_state`) into the replicated state just before the root hash is +/// computed, and `validator_set_update` moves above the root-hash computation so the +/// stored reduced state reflects the post-rotation validator set. The full platform +/// state only lives in non-replicated aux storage, so without this a state-synced node +/// would have no way to rebuild its in-memory state. +/// * `consensus_params_update` 1 -> 2: the first block of v15 also emits evidence +/// params sized for state-synced nodes that do not hold full history (issue #2512). +/// * `perform_events_on_first_block_of_protocol_change` writes the initial reduced state +/// at the v15 activation block, so every snapshot taken at or after activation is +/// restorable. Snapshots from before activation lack the key and are not served. +/// +/// Everything else matches v14. The grovedb state sync protocol version used for +/// snapshots is `DRIVE_ABCI_STATE_SYNC_VERSIONS_V1.protocol_version` (1), shared by all +/// platform versions; grovedb updates its replication protocol in place, so exactly one +/// version exists. +pub const PLATFORM_V15: PlatformVersion = PlatformVersion { + protocol_version: PROTOCOL_VERSION_15, + drive: DRIVE_VERSION_V9, + drive_abci: DriveAbciVersion { + structs: DRIVE_ABCI_STRUCTURE_VERSIONS_V1, + methods: DRIVE_ABCI_METHOD_VERSIONS_V11, // changed: run_block_proposal v1 (reduced state write + validator rotation above root hash) and consensus_params_update v2 (evidence params on the v15 activation block) + validation_and_processing: DRIVE_ABCI_VALIDATION_VERSIONS_V10, + withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V3, + query: DRIVE_ABCI_QUERY_VERSIONS_V3, + checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, + }, + dpp: DPPVersion { + costs: DPP_COSTS_VERSIONS_V1, + validation: DPP_VALIDATION_VERSIONS_V5, + state_transition_serialization_versions: STATE_TRANSITION_SERIALIZATION_VERSIONS_V3, + state_transition_conversion_versions: STATE_TRANSITION_CONVERSION_VERSIONS_V2, + state_transition_method_versions: STATE_TRANSITION_METHOD_VERSIONS_V1, + state_transitions: STATE_TRANSITION_VERSIONS_V3, + contract_versions: CONTRACT_VERSIONS_V6, + document_versions: DOCUMENT_VERSIONS_V4, + identity_versions: IDENTITY_VERSIONS_V1, + voting_versions: VOTING_VERSION_V2, + token_versions: TOKEN_VERSIONS_V2, + asset_lock_versions: DPP_ASSET_LOCK_VERSIONS_V1, + methods: DPP_METHOD_VERSIONS_V3, + factory_versions: DPP_FACTORY_VERSIONS_V1, + }, + system_data_contracts: SYSTEM_DATA_CONTRACT_VERSIONS_V3, + fee_version: FEE_VERSION2, + system_limits: SYSTEM_LIMITS_V4, + consensus: ConsensusVersions { + tenderdash_consensus_version: 1, + }, +}; + +#[cfg(test)] +mod tests { + use super::*; + use crate::version::v14::PLATFORM_V14; + + /// The state sync consensus changes live in v15's own method table, so a v14 node + /// keeps running run_block_proposal v0 (no reduced-state write, rotation after the + /// root hash) and consensus_params_update v1. Making v14 non-zero here would be + /// consensus-breaking for already-deployed nodes. + #[test] + fn state_sync_consensus_changes_gate_at_v15() { + assert_eq!(PLATFORM_V14.drive_abci.methods.engine.run_block_proposal, 0); + assert_eq!( + PLATFORM_V14 + .drive_abci + .methods + .engine + .consensus_params_update, + 1 + ); + assert_eq!(PLATFORM_V15.drive_abci.methods.engine.run_block_proposal, 1); + assert_eq!( + PLATFORM_V15 + .drive_abci + .methods + .engine + .consensus_params_update, + 2 + ); + } + + /// All platform versions share grovedb state sync protocol version 1 — the only + /// version that exists, since grovedb updates its replication protocol in place. + /// The supported set lives next to the snapshot types in drive-abci. + #[test] + fn state_sync_wire_protocol_version_is_one() { + assert_eq!(PLATFORM_V15.drive_abci.state_sync.protocol_version, 1); + assert_eq!(PLATFORM_V14.drive_abci.state_sync.protocol_version, 1); + } +} diff --git a/packages/rs-platform-version/src/version/v2.rs b/packages/rs-platform-version/src/version/v2.rs index 93cd7b07232..0bcfcb9fbeb 100644 --- a/packages/rs-platform-version/src/version/v2.rs +++ b/packages/rs-platform-version/src/version/v2.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v1::DRIVE_ABCI_METHOD_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v2::DRIVE_ABCI_VALIDATION_VERSIONS_V2; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v1::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V1; @@ -40,6 +41,7 @@ pub const PLATFORM_V2: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V1, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v3.rs b/packages/rs-platform-version/src/version/v3.rs index c125b94ff9b..c8bfa8b212d 100644 --- a/packages/rs-platform-version/src/version/v3.rs +++ b/packages/rs-platform-version/src/version/v3.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v2::DRIVE_ABCI_METHOD_VERSIONS_V2; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v2::DRIVE_ABCI_VALIDATION_VERSIONS_V2; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v1::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V1; @@ -46,6 +47,7 @@ pub const PLATFORM_V3: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V1, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v4.rs b/packages/rs-platform-version/src/version/v4.rs index dba41251e96..c7268418092 100644 --- a/packages/rs-platform-version/src/version/v4.rs +++ b/packages/rs-platform-version/src/version/v4.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v3::DRIVE_ABCI_METHOD_VERSIONS_V3; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v3::DRIVE_ABCI_VALIDATION_VERSIONS_V3; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -41,6 +42,7 @@ pub const PLATFORM_V4: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v5.rs b/packages/rs-platform-version/src/version/v5.rs index 3c288cfe63d..e0e9150dfd6 100644 --- a/packages/rs-platform-version/src/version/v5.rs +++ b/packages/rs-platform-version/src/version/v5.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v4::DRIVE_ABCI_METHOD_VERSIONS_V4; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v3::DRIVE_ABCI_VALIDATION_VERSIONS_V3; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -41,6 +42,7 @@ pub const PLATFORM_V5: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v6.rs b/packages/rs-platform-version/src/version/v6.rs index 7d948da6f00..43def28162d 100644 --- a/packages/rs-platform-version/src/version/v6.rs +++ b/packages/rs-platform-version/src/version/v6.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v4::DRIVE_ABCI_METHOD_VERSIONS_V4; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v4::DRIVE_ABCI_VALIDATION_VERSIONS_V4; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -41,6 +42,7 @@ pub const PLATFORM_V6: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v7.rs b/packages/rs-platform-version/src/version/v7.rs index 09755d462e1..eabdc58c585 100644 --- a/packages/rs-platform-version/src/version/v7.rs +++ b/packages/rs-platform-version/src/version/v7.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v4::DRIVE_ABCI_METHOD_VERSIONS_V4; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v5::DRIVE_ABCI_VALIDATION_VERSIONS_V5; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -41,6 +42,7 @@ pub const PLATFORM_V7: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v8.rs b/packages/rs-platform-version/src/version/v8.rs index 2096142ac18..f4c2f9dd1f5 100644 --- a/packages/rs-platform-version/src/version/v8.rs +++ b/packages/rs-platform-version/src/version/v8.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v5::DRIVE_ABCI_METHOD_VERSIONS_V5; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v5::DRIVE_ABCI_VALIDATION_VERSIONS_V5; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -45,6 +46,7 @@ pub const PLATFORM_V8: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v9.rs b/packages/rs-platform-version/src/version/v9.rs index a27803f6da8..8ee4fc891bf 100644 --- a/packages/rs-platform-version/src/version/v9.rs +++ b/packages/rs-platform-version/src/version/v9.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v6::DRIVE_ABCI_METHOD_VERSIONS_V6; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v6::DRIVE_ABCI_VALIDATION_VERSIONS_V6; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -41,6 +42,7 @@ pub const PLATFORM_V9: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/wasm-sdk/src/context_provider.rs b/packages/wasm-sdk/src/context_provider.rs index cd17e74aa3c..1d1367dd181 100644 --- a/packages/wasm-sdk/src/context_provider.rs +++ b/packages/wasm-sdk/src/context_provider.rs @@ -257,7 +257,27 @@ impl WasmTrustedContext { .await .map_err(|e| WasmSdkError::generic(format!("Failed to prefetch quorums: {}", e)))?; - let discovered_addresses = Self::fetch_addresses_from(&inner).await?; + // Masternode discovery is an optional convenience: it only feeds the + // no-explicit-addresses path in `withTrustedContext`, while the quorum + // data prefetched above is what proof verification actually needs. On + // a local network the sidecar's per-masternode version checks reject + // the gateway's self-signed TLS, so discovery failing there is the + // NORMAL case and must not make the whole trusted context unusable + // for an SDK constructed with explicit addresses. On public networks + // the failure stays fatal: it signals a genuine outage of the trusted + // endpoint, and degrading silently would hide it. + let discovered_addresses = match Self::fetch_addresses_from(&inner).await { + Ok(addresses) => addresses, + Err(e) if network == dash_sdk::dpp::dashcore::Network::Regtest => { + tracing::warn!( + error = %e, + "trusted context: masternode discovery unavailable, continuing without \ + discovered addresses (explicitly configured addresses are unaffected)" + ); + Vec::new() + } + Err(e) => return Err(e), + }; Ok(WasmTrustedContext { inner, diff --git a/yarn.lock b/yarn.lock index 659b772ef72..eeeb07c48bb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7315,6 +7315,8 @@ __metadata: "@dashevo/dashcore-lib": "npm:~0.22.0" "@dashevo/dashd-rpc": "npm:^19.0.0" "@dashevo/docker-compose": "npm:^0.24.4" + "@dashevo/dpns-contract": "workspace:*" + "@dashevo/evo-sdk": "workspace:*" "@dashevo/wallet-lib": "workspace:*" "@dashevo/withdrawals-contract": "workspace:*" "@oclif/core": "npm:^3.10.8" @@ -7328,6 +7330,7 @@ __metadata: chai-as-promised: "npm:^7.1.1" chalk: "npm:^4.1.0" cron: "npm:^2.1.0" + dash: "workspace:*" dirty-chai: "npm:^2.0.1" diskusage-ng: "npm:^1.0.4" dockerode: "npm:^4.0.9"