test(dashmate): live state sync e2e — join tooling, churn, re-sync and fallback coverage

.github/workflows/tests.yml

@@ -530,6 +530,9 @@ jobs:
           - name: Local network
             test-pattern: test/e2e/localNetwork.spec.js
             restore_local_network_data: true
+          - name: Local network state sync
+            test-pattern: test/e2e/localNetworkStateSync.spec.js
+            restore_local_network_data: true
           - name: Testnet fullnode
             test-pattern: test/e2e/testnetFullnode.spec.js
             restore_local_network_data: false

Cargo.toml

@@ -131,3 +131,32 @@ opt-level = 3
 
 version = "4.2.0-dev.5"
 rust-version = "1.92"
+
+# ============================================================================
+# TEMPORARY — QA ONLY. DROP THIS AT THE REAL RE-PIN.
+#
+# Every grovedb crate is pinned in the package manifests to dashpay/grovedb
+# rev 6c882c3, which carries a restore defect for sum trees: applying a state
+# sync snapshot chunk fails GroveDB verification ("apply_snapshot_chunk grovedb
+# verification failed with N incorrect hashes"), so a joining node can never
+# complete a state sync. packages/rs-drive-abci/tests/sum_tree_sync_probe.rs
+# pins that defect in-tree.
+#
+# dashpay/grovedb#840 (branch feat/state-sync-v2 on the PastaPastaPasta fork,
+# head 10a63e1) fixes it, and its restore wire version is 1, which is inside
+# drive-abci's supported set — so no protocol plumbing changes with it.
+#
+# This section exists so the state sync e2e can be run against the fix before
+# the fork is merged and re-pinned by rev. It MUST be removed once the real
+# pin lands: a branch reference is not reproducible, and a workspace [patch]
+# silently overrides every manifest's rev.
+# ============================================================================
+[patch."https://github.com/dashpay/grovedb"]
+grovedb = { git = "https://github.com/PastaPastaPasta/grovedb", branch = "feat/state-sync-v2" }
+grovedb-costs = { git = "https://github.com/PastaPastaPasta/grovedb", branch = "feat/state-sync-v2" }
+grovedb-path = { git = "https://github.com/PastaPastaPasta/grovedb", branch = "feat/state-sync-v2" }
+grovedb-storage = { git = "https://github.com/PastaPastaPasta/grovedb", branch = "feat/state-sync-v2" }
+grovedb-version = { git = "https://github.com/PastaPastaPasta/grovedb", branch = "feat/state-sync-v2" }
+grovedb-epoch-based-storage-flags = { git = "https://github.com/PastaPastaPasta/grovedb", branch = "feat/state-sync-v2" }
+grovedb-query = { git = "https://github.com/PastaPastaPasta/grovedb", branch = "feat/state-sync-v2" }
+grovedb-commitment-tree = { git = "https://github.com/PastaPastaPasta/grovedb", branch = "feat/state-sync-v2" }

packages/dashmate/configs/defaults/getBaseConfigFactory.js

@@ -356,6 +356,13 @@ export default function getBaseConfigFactory() {
               txProcessingTimeLimit: null,
             },
             epochTime: 788400,
+            stateSync: {
+              snapshots: {
+                enabled: true,
+                frequencySeconds: 600,
+                maxCount: 6,
+              },
+            },
           },
           tenderdash: {
             mode: 'full',
@@ -460,6 +467,16 @@ export default function getBaseConfigFactory() {
               },
             },
             moniker: null,
+            // Serving snapshots to peers is always on in Tenderdash; `enabled`
+            // only makes a fresh node bootstrap from a snapshot, and Tenderdash
+            // ignores it once the node has local state, so it is safe on by
+            // default for existing nodes.
+            stateSync: {
+              enabled: true,
+              retries: 3,
+              chunkRequestTimeout: '15s',
+              fetchersCount: 4,
+            },
           },
         },
         sourcePath: null,

packages/dashmate/configs/defaults/getLocalConfigFactory.js

@@ -105,6 +105,11 @@ export default function getLocalConfigFactory(getBaseConfig) {
             metrics: {
               port: 46660,
             },
+            // A local network genesis starts every node from scratch at the
+            // same time, so there is no populated peer to state sync from.
+            stateSync: {
+              enabled: false,
+            },
           },
           abci: {
             tokioConsole: {
@@ -141,6 +146,11 @@ export default function getLocalConfigFactory(getBaseConfig) {
                 rotation: false,
               },
             },
+            stateSync: {
+              snapshots: {
+                enabled: false,
+              },
+            },
           },
         },
       },

packages/dashmate/configs/getConfigFileMigrationsFactory.js

@@ -1764,6 +1764,23 @@ export default function getConfigFileMigrationsFactory(homeDir, defaultConfigs)
 
         return configFile;
       },
+      '4.2.0-dev.6': (configFile) => {
+        // State sync options are required by the schema now. Pulled from the
+        // default config matching each config's name or group, so the local
+        // preset gets its disables while everything else gets the base
+        // defaults (consume and serve snapshots).
+        Object.entries(configFile.configs)
+          .forEach(([name, options]) => {
+            const defaultConfig = getDefaultConfigByNameOrGroup(name, options.group);
+
+            options.platform.drive.tenderdash.stateSync = defaultConfig
+              .getStored('platform.drive.tenderdash.stateSync');
+            options.platform.drive.abci.stateSync = defaultConfig
+              .getStored('platform.drive.abci.stateSync');
+          });
+
+        return configFile;
+      },
     };
   }
 

packages/dashmate/docker-compose.yml

@@ -96,6 +96,11 @@ services:
       - GROVEDB_VISUALIZER_ADDRESS=0.0.0.0:${PLATFORM_DRIVE_ABCI_GROVEDB_VISUALIZER_PORT:?err}
       - PROPOSER_TX_PROCESSING_TIME_LIMIT=${PLATFORM_DRIVE_ABCI_PROPOSER_TX_PROCESSING_TIME_LIMIT}
       - NETWORK=${NETWORK:?err}
+      # Checkpoints live in the default CHECKPOINTS_PATH (DB_PATH/checkpoints),
+      # inside the drive_abci_data volume
+      - SNAPSHOTS_ENABLED=${PLATFORM_DRIVE_ABCI_STATE_SYNC_SNAPSHOTS_ENABLED:?err}
+      - SNAPSHOTS_FREQUENCY_SECONDS=${PLATFORM_DRIVE_ABCI_STATE_SYNC_SNAPSHOTS_FREQUENCY_SECONDS:?err}
+      - MAX_NUM_SNAPSHOTS=${PLATFORM_DRIVE_ABCI_STATE_SYNC_SNAPSHOTS_MAX_COUNT:?err}
     stop_grace_period: 30s
     expose:
       - 26658

packages/dashmate/docs/config/drive-abci.md

@@ -124,6 +124,18 @@ These settings control developer and debugging tools:
 - Tokio Console: A debugging tool for Rust's async runtime
 - GroveDB Visualizer: A visualization tool for the GroveDB database structure
 
+## State Sync Snapshots
+
+These settings control the serving side of state sync: Drive periodically takes GroveDB checkpoints and hands them to Tenderdash, which offers them to peers bootstrapping via state sync. The consuming side is configured on Tenderdash (see [Tenderdash configuration](./tenderdash.md#state-sync)).
+
+| Option | Description | Default | Example |
+|--------|-------------|---------|---------|
+| `platform.drive.abci.stateSync.snapshots.enabled` | Take and serve state sync snapshots | `true` | `false` |
+| `platform.drive.abci.stateSync.snapshots.frequencySeconds` | How often to take a snapshot, in seconds, at least 60 | `600` | `3600` |
+| `platform.drive.abci.stateSync.snapshots.maxCount` | Snapshots kept before pruning the oldest, at least 2 | `6` | `10` |
+
+Checkpoints are stored inside the Drive data volume under `db/checkpoints`. They hard-link unchanged data, so keeping several costs only a fraction of the database size. The local preset disables snapshots along with state sync consumption.
+
 ## Other options
 
 | Option | Description | Default | Example |

packages/dashmate/docs/config/tenderdash.md

@@ -80,6 +80,21 @@ The RPC interface is used for:
 - Submitting transactions
 - Fetching network status
 
+## State Sync
+
+State sync bootstraps a fresh node from a recent state snapshot fetched from peers instead of replaying every block. These settings control the consuming side; serving snapshots to peers is always on in Tenderdash and is fed by Drive's snapshots (see [Drive ABCI configuration](./drive-abci.md#state-sync-snapshots)).
+
+| Option | Description | Default | Example |
+|--------|-------------|---------|---------|
+| `platform.drive.tenderdash.stateSync.enabled` | Bootstrap a fresh node from a snapshot | `true` | `false` |
+| `platform.drive.tenderdash.stateSync.retries` | Retries before falling back to block sync, `0` disables retries | `3` | `5` |
+| `platform.drive.tenderdash.stateSync.chunkRequestTimeout` | Timeout before re-requesting a snapshot chunk, at least `5s` | `15s` | `30s` |
+| `platform.drive.tenderdash.stateSync.fetchersCount` | Concurrent chunk fetchers, 1 to 64 | `4` | `8` |
+
+- Enabling is safe for existing nodes: Tenderdash only attempts state sync when the node has no local state and disables it by itself otherwise. Only a freshly set up (or reset) node consumes a snapshot, and it ends up with a truncated block history starting at the snapshot height.
+- Snapshots are verified through the P2P layer. The alternative RPC state provider needs at least two reachable Tenderdash RPC servers, but dashmate publishes the Tenderdash RPC on loopback only, without TLS and unproxied by the gateway, so the rendered config hardcodes `use-p2p = true`.
+- The local preset disables state sync: a local network genesis starts every node from scratch, so there is no populated peer to sync from.
+
 ## Metrics and Profiling
 
 These settings control monitoring and profiling tools:

packages/dashmate/package.json

@@ -102,8 +102,11 @@
   },
   "devDependencies": {
     "@babel/core": "^7.26.10",
+    "@dashevo/dpns-contract": "workspace:*",
+    "@dashevo/evo-sdk": "workspace:*",
     "chai": "^4.3.10",
     "chai-as-promised": "^7.1.1",
+    "dash": "workspace:*",
     "dirty-chai": "^2.0.1",
     "eslint": "^9.18.0",
     "globby": "^11",