Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -24,4 +24,5 @@ node2
config.yaml
.vscode
.vagrant
.chain_code
.chain_code
node_public_keys.json
19 changes: 19 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,25 @@ configs := []client.KMSSignerOptions{

**Note**: AWS KMS only supports P256 (ECDSA) keys, not Ed25519. If you need Ed25519, use the local signer.

#### Verifying node-signed results

Each node signs the results it publishes with its identity key, in the `Mpc-Node-Id` and `Mpc-Result-Sig` NATS headers. The message body is unchanged. A client pins the node public keys and checks every result before using it:

```go
mpcClient := client.NewMPCClient(client.Options{
NatsConn: natsConn,
Signer: localSigner,
ResultVerifyMode: client.ResultVerifyEnforce, // off (default) | log | enforce
NodePublicKeys: nodePublicKeys, // node_id -> hex public key
RequiredNodeSignatures: 3, // mpc_threshold + 1
})
```

- `setup_identities.sh` writes `node_public_keys.json` for new local setups. For an existing setup or a production cluster, run `mpcium-cli export-node-keys --identity-dir <dir>` against any node's identity directory, which holds every node's `*_identity.json`.
- `enforce` rejects unsigned or tampered results and delivers a result only after `RequiredNodeSignatures` nodes sent the same one. Set `result_signing.per_node_results: true` on the nodes, or only one copy of each result reaches the client.
- `log` reports failed checks but still delivers every result. Use it while rolling out.
- With `RequiredNodeSignatures` above 1, give each client process its own `ClientID`. Processes that share one also share a JetStream consumer, which splits the node copies of a result between them, so none may reach the required count.

## Test with AWS KMS (LocalStack)

For local development and testing with AWS KMS functionality, you can use LocalStack to simulate AWS KMS services.
Expand Down
84 changes: 84 additions & 0 deletions cmd/mpcium-cli/export-node-keys.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
package main

import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"slices"

"github.com/fystack/mpcium/pkg/resultauth"
"github.com/urfave/cli/v3"
)

func exportNodeKeys(ctx context.Context, c *cli.Command) error {
publicKeys := make(map[string]string)
for _, identityDir := range c.StringSlice("identity-dir") {
dirKeys, err := resultauth.LoadPublicKeys(identityDir)
if err != nil {
return err
}
for nodeID, publicKey := range dirKeys {
if existing, ok := publicKeys[nodeID]; ok && existing != publicKey {
return fmt.Errorf("node %s has different public keys in the identity directories", nodeID)
}
publicKeys[nodeID] = publicKey
}
}
if len(publicKeys) == 0 {
return fmt.Errorf("no *_identity.json files found in %v", c.StringSlice("identity-dir"))
}
if _, err := resultauth.NewKeyLookup(publicKeys); err != nil {
return err
}
if err := checkAllPeersHaveKeys(c.String("peers"), publicKeys); err != nil {
return err
}

data, err := json.MarshalIndent(publicKeys, "", " ")
if err != nil {
return fmt.Errorf("failed to encode node public keys: %w", err)
}
data = append(data, '\n')

outputPath := c.String("output")
if outputPath == "-" {
_, err := os.Stdout.Write(data)
return err
}
if _, err := os.Stat(outputPath); err == nil && !c.Bool("overwrite") {
return fmt.Errorf("file %s already exists, use --overwrite to replace it", outputPath)
}
if err := os.WriteFile(outputPath, data, 0o600); err != nil {
return fmt.Errorf("failed to write %s: %w", outputPath, err)
}
fmt.Printf("Wrote %d node public keys to %s\n", len(publicKeys), outputPath)
return nil
}

func checkAllPeersHaveKeys(peersPath string, publicKeys map[string]string) error {
data, err := os.ReadFile(peersPath)
if errors.Is(err, os.ErrNotExist) {
return nil
}
if err != nil {
return fmt.Errorf("failed to read %s: %w", peersPath, err)
}
var peers map[string]string
if err := json.Unmarshal(data, &peers); err != nil {
return fmt.Errorf("failed to parse %s: %w", peersPath, err)
}

var missing []string
for name, nodeID := range peers {
if _, ok := publicKeys[nodeID]; !ok {
missing = append(missing, name)
}
}
if len(missing) > 0 {
slices.Sort(missing)
return fmt.Errorf("no identity file found for peers %v", missing)
}
return nil
}
30 changes: 30 additions & 0 deletions cmd/mpcium-cli/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,36 @@ func main() {
},
Action: generateIdentity,
},
{
Name: "export-node-keys",
Usage: "Write node_id -> public key JSON from node identity files, for clients that verify node-signed results",
Flags: []cli.Flag{
&cli.StringSliceFlag{
Name: "identity-dir",
Aliases: []string{"i"},
Value: []string{"identity"},
Usage: "Directory with <node>_identity.json files; repeat for several directories",
},
&cli.StringFlag{
Name: "peers",
Aliases: []string{"p"},
Value: "peers.json",
Usage: "peers.json used to check that every node has a key; skipped if the file does not exist",
},
&cli.StringFlag{
Name: "output",
Aliases: []string{"o"},
Value: "node_public_keys.json",
Usage: "Output file, or - for stdout",
},
&cli.BoolFlag{
Name: "overwrite",
Aliases: []string{"f"},
Usage: "Overwrite the output file if it already exists",
},
},
Action: exportNodeKeys,
},
{
Name: "generate-initiator",
Usage: "Generate identity files for an event initiator node",
Expand Down
17 changes: 14 additions & 3 deletions cmd/mpcium/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"github.com/fystack/mpcium/pkg/logger"
"github.com/fystack/mpcium/pkg/messaging"
"github.com/fystack/mpcium/pkg/mpc"
"github.com/fystack/mpcium/pkg/resultauth"
"github.com/fystack/mpcium/pkg/security"
"github.com/hashicorp/consul/api"
"github.com/nats-io/nats.go"
Expand Down Expand Up @@ -225,11 +226,21 @@ func runNode(ctx context.Context, c *cli.Command) error {
directMessaging := messaging.NewNatsDirectMessaging(natsConn)
mqManager := messaging.NewNATsMessageQueueManager("mpc", event.ResultStreamSubjects(), natsConn)

genKeyResultQueue := mqManager.NewMessageQueue("mpc_keygen_result", event.KeygenResultSubscriptionSubject(""))
resultQueueOpts := resultauth.QueueOptions{PerNodeResults: viper.GetBool("result_signing.per_node_results")}
genKeyResultQueue := resultauth.NewSigningQueue(
mqManager.NewMessageQueue("mpc_keygen_result", event.KeygenResultSubscriptionSubject("")),
identityStore, nodeID, resultQueueOpts,
)
defer genKeyResultQueue.Close()
singingResultQueue := mqManager.NewMessageQueue("mpc_signing_result", event.SigningResultSubscriptionSubject(""))
singingResultQueue := resultauth.NewSigningQueue(
mqManager.NewMessageQueue("mpc_signing_result", event.SigningResultSubscriptionSubject("")),
identityStore, nodeID, resultQueueOpts,
)
defer singingResultQueue.Close()
reshareResultQueue := mqManager.NewMessageQueue("mpc_reshare_result", event.ReshareResultSubscriptionSubject(""))
reshareResultQueue := resultauth.NewSigningQueue(
mqManager.NewMessageQueue("mpc_reshare_result", event.ReshareResultSubscriptionSubject("")),
identityStore, nodeID, resultQueueOpts,
)
defer reshareResultQueue.Close()

logger.Info("Starting mpcium node", "version", Version, "ID", nodeID, "name", nodeName)
Expand Down
5 changes: 5 additions & 0 deletions config.prod.yaml.template
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,8 @@ session_warm_up_delay_ms: 100
healthcheck:
enabled: false # disabled by default, set to true for cloud deployment
address: "127.0.0.1:8080"

# Keep every node's signed copy of a result so clients can require t+1 matching results.
# Enable only after all clients verify node signatures.
result_signing:
per_node_results: false
5 changes: 5 additions & 0 deletions config.yaml.template
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ healthcheck:
enabled: false # disabled by default, set to true for cloud deployment
address: "0.0.0.0:8080"

# Keep every node's signed copy of a result so clients can require t+1 matching results.
# Enable only after all clients verify node signatures.
result_signing:
per_node_results: false


# Authorization (optional)
# authorization:
Expand Down
11 changes: 11 additions & 0 deletions deployments/ecs/ecs-instruction.md
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,12 @@ identity/node0/
└── node2_identity.json
```

Collect the node public keys for MPC clients. Clients pin them to check that results were signed by these nodes. They are public keys, not secrets. For a deployment that is already running, run the same command against any node's identity directory, which holds every node's `*_identity.json`:

```bash
mpcium-cli export-node-keys --identity-dir identity/node0
```

### Step 5: Generate Chain Code

```bash
Expand Down Expand Up @@ -660,6 +666,11 @@ backup_period_seconds: 300
healthcheck:
enabled: true
address: "0.0.0.0:8080"

# Keep every node's signed copy of a result so clients can require t+1 matching results.
# Enable only after all clients verify node signatures.
result_signing:
per_node_results: false
```

## Post-Deployment Verification
Expand Down
9 changes: 9 additions & 0 deletions deployments/kubernetes/k8s-instruction.md
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,12 @@ identity/node0/
└── node0_private.key.age (encrypted private key)
```

Collect the node public keys for MPC clients. Clients pin them to check that results were signed by these nodes. They are public keys, not secrets. For a deployment that is already running, run the same command against any node's identity directory, which holds every node's `*_identity.json`:

```bash
mpcium-cli export-node-keys --identity-dir identity/node0 --identity-dir identity/node1 --identity-dir identity/node2
```

### Step 5: Prepare Secrets

You'll need the following secrets for each node:
Expand Down Expand Up @@ -303,6 +309,9 @@ data:
chain_code: "<64-char-hex-string>" # Required: 32-byte hex for HD key derivation
backup_enabled: false
backup_period_seconds: 300
# Enable only after all clients verify node signatures (t+1 needs every node's copy)
result_signing:
per_node_results: false
```

### Step 5: Register Peers in Consul
Expand Down
6 changes: 6 additions & 0 deletions deployments/systemd/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,12 @@ scp /opt/mpcium/identity/node0_identity.json root@<node2-ip>:/opt/mpcium/identit

⚠️ Never copy `*_private.key` / `*_private.key.age` to other nodes.

Collect the node public keys for MPC clients. Clients pin them to check that results were signed by these nodes. They are public keys, not secrets. For a deployment that is already running, run the same command against any node's identity directory, which holds every node's `*_identity.json`:

```bash
cd /opt/mpcium && mpcium-cli export-node-keys
```

Before continuing, double-check on every node that:

- `/opt/mpcium/peers.json` exists
Expand Down
21 changes: 18 additions & 3 deletions e2e/base_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/fystack/mpcium/pkg/client"
"github.com/fystack/mpcium/pkg/event"
"github.com/fystack/mpcium/pkg/kvstore"
"github.com/fystack/mpcium/pkg/resultauth"
"github.com/fystack/mpcium/pkg/types"
"github.com/hashicorp/consul/api"
"github.com/nats-io/nats.go"
Expand Down Expand Up @@ -186,6 +187,17 @@ func (s *E2ETestSuite) setupClients(t *testing.T) {
t.Log("Clients setup completed")
}

// e2eRequiredNodeSignatures is mpc_threshold + 1 from config.test.yaml.template.
const e2eRequiredNodeSignatures = 2

func (s *E2ETestSuite) nodePublicKeys(t *testing.T) map[string]string {
t.Helper()
publicKeys, err := resultauth.LoadPublicKeys(filepath.Join(s.testDir, "test_node0", "identity"))
require.NoError(t, err, "Failed to load node public keys")
require.NotEmpty(t, publicKeys, "No node identity files found")
return publicKeys
}

func (s *E2ETestSuite) SetupMPCClient(t *testing.T) {
t.Log("Setting up MPC client...")

Expand All @@ -207,9 +219,12 @@ func (s *E2ETestSuite) SetupMPCClient(t *testing.T) {
}

mpcClient := client.NewMPCClient(client.Options{
NatsConn: s.natsConn,
Signer: localSigner,
ClientID: "e2e-suite",
NatsConn: s.natsConn,
Signer: localSigner,
ClientID: "e2e-suite",
ResultVerifyMode: client.ResultVerifyEnforce,
NodePublicKeys: s.nodePublicKeys(t),
RequiredNodeSignatures: e2eRequiredNodeSignatures,
})
s.mpcClient = mpcClient
t.Log("MPC client created")
Expand Down
2 changes: 2 additions & 0 deletions e2e/config.test.yaml.template
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ nats:
max_concurrent_keygen: 1
max_concurrent_signing: 10
session_warm_up_delay_ms: 500
result_signing:
per_node_results: true

# Chain Code for HD Wallet Child Key Derivation (REQUIRED)
# All nodes MUST use the same chain_code value
Expand Down
9 changes: 6 additions & 3 deletions e2e/multi_client_routing_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -170,9 +170,12 @@ func newScopedMPCClient(t *testing.T, suite *E2ETestSuite, clientID string) (cli
require.NoError(t, err, "Failed to connect scoped client %s to NATS", clientID)

mpcClient := client.NewMPCClient(client.Options{
NatsConn: natsConn,
Signer: signer,
ClientID: clientID,
NatsConn: natsConn,
Signer: signer,
ClientID: clientID,
ResultVerifyMode: client.ResultVerifyEnforce,
NodePublicKeys: suite.nodePublicKeys(t),
RequiredNodeSignatures: e2eRequiredNodeSignatures,
})

return mpcClient, natsConn
Expand Down
5 changes: 3 additions & 2 deletions examples/authorizers/generate/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"syscall"
"time"

"github.com/fystack/mpcium/examples/internal/exampleclient"
"github.com/fystack/mpcium/pkg/client"
"github.com/fystack/mpcium/pkg/config"
"github.com/fystack/mpcium/pkg/event"
Expand Down Expand Up @@ -86,11 +87,11 @@ func main() {
authorizerSigners[authorizerID] = signer
}

mpcClient := client.NewMPCClient(client.Options{
mpcClient := client.NewMPCClient(exampleclient.WithResultVerification(client.Options{
NatsConn: natsConn,
Signer: localSigner,
ClientID: *clientID,
})
}))

var walletStartTimes sync.Map
var walletIDs []string
Expand Down
5 changes: 3 additions & 2 deletions examples/authorizers/sign/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"slices"
"syscall"

"github.com/fystack/mpcium/examples/internal/exampleclient"
"github.com/fystack/mpcium/pkg/client"
"github.com/fystack/mpcium/pkg/config"
"github.com/fystack/mpcium/pkg/event"
Expand Down Expand Up @@ -80,11 +81,11 @@ func main() {
authorizerSigners[authorizerID] = signer
}

mpcClient := client.NewMPCClient(client.Options{
mpcClient := client.NewMPCClient(exampleclient.WithResultVerification(client.Options{
NatsConn: natsConn,
Signer: localSigner,
ClientID: *clientID,
})
}))

// Create a signing request with authorizers
txID := uuid.New().String()
Expand Down
Loading
Loading