Merge pull request #134 from synonymdev/feat/channel-monitor-desync-repro

feat: add channel monitor desync repro tooling

Author: piotr-iohk

docker/bitcoin-cli

@@ -28,6 +28,8 @@ Commands:
     getInvoice <amount>                Get a new BIP21 URI with a bech32 address
   LND:
     getinfo                            Show LND node info (for connectivity debugging)
+    openchannel <node_id> [amount]     Open channel from LND to node (default: 500000 sats)
+    payinvoice <invoice> [amount]      Pay a Lightning invoice via LND
     holdinvoice [amount] [-m memo]     Create a hold invoice
     settleinvoice <preimage>           Reveal a preimage and use it to settle the corresponding invoice
     cancelinvoice <payment_hash>       Cancels a currently open invoice
@@ -196,6 +198,127 @@ if [[ "$command" = "getinfo" ]]; then
   exit
 fi
 
+# Open channel from LND to a node
+if [[ "$command" = "openchannel" ]]; then
+  shift
+
+  node_id="${1:-}"
+  amount="${2:-500000}"
+
+  if [ -z "$node_id" ]; then
+    echo "Usage: $CLI_NAME openchannel <node_id> [amount_sats]"
+    echo ""
+    echo "  node_id: app's Lightning node ID (Settings > Advanced > Lightning Node Info)"
+    echo "  amount:  channel size in sats (default: 500000)"
+    exit 1
+  fi
+
+  # Check peer connection
+  echo "→ Checking peer connection..."
+  peer_count=$("${LNCLI_CMD[@]}" listpeers 2>/dev/null | jq "[.peers[] | select(.pub_key==\"$node_id\")] | length")
+
+  if [ "$peer_count" = "0" ]; then
+    lnd_pubkey=$("${LNCLI_CMD[@]}" getinfo 2>/dev/null | jq -r '.identity_pubkey')
+    echo "✗ Node is not connected as a peer."
+    echo ""
+    echo "  Paste this in the app (Settings > Advanced > Channels > Add Connection):"
+    echo "    ${lnd_pubkey}@0.0.0.0:9735"
+    echo ""
+    echo "  Then re-run this command."
+    exit 1
+  fi
+
+  echo "✓ Peer connected"
+
+  # Fund LND if needed
+  balance=$("${LNCLI_CMD[@]}" walletbalance 2>/dev/null | jq -r '.confirmed_balance')
+  echo "→ LND on-chain balance: $balance sats"
+
+  if [ "$balance" -lt "$amount" ]; then
+    echo "→ Funding LND..."
+    lnd_addr=$("${LNCLI_CMD[@]}" newaddress p2wkh 2>/dev/null | jq -r '.address')
+    "${BASE_COMMAND[@]}" -named sendtoaddress address="$lnd_addr" amount=1 fee_rate=25 > /dev/null
+    "${BASE_COMMAND[@]}" -generate 6 > /dev/null
+    echo "✓ Funded LND with 1 BTC"
+    sleep 2
+  fi
+
+  # Open channel
+  echo "→ Opening ${amount} sat channel to ${node_id:0:20}..."
+  result=$("${LNCLI_CMD[@]}" openchannel --node_key "$node_id" --local_amt "$amount" --private 2>&1) || {
+    echo "✗ Failed: $result"
+    exit 1
+  }
+
+  txid=$(echo "$result" | jq -r '.funding_txid // empty' 2>/dev/null)
+  if [ -z "$txid" ]; then
+    echo "✗ Failed: $result"
+    exit 1
+  fi
+
+  echo "✓ Channel opened, funding txid: $txid"
+
+  # Mine and wait
+  echo "→ Mining 6 blocks..."
+  "${BASE_COMMAND[@]}" -generate 6 > /dev/null
+  echo "✓ Mined 6 blocks"
+
+  echo "→ Waiting for channel to become active..."
+  for i in $(seq 1 30); do
+    sleep 2
+    active=$("${LNCLI_CMD[@]}" listchannels --peer "$node_id" --active_only 2>/dev/null | jq '.channels | length')
+    if [ "$active" != "0" ]; then
+      echo "✓ Channel is active!"
+      break
+    fi
+    if [ $((i % 5)) -eq 0 ]; then echo "  still waiting... ($i)"; fi
+  done
+
+  if [ "$active" = "0" ]; then
+    echo "⚠ Channel not active yet. May need more time or app needs to sync."
+  fi
+
+  # Summary
+  echo ""
+  echo "══════════════════════════════════"
+  "${LNCLI_CMD[@]}" channelbalance 2>/dev/null | jq -r '"  LND outbound:  \(.local_balance.sat) sats (can pay app)\n  LND inbound:   \(.remote_balance.sat) sats (can receive from app)"'
+  echo "══════════════════════════════════"
+  exit
+fi
+
+# Pay a Lightning invoice via LND
+if [[ "$command" = "payinvoice" ]]; then
+  shift
+
+  invoice="${1:-}"
+  amount="${2:-}"
+
+  if [ -z "$invoice" ]; then
+    echo "Usage: $CLI_NAME payinvoice <invoice> [amount_sats]"
+    exit 1
+  fi
+
+  if [ -n "$amount" ]; then
+    echo "→ Paying invoice via LND (${amount} sats)..."
+    result=$("${LNCLI_CMD[@]}" payinvoice --force --amt "$amount" "$invoice" 2>&1)
+  else
+    echo "→ Paying invoice via LND..."
+    result=$("${LNCLI_CMD[@]}" payinvoice --force "$invoice" 2>&1)
+  fi
+
+  status=$(echo "$result" | grep -i "status" | head -1)
+  if echo "$result" | grep -qi "SUCCEEDED"; then
+    echo "✓ Payment succeeded"
+    echo "$result" | grep -i "payment_hash\|payment_preimage" | head -2
+  else
+    echo "✗ Payment failed"
+    echo "$result"
+    exit 1
+  fi
+
+  exit
+fi
+
 # Create a hold invoice (LND)
 if [[ "$command" = "holdinvoice" ]]; then
   shift

docs/lightning-primer-for-qa.md

@@ -0,0 +1,64 @@
+# Lightning primer for QA
+
+Background for [channel monitor desync repro](./repro-channel-monitor-desync.md) and any work that touches Lightning storage, migration, or startup.
+
+## What a Lightning channel is (operationally)
+
+Two parties lock funds in a 2-of-2 on-chain output. **Off-chain**, they exchange **commitment transactions** that encode “who gets what if we publish now.” Each new off-chain state is a **commitment update**. LDK tracks progress with an internal **`update_id`** (a monotonic counter per channel).
+
+- **ChannelManager** — current view of all channels, balances, and pending HTLCs.
+- **ChannelMonitor** — per-channel state used to watch the chain, enforce penalties, and react to force-closes. It must stay **consistent** with what the ChannelManager believes.
+
+The **chain::Watch** contract (simplified): durable storage must reflect **latest** ChannelMonitor data **before** the app continues as if that state is live. If an old monitor is paired with an advanced manager, LDK reports **`DangerousValue`** and refuses to start — that protects funds.
+
+## HTLCs
+
+**HTLC** means **Hash Time-Locked Contract**. It is a conditional payment: pay the peer if they reveal a preimage by a deadline; otherwise revert. HTLCs live **inside** commitment updates. Each hop of a multi-hop payment adds HTLCs; resolving them advances commitment state again.
+
+Testing “payments” matters because each payment usually causes **multiple** commitment updates, not a 1:1 mapping to “one payment = one update_id step.”
+
+## “Gap” in the test matrix (e.g. 21 / 30 payments)
+
+The doc’s payment counts are a **proxy for many `update_id` advances**, not a magic number from BOLT math.
+
+- **Small** mismatch between an old backup and the live node may be **healed** via peer reconnection and commitment replay.
+- **Large** mismatch, or injecting a **stale monitor** on top of an **advanced** manager, triggers **stale ChannelMonitor** errors and a refused start until recovery.
+
+## What went wrong in the ChannelMonitor desync bug
+
+1. **ChannelManager** on device was **ahead** (normal usage after RN migration).
+2. **Old ChannelMonitor** data (e.g. from RN remote backup) was applied without matching the current manager.
+3. On load: monitor `update_id` ≪ manager → **stale monitor** → **`DangerousValue`** → node will not run.
+
+The **fix path** uses **`accept_stale_channel_monitors`** so ldk-node can align state and **self-heal** (commitment round-trips, chain sync). That is why recovery logs show retries, healing, and sometimes **over a minute** before balances and payments look normal — especially with **many blocks** to sync (e.g. T5) or **local LND** setups vs Blocktank-only flows.
+
+## What to test when Lightning / LDK storage changes
+
+| Area | Why |
+|------|-----|
+| **Cold start** | Any path that reads/writes ChannelManager, monitors, or VSS must not pair **new** manager with **old** monitor. |
+| **Backup / restore** | Restoring must be **consistent snapshots**; partial or older monitor alone is high risk. |
+| **Migration** | RN → native or schema changes: avoid overwriting live data with **stale** remote copies. |
+| **Recovery** | After `DangerousValue` / `accept_stale`: peers reconnect, chain sync completes, **inbound and outbound** payments work, **second launch** does not repeat recovery forever. |
+| **Infra noise** | On regtest, **stale RGS** / gossip can cause transient **“route not found”** — distinguish from persistence bugs (see logs for `DangerousValue` vs routing errors). |
+
+## Risks of incorrect “fixes”
+
+- Skipping or weakening persistence checks can lead to **wrong** enforcement keys or **missed** on-chain reactions.
+- Blindly merging backups can recreate the **stale monitor** class of bug.
+- Recovery paths should always be validated with **real sends/receives** and **restart**, not only “app opens.”
+
+## Glossary
+
+| Term | Meaning |
+|------|--------|
+| **Commitment update** | New off-chain state (balances + HTLC set). |
+| **`update_id`** | LDK’s persisted notion of how far the ChannelMonitor has advanced vs the ChannelManager for that channel. |
+| **HTLC** | **Hash Time-Locked Contract** — conditional payment inside a commitment (hash lock + time lock). |
+| **ChannelMonitor** | Per-channel persisted state for chain watching and dispute handling. |
+| **DangerousValue** | LDK/ldk-node refusing to load because continuing would violate safety assumptions (e.g. stale monitor). |
+| **accept_stale_channel_monitors** | Explicit recovery mode to load despite mismatch, then heal via protocol + sync (use only in controlled recovery). |
+
+## See also
+
+- [repro-channel-monitor-desync.md](./repro-channel-monitor-desync.md) — repro steps, matrix, recovery timing notes