Revamp Message Pool Nonce calculation

.github/workflows/forest.yml

@@ -218,6 +218,9 @@ jobs:
         run: ./scripts/tests/calibnet_migration_regression_tests.sh
         timeout-minutes: ${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}
   calibnet-wallet-check:
+    concurrency:
+      group: calibnet-wallet-tests
+      cancel-in-progress: false
     needs:
       - build-ubuntu
     name: Wallet tests
@@ -244,10 +247,12 @@ jobs:
           fi
         timeout-minutes: ${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}
   calibnet-delegated-wallet-check:
-    # Disabling this job until message pool nonce calculation is fixed. See: https://github.com/ChainSafe/forest/issues/4899
-    if: false
+    concurrency:
+      group: calibnet-wallet-tests
+      cancel-in-progress: false
     needs:
       - build-ubuntu
+      - calibnet-wallet-check
     name: Delegated wallet tests
     runs-on: ubuntu-24.04
     steps:
@@ -629,6 +634,7 @@ jobs:
       - calibnet-stateless-rpc-check
       - state-migrations-check
       - calibnet-wallet-check
+      - calibnet-delegated-wallet-check
       - calibnet-no-discovery-checks
       - calibnet-kademlia-checks
       - calibnet-eth-mapping-check

CHANGELOG.md

@@ -45,6 +45,8 @@
 
 ### Fixed
 
+- [#6788](https://github.com/ChainSafe/forest/pull/6788): Fixed message pool nonce calculation to align with Lotus.
+
 - [#6707](https://github.com/ChainSafe/forest/issues/6707): Added missing `GoldenWeek` network upgrade entry in `Filecoin.StateGetNetworkParams` RPC method.
 
 - [#6817](https://github.com/ChainSafe/forest/pull/6817): Fixed `StateSearchMsg` to return null instead of error when not found to match Lotus behaviour.

scripts/tests/calibnet_delegated_wallet_check.sh

@@ -15,7 +15,7 @@ FIL_AMT="500 atto FIL"
 FIL_ZERO="0 FIL"
 
 # The preloaded address
-ADDR_ONE=$($FOREST_WALLET_PATH list | tail -1 | cut -d ' ' -f1)
+ADDR_ONE=$($FOREST_WALLET_PATH list | tail -1 | cut -d ' ' -f2)
 
 sleep 5s
 
@@ -30,7 +30,7 @@ $FOREST_WALLET_PATH export "$DELEGATE_ADDR_ONE" > delegated_wallet.key
 $FOREST_WALLET_PATH --remote-wallet import delegated_wallet.key
 
 # Fund delegated wallet from preloaded wallet
-DELEGATE_FUND_AMT="2 micro FIL"
+DELEGATE_FUND_AMT="3 micro FIL"
 $FOREST_WALLET_PATH set-default "$ADDR_ONE"
 MSG_DELEGATE_FUND=$($FOREST_WALLET_PATH send "$DELEGATE_ADDR_ONE" "$DELEGATE_FUND_AMT")
 : "$MSG_DELEGATE_FUND"
@@ -43,6 +43,10 @@ while [[ $i != 20 && $DELEGATE_ADDR_ONE_BALANCE == "$FIL_ZERO" ]]; do
   sleep 30s
   DELEGATE_ADDR_ONE_BALANCE=$($FOREST_WALLET_PATH balance "$DELEGATE_ADDR_ONE" --exact-balance)
 done
+if [[ $DELEGATE_ADDR_ONE_BALANCE == "$FIL_ZERO" ]]; then
+  echo "Timed out waiting for $DELEGATE_ADDR_ONE balance to update after 20 retries"
+  exit 1
+fi
 
 echo "Creating delegated wallet DELEGATE_ADDR_TWO"
 DELEGATE_ADDR_TWO=$($FOREST_WALLET_PATH new delegated)
@@ -71,6 +75,10 @@ while [[ $i != 20 && $DELEGATE_ADDR_TWO_BALANCE == "$FIL_ZERO" ]]; do
   sleep 30s
   DELEGATE_ADDR_TWO_BALANCE=$($FOREST_WALLET_PATH balance "$DELEGATE_ADDR_TWO" --exact-balance)
 done
+if [[ $DELEGATE_ADDR_TWO_BALANCE == "$FIL_ZERO" ]]; then
+  echo "Timed out waiting for $DELEGATE_ADDR_TWO balance to update after 20 retries"
+  exit 1
+fi
 
 DELEGATE_ADDR_THREE_BALANCE=$FIL_ZERO
 i=0
@@ -80,6 +88,29 @@ while [[ $i != 20 && $DELEGATE_ADDR_THREE_BALANCE == "$FIL_ZERO" ]]; do
   sleep 30s
   DELEGATE_ADDR_THREE_BALANCE=$($FOREST_WALLET_PATH --remote-wallet balance "$DELEGATE_ADDR_THREE" --exact-balance)
 done
+if [[ $DELEGATE_ADDR_THREE_BALANCE == "$FIL_ZERO" ]]; then
+  echo "Timed out waiting for $DELEGATE_ADDR_THREE balance to update after 20 retries"
+  exit 1
+fi
+
+$FOREST_WALLET_PATH list
+$FOREST_WALLET_PATH --remote-wallet list
+
+MSG_DELEGATE_FOUR=$($FOREST_WALLET_PATH --remote-wallet send "$DELEGATE_ADDR_THREE" "$FIL_AMT")
+: "$MSG_DELEGATE_FOUR"
+
+DELEGATE_ADDR_REMOTE_THREE_BALANCE=$DELEGATE_ADDR_THREE_BALANCE
+i=0
+while [[ $i != 20 && $DELEGATE_ADDR_REMOTE_THREE_BALANCE == "$DELEGATE_ADDR_THREE_BALANCE" ]]; do
+  i=$((i+1))
+  : "Checking DELEGATE_ADDR_THREE balance $i/20"
+  sleep 30s
+  DELEGATE_ADDR_REMOTE_THREE_BALANCE=$($FOREST_WALLET_PATH --remote-wallet balance "$DELEGATE_ADDR_THREE" --exact-balance)
+done
+if [[ $DELEGATE_ADDR_REMOTE_THREE_BALANCE == "$DELEGATE_ADDR_THREE_BALANCE" ]]; then
+  echo "Timed out waiting for $DELEGATE_ADDR_THREE balance to update after 20 retries"
+  exit 1
+fi
 
 $FOREST_WALLET_PATH list
 $FOREST_WALLET_PATH --remote-wallet list

scripts/tests/calibnet_wallet_check.sh

@@ -18,7 +18,7 @@ forest_wallet_init "$@"
 # MARKET_FIL_AMT="23"
 # 
 # # The preloaded address
-# REMOTE_ADDR=$($FOREST_WALLET_PATH --remote-wallet list | tail -1 | cut -d ' ' -f1)
+# REMOTE_ADDR=$($FOREST_WALLET_PATH --remote-wallet list | tail -1 | cut -d ' ' -f2)
 # 
 # JSON=$(curl -s -X POST "$FOREST_URL" \
 #   --header 'Accept: application/json' \
@@ -73,7 +73,7 @@ FIL_AMT="500 atto FIL"
 FIL_ZERO="0 FIL"
 
 # The preloaded address
-ADDR_ONE=$($FOREST_WALLET_PATH list | tail -1 | cut -d ' ' -f1)
+ADDR_ONE=$($FOREST_WALLET_PATH list | tail -1 | cut -d ' ' -f2)
 
 sleep 5s
 
@@ -124,11 +124,14 @@ ADDR_TWO_BALANCE=$FIL_ZERO
 i=0
 while [[ $i != 20 && $ADDR_TWO_BALANCE == "$FIL_ZERO" ]]; do
   i=$((i+1))
-
   : "Checking balance $i/20"
   sleep 30s
   ADDR_TWO_BALANCE=$($FOREST_WALLET_PATH balance "$ADDR_TWO" --exact-balance)
 done
+if [[ $ADDR_TWO_BALANCE == "$FIL_ZERO" ]]; then
+  echo "Timed out waiting for $ADDR_TWO balance to update after 20 retries"
+  exit 1
+fi
 
 ADDR_THREE_BALANCE=$FIL_ZERO
 i=0
@@ -139,47 +142,58 @@ while [[ $i != 20 && $ADDR_THREE_BALANCE == "$FIL_ZERO" ]]; do
   sleep 30s
   ADDR_THREE_BALANCE=$($FOREST_WALLET_PATH --remote-wallet balance "$ADDR_THREE" --exact-balance)
 done
+if [[ $ADDR_THREE_BALANCE == "$FIL_ZERO" ]]; then
+  echo "Timed out waiting for $ADDR_THREE balance to update after 20 retries"
+  exit 1
+fi
+
+ETH_ADDR_TWO=$(curl -s -X POST "$FOREST_URL" \
+  -H 'Content-Type: application/json' \
+  -H "Authorization: Bearer $ADMIN_TOKEN" \
+  --data "$(jq -n --arg addr "$ADDR_TWO" '{jsonrpc: "2.0", id: 1, method: "Filecoin.FilecoinAddressToEthAddress", params: [$addr, "pending"]}')" \
+  | jq -r '.result')
+echo "ETH address: $ETH_ADDR_TWO"
+
+ETH_ADDR_THREE=$(curl -s -X POST "$FOREST_URL" \
+  -H 'Content-Type: application/json' \
+  -H "Authorization: Bearer $ADMIN_TOKEN" \
+  --data "$(jq -n --arg addr "$ADDR_THREE" '{jsonrpc: "2.0", id: 1, method: "Filecoin.FilecoinAddressToEthAddress", params: [$addr, "pending"]}')" \
+  | jq -r '.result')
+echo "ETH address: $ETH_ADDR_THREE"
+
+MSG_ETH=$($FOREST_WALLET_PATH send "$ETH_ADDR_TWO" "$FIL_AMT")
+: "$MSG_ETH"
 
-# Test commented out due to it being flaky. See the tracking issue: https://github.com/ChainSafe/forest/issues/4899
-# ETH_ADDR_TWO=$(curl -s -X POST "$FOREST_URL" \
-#   -H 'Content-Type: application/json' \
-#   -H "Authorization: Bearer $ADMIN_TOKEN" \
-#   --data "$(jq -n --arg addr "$ADDR_TWO" '{jsonrpc: "2.0", id: 1, method: "Filecoin.FilecoinAddressToEthAddress", params: [$addr, "pending"]}')" \
-#   | jq -r '.result')
-# echo "ETH address: $ETH_ADDR_TWO"
-
-# ETH_ADDR_THREE=$(curl -s -X POST "$FOREST_URL" \
-#   -H 'Content-Type: application/json' \
-#   -H "Authorization: Bearer $ADMIN_TOKEN" \
-#   --data "$(jq -n --arg addr "$ADDR_THREE" '{jsonrpc: "2.0", id: 1, method: "Filecoin.FilecoinAddressToEthAddress", params: [$addr, "pending"]}')" \
-#   | jq -r '.result')
-# echo "ETH address: $ETH_ADDR_THREE"
-
-# MSG_ETH=$($FOREST_WALLET_PATH send "$ETH_ADDR_TWO" "$FIL_AMT")
-# : "$MSG_ETH"
-
-# MSG_ETH_REMOTE=$($FOREST_WALLET_PATH --remote-wallet send "$ETH_ADDR_THREE" "$FIL_AMT")
-# : "$MSG_ETH_REMOTE"
-
-# ETH_ADDR_TWO_BALANCE=$ADDR_TWO_BALANCE
-# i=0
-# while [[ $i != 20 && $ETH_ADDR_TWO_BALANCE == "$ADDR_TWO_BALANCE" ]]; do
-#   i=$((i+1))
+MSG_ETH_REMOTE=$($FOREST_WALLET_PATH --remote-wallet send "$ETH_ADDR_THREE" "$FIL_AMT")
+: "$MSG_ETH_REMOTE"
+
+ETH_ADDR_TWO_BALANCE=$ADDR_TWO_BALANCE
+i=0
+while [[ $i != 20 && $ETH_ADDR_TWO_BALANCE == "$ADDR_TWO_BALANCE" ]]; do
+  i=$((i+1))
 
-#   : "Checking balance $i/20"
-#   sleep 30s
-#   ETH_ADDR_TWO_BALANCE=$($FOREST_WALLET_PATH balance "$ADDR_TWO" --exact-balance)
-# done
+  : "Checking balance $i/20"
+  sleep 30s
+  ETH_ADDR_TWO_BALANCE=$($FOREST_WALLET_PATH balance "$ADDR_TWO" --exact-balance)
+done
+if [[ $ETH_ADDR_TWO_BALANCE == "$ADDR_TWO_BALANCE" ]]; then
+  echo "Timed out waiting for $ETH_ADDR_TWO balance to update after 20 retries"
+  exit 1
+fi
 
-# ETH_ADDR_THREE_BALANCE=$ADDR_THREE_BALANCE
-# i=0
-# while [[ $i != 20 && $ETH_ADDR_THREE_BALANCE == "$ADDR_THREE_BALANCE" ]]; do
-#   i=$((i+1))
+ETH_ADDR_THREE_BALANCE=$ADDR_THREE_BALANCE
+i=0
+while [[ $i != 20 && $ETH_ADDR_THREE_BALANCE == "$ADDR_THREE_BALANCE" ]]; do
+  i=$((i+1))
 
-#   : "Checking balance $i/20"
-#   sleep 30s
-#   ETH_ADDR_THREE_BALANCE=$($FOREST_WALLET_PATH --remote-wallet balance "$ADDR_THREE" --exact-balance)
-# done
+  : "Checking balance $i/20"
+  sleep 30s
+  ETH_ADDR_THREE_BALANCE=$($FOREST_WALLET_PATH --remote-wallet balance "$ADDR_THREE" --exact-balance)
+done
+if [[ $ETH_ADDR_THREE_BALANCE == "$ADDR_THREE_BALANCE" ]]; then
+  echo "Timed out waiting for $ETH_ADDR_THREE balance to update after 20 retries"
+  exit 1
+fi
 
 # wallet list should contain address two with transferred FIL amount
 $FOREST_WALLET_PATH list

src/daemon/mod.rs

@@ -23,7 +23,7 @@ use crate::daemon::{
 use crate::db::gc::SnapshotGarbageCollector;
 use crate::db::ttl::EthMappingCollector;
 use crate::libp2p::{Libp2pService, PeerManager};
-use crate::message_pool::{MessagePool, MpoolConfig};
+use crate::message_pool::{MessagePool, MpoolConfig, MpoolLocker, NonceTracker};
 use crate::networks::{self, ChainConfig};
 use crate::rpc::RPCState;
 use crate::rpc::eth::filter::EthEventHandler;
@@ -399,6 +399,8 @@ fn maybe_start_rpc_service(
             let tipset_send = chain_follower.tipset_sender.clone();
             let keystore = ctx.keystore.clone();
             let snapshot_progress_tracker = ctx.snapshot_progress_tracker.clone();
+            let nonce_tracker = NonceTracker::new();
+            let mpool_locker = MpoolLocker::new();
             async move {
                 let rpc_listener = tokio::net::TcpListener::bind(rpc_address)
                     .await
@@ -419,6 +421,8 @@ fn maybe_start_rpc_service(
                         shutdown,
                         tipset_send,
                         snapshot_progress_tracker,
+                        mpool_locker,
+                        nonce_tracker,
                     },
                     rpc_listener,
                     rpc_stop_handle,