Expire local nonce after a certain timeout

We want to be able to recover from a situation when mempool crashes. To
achieve it, we cache the last local nonce value only for 5 seconds.

Author: pdyraga

pkg/chain/ethereum/ethutil/nonce.go

@@ -2,11 +2,17 @@ package ethutil
 
 import (
 	"context"
+	"time"
 
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
 )
 
+// The time for which the nonce value cached locally is valid. The local copy
+// is invalidated after the certain duration to let the nonce recover in case
+// the mempool crashed before propagating the last transaction sent.
+const localNonceTrustDuration = 5 * time.Second
+
 // NonceManager tracks the nonce for the account and allows to update it after
 // each successfully submitted transaction. Tracking the nonce locall is
 // required when transactions are submitted from multiple goroutines or when
@@ -23,9 +29,10 @@ import (
 // 4. Call IncrementNonce(),
 // 5. Release transaction lock.
 type NonceManager struct {
-	account    common.Address
-	transactor bind.ContractTransactor
-	localNonce uint64
+	account        common.Address
+	transactor     bind.ContractTransactor
+	localNonce     uint64
+	expirationDate time.Time
 }
 
 // NewNonceManager creates NonceManager instance for the provided account using
@@ -45,7 +52,9 @@ func NewNonceManager(
 
 // CurrentNonce returns the nonce value that should be used for the next
 // transaction. The nonce is evaluated as the higher value from the local
-// nonce and pending nonce fetched from the Ethereum client.
+// nonce and pending nonce fetched from the Ethereum client. The local nonce
+// is cached for the specific duration. If the local nonce expired, the pending
+// nonce returned from the chain is used.
 //
 // CurrentNonce is NOT safe for concurrent use. It is up to the code using this
 // function to provide the required synchronization, optionally including
@@ -59,17 +68,32 @@ func (nm *NonceManager) CurrentNonce() (uint64, error) {
 		return 0, err
 	}
 
+	now := time.Now()
+
 	if pendingNonce < nm.localNonce {
-		logger.Infof(
-			"local nonce [%v] is higher than pending [%v]; using the local one",
-			nm.localNonce,
-			pendingNonce,
-		)
+		if now.Before(nm.expirationDate) {
+			logger.Infof(
+				"local nonce [%v] is higher than pending [%v]; using the local one",
+				nm.localNonce,
+				pendingNonce,
+			)
+		} else {
+			logger.Infof(
+				"local nonce [%v] is higher than pending [%v] but local "+
+					"nonce expired; updating local nonce",
+				nm.localNonce,
+				pendingNonce,
+			)
+
+			nm.localNonce = pendingNonce
+		}
 	}
 
+	nm.expirationDate = now.Add(localNonceTrustDuration)
+
 	if pendingNonce > nm.localNonce {
 		logger.Infof(
-			"local nonce [%v] is lower than pending [%v]; updating",
+			"local nonce [%v] is lower than pending [%v]; updating local nonce",
 			nm.localNonce,
 			pendingNonce,
 		)

pkg/chain/ethereum/ethutil/nonce_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"math/big"
 	"testing"
+	"time"
 
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
@@ -14,35 +15,47 @@ func TestResolveAndIncrement(t *testing.T) {
 	tests := map[string]struct {
 		pendingNonce      uint64
 		localNonce        uint64
+		expirationDate    time.Time
 		expectedNonce     uint64
 		expectedNextNonce uint64
 	}{
 		"pending and local the same": {
 			pendingNonce:      10,
 			localNonce:        10,
+			expirationDate:    time.Now().Add(time.Second),
 			expectedNonce:     10,
 			expectedNextNonce: 11,
 		},
 		"pending nonce higher": {
 			pendingNonce:      121,
 			localNonce:        120,
+			expirationDate:    time.Now().Add(time.Second),
 			expectedNonce:     121,
 			expectedNextNonce: 122,
 		},
 		"pending nonce lower": {
 			pendingNonce:      110,
 			localNonce:        111,
+			expirationDate:    time.Now().Add(time.Second),
 			expectedNonce:     111,
 			expectedNextNonce: 112,
 		},
+		"pending nonce lower and local one expired": {
+			pendingNonce:      110,
+			localNonce:        111,
+			expirationDate:    time.Now().Add(-1 * time.Second),
+			expectedNonce:     110,
+			expectedNextNonce: 111,
+		},
 	}
 
 	for testName, test := range tests {
 		t.Run(testName, func(t *testing.T) {
 			transactor := &mockTransactor{test.pendingNonce}
 			manager := &NonceManager{
-				transactor: transactor,
-				localNonce: test.localNonce,
+				transactor:     transactor,
+				localNonce:     test.localNonce,
+				expirationDate: test.expirationDate,
 			}
 
 			nonce, err := manager.CurrentNonce()