crypto/rsa: bound public modulus size to prevent verify-path CPU-exhaustion DoS

crypto/rsa/rsa.go

@@ -328,6 +328,12 @@ func checkKeySize(size int) error {
 // operation under a sync mechanism of their choice.
 var MaxPublicExponentBitLen = 1024
 
+// MaxPublicModulusBitLen bounds the bit length of the RSA public modulus N
+// accepted by public-key operations (verify/encrypt), gating the modular
+// exponentiation before it runs so an oversized modulus cannot cause a
+// CPU-exhaustion DoS. Parsing is unaffected. Default 16384; set to 0 to disable.
+var MaxPublicModulusBitLen = 16384
+
 func checkPublicKeySize(k *PublicKey) error {
 	if k.N == nil {
 		return errors.New("crypto/rsa: missing public modulus")
@@ -338,6 +344,9 @@ func checkPublicKeySize(k *PublicKey) error {
 	if MaxPublicExponentBitLen > 0 && k.E.BitLen() > MaxPublicExponentBitLen {
 		return errors.New("crypto/rsa: public exponent exceeds MaxPublicExponentBitLen")
 	}
+	if MaxPublicModulusBitLen > 0 && k.N.BitLen() > MaxPublicModulusBitLen {
+		return errors.New("crypto/rsa: public modulus exceeds MaxPublicModulusBitLen")
+	}
 	return checkKeySize(k.N.BitLen())
 }
 

crypto/rsa/rsa_test.go

@@ -18,6 +18,7 @@ import (
 	"slices"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/runZeroInc/excrypto/crypto"
 	"github.com/runZeroInc/excrypto/crypto/internal/boring"
@@ -1377,6 +1378,51 @@ func TestMaxPublicExponentBitLen(t *testing.T) {
 	}
 }
 
+// TestMaxPublicModulusBitLen verifies the DoS-mitigation knob: an oversized RSA
+// modulus is rejected by the verify path *before* the expensive modular
+// exponentiation runs, so a malicious certificate with a multi-megabit modulus
+// cannot pin a scan worker. Parsing/constructing the key is unaffected.
+func TestMaxPublicModulusBitLen(t *testing.T) {
+	prev := MaxPublicModulusBitLen
+	defer func() { MaxPublicModulusBitLen = prev }()
+	MaxPublicModulusBitLen = 16384
+
+	// Build a public key with a ~2,000,000-bit modulus. We never want the modexp
+	// to run on this, so the test must complete fast.
+	const nbits = 2_000_000
+	N := new(big.Int).Lsh(big.NewInt(1), nbits)
+	N.SetBit(N, nbits-1, 1) // full bit length
+	N.SetBit(N, 0, 1)       // odd
+	pub := &PublicKey{N: N, E: big.NewInt(65537)}
+
+	hashed := make([]byte, 32) // fake SHA-256 digest
+	sig := make([]byte, (nbits+7)/8)
+	sig[0] = 0x01
+
+	start := time.Now()
+	err := VerifyPKCS1v15(pub, crypto.SHA256, hashed, sig)
+	elapsed := time.Since(start)
+
+	if err == nil || !strings.Contains(err.Error(), "MaxPublicModulusBitLen") {
+		t.Fatalf("expected MaxPublicModulusBitLen error, got %v", err)
+	}
+	// The unmitigated modexp on a 2,000,000-bit modulus takes ~60s; the cap must
+	// reject it near-instantly. Allow generous slack for slow CI.
+	if elapsed > 2*time.Second {
+		t.Fatalf("oversized modulus was not rejected quickly: took %v", elapsed)
+	}
+
+	// Disabling the bound (0) restores the previous unbounded behavior for the
+	// size gate. We use a small modulus so the test stays fast.
+	MaxPublicModulusBitLen = 0
+	small := &PublicKey{N: new(big.Int).Lsh(big.NewInt(1), 2047), E: big.NewInt(65537)}
+	small.N.SetBit(small.N, 0, 1)
+	if err := VerifyPKCS1v15(small, crypto.SHA256, hashed, make([]byte, 256)); err != nil &&
+		strings.Contains(err.Error(), "MaxPublicModulusBitLen") {
+		t.Fatalf("MaxPublicModulusBitLen=0 should not gate: %v", err)
+	}
+}
+
 // TestPublicKeyDefensiveCopy verifies that mutating the caller's PublicKey
 // after performing an operation does not affect subsequent operations on the
 // same key via the precomputed FIPS state.