feat(wasm-utxo): add utxolibCompat option for consistent size estimates

Adds compatibility mode to make input size estimates match @bitgo/unspents.
This allows wasm-utxo to produce the same transaction size estimates as
utxo-lib when needed, while still supporting accurate min/max bounds.

The key change is using 72-byte signatures for ECDSA inputs in compat mode
instead of 73-byte for the max calculation, which matches @bitgo/unspents
behavior. The implementation also replicates specific encoding choices
used in utxo-lib.

Issue: BTC-2908

Co-authored-by: llm-git <llm-git@ttll.de>

Author: OttoAllmendinger

package-lock.json

@@ -6,6 +6,17 @@ import { toOutputScriptWithCoin } from "../address.js";
 
 type FromInputParams = { chain: number; signPath?: SignPath } | { scriptType: InputScriptType };
 
+/**
+ * Options for input dimension calculation
+ */
+export type FromInputOptions = {
+  /**
+   * When true, use @bitgo/unspents-compatible signature sizes (72 bytes)
+   * for the "max" calculation instead of true maximum (73 bytes).
+   */
+  utxolibCompat?: boolean;
+};
+
 /**
  * Dimensions class for estimating transaction virtual size.
  *
@@ -39,13 +50,20 @@ export class Dimensions {
    * Create dimensions for a single input
    *
    * @param params - Either `{ chain, signPath? }` or `{ scriptType }`
+   * @param options - Optional settings like `{ utxolibCompat: true }` for @bitgo/unspents-compatible sizing
    */
-  static fromInput(params: FromInputParams): Dimensions {
+  static fromInput(params: FromInputParams, options?: FromInputOptions): Dimensions {
+    const compat = options?.utxolibCompat;
     if ("scriptType" in params) {
-      return new Dimensions(WasmDimensions.from_input_script_type(params.scriptType));
+      return new Dimensions(WasmDimensions.from_input_script_type(params.scriptType, compat));
     }
     return new Dimensions(
-      WasmDimensions.from_input(params.chain, params.signPath?.signer, params.signPath?.cosigner),
+      WasmDimensions.from_input(
+        params.chain,
+        params.signPath?.signer,
+        params.signPath?.cosigner,
+        compat,
+      ),
     );
   }
 

packages/wasm-utxo/js/fixedScriptWallet/Dimensions.ts

@@ -52,6 +52,7 @@
     "lint:fix": "eslint . --fix"
   },
   "devDependencies": {
+    "@bitgo/unspents": "^0.50.13",
     "@bitgo/utxo-lib": "^10.1.0",
     "@eslint/js": "^9.17.0",
     "@types/mocha": "^10.0.7",