feat(abstract-utxo): filter input script types for legacy tx format

When using legacy tx format, restrict input selection to script types
that are compatible with the legacy signing flow (p2sh, p2shP2wsh, p2wsh).
This prevents errors when using newer script types like p2tr with legacy
transaction format.

Issue: BTC-2768

Co-authored-by: llm-git <llm-git@ttll.de>

Author: OttoAllmendinger

modules/abstract-utxo/src/abstractUtxoCoin.ts

@@ -1005,8 +1005,10 @@ export abstract class AbstractUtxoCoin
   async getExtraPrebuildParams(buildParams: ExtraPrebuildParamsOptions & { wallet: Wallet }): Promise<{
     txFormat?: TxFormat;
     changeAddressType?: ScriptType2Of3[] | ScriptType2Of3;
+    allowedInputScriptTypes?: ScriptType2Of3[];
   }> {
-    const txFormat = this.getDefaultTxFormat(buildParams.wallet, buildParams.txFormat as TxFormat | undefined);
+    const requestedFormat = buildParams.txFormat as TxFormat | undefined;
+    const txFormat = this.getDefaultTxFormat(buildParams.wallet, requestedFormat);
     let changeAddressType = buildParams.changeAddressType as ScriptType2Of3[] | ScriptType2Of3 | undefined;
 
     // if the addressType is not specified, we need to default to p2trMusig2 for testnet hot wallets for staged rollout of p2trMusig2
@@ -1019,9 +1021,26 @@ export abstract class AbstractUtxoCoin
       changeAddressType = ['p2trMusig2', 'p2wsh', 'p2shP2wsh', 'p2sh', 'p2tr'];
     }
 
+    // getHalfSignedLegacyFormat() only supports p2ms-based types (p2sh, p2shP2wsh, p2wsh).
+    // Filter change outputs and restrict input selection to these types.
+    const legacyCompatibleTypes: ScriptType2Of3[] = ['p2sh', 'p2shP2wsh', 'p2wsh'];
+    let allowedInputScriptTypes: ScriptType2Of3[] | undefined;
+
+    if (requestedFormat === 'legacy') {
+      allowedInputScriptTypes = legacyCompatibleTypes;
+      if (Array.isArray(changeAddressType)) {
+        changeAddressType = changeAddressType.filter((t): t is ScriptType2Of3 =>
+          legacyCompatibleTypes.includes(t as ScriptType2Of3)
+        );
+      } else if (changeAddressType !== undefined && !legacyCompatibleTypes.includes(changeAddressType)) {
+        changeAddressType = legacyCompatibleTypes;
+      }
+    }
+
     return {
       txFormat,
       changeAddressType,
+      allowedInputScriptTypes,
     };
   }
 

modules/abstract-utxo/test/unit/txFormat.ts

@@ -136,4 +136,46 @@ describe('txFormat', function () {
       requestedTxFormat: 'psbt-lite',
     });
   });
+
+  describe('getExtraPrebuildParams with legacy format', function () {
+    const legacyCompatibleTypes = ['p2sh', 'p2shP2wsh', 'p2wsh'];
+
+    it('should filter changeAddressType to legacy-compatible types for hot wallets', async function () {
+      for (const coin of utxoCoins) {
+        const wallet = createMockWallet(coin, { type: 'hot' });
+        const result = await coin.getExtraPrebuildParams({ txFormat: 'legacy', wallet } as any);
+        assert.ok(Array.isArray(result.changeAddressType), `${coin.getChain()}: changeAddressType should be an array`);
+        for (const t of result.changeAddressType as string[]) {
+          assert.ok(
+            legacyCompatibleTypes.includes(t),
+            `${coin.getChain()}: changeAddressType contains ${t} which is not legacy-compatible`
+          );
+        }
+      }
+    });
+
+    it('should set allowedInputScriptTypes to legacy-compatible types', async function () {
+      for (const coin of utxoCoins) {
+        const wallet = createMockWallet(coin, { type: 'hot' });
+        const result = await coin.getExtraPrebuildParams({ txFormat: 'legacy', wallet } as any);
+        assert.deepStrictEqual(
+          result.allowedInputScriptTypes,
+          legacyCompatibleTypes,
+          `${coin.getChain()}: allowedInputScriptTypes should be legacy-compatible`
+        );
+      }
+    });
+
+    it('should not set allowedInputScriptTypes when txFormat is not legacy', async function () {
+      for (const coin of utxoCoins) {
+        const wallet = createMockWallet(coin, { type: 'hot' });
+        const result = await coin.getExtraPrebuildParams({ wallet } as any);
+        assert.strictEqual(
+          result.allowedInputScriptTypes,
+          undefined,
+          `${coin.getChain()}: allowedInputScriptTypes should be undefined for default format`
+        );
+      }
+    });
+  });
 });

modules/sdk-core/src/bitgo/wallet/BuildParams.ts

@@ -31,6 +31,8 @@ export const BuildParamsUTXO = t.partial({
   enforceMinConfirmsForChange: t.unknown,
   /* legacy or psbt */
   txFormat: t.unknown,
+  /* restrict which input script types WP may select (e.g. for legacy format compatibility) */
+  allowedInputScriptTypes: t.unknown,
   maxChangeOutputs: t.unknown,
   /* rbf */
   rbfTxIds: t.array(t.string),