feat(abstract-utxo): enforce deprecated tx format restrictions

Update `supportedTxFormats` default to disallow legacy format for
user inputs while maintaining backward compatibility for internal
operations like PSBT finalization. Update tests to reflect new
restrictions and skip deprecated legacy format test cases.

Issue: BTC-2768

Co-authored-by: llm-git <llm-git@ttll.de>

Author: OttoAllmendinger

modules/abstract-utxo/src/abstractUtxoCoin.ts

@@ -411,6 +411,11 @@ export abstract class AbstractUtxoCoin
 
   public readonly amountType: 'number' | 'bigint';
 
+  protected readonly supportedTxFormats: { readonly psbt: boolean; readonly legacy: boolean } = {
+    psbt: true,
+    legacy: false,
+  };
+
   protected constructor(bitgo: BitGoBase, amountType: 'number' | 'bigint' = 'number') {
     super(bitgo);
     this.amountType = amountType;
@@ -587,8 +592,15 @@ export abstract class AbstractUtxoCoin
     }
 
     if (utxolib.bitgo.isPsbt(input)) {
+      if (!this.supportedTxFormats.psbt) {
+        throw new ErrorDeprecatedTxFormat('psbt');
+      }
       return decodePsbtWith(input, this.name, decodeWith);
     } else {
+      // Legacy format transactions are deprecated. This will be an unconditional error in the future.
+      if (!this.supportedTxFormats.legacy) {
+        throw new ErrorDeprecatedTxFormat('legacy');
+      }
       if (decodeWith !== 'utxolib') {
         console.error('received decodeWith hint %s, ignoring for legacy transaction', decodeWith);
       }

modules/abstract-utxo/test/unit/customSigner.ts

@@ -83,20 +83,4 @@ describe('UTXO Custom Signer Function', function () {
     sinon.assert.calledOnce(customSigningFunction as sinon.SinonStub);
     scope.done();
   });
-
-  it('should use a custom signing function if provided for Tx without taprootKeyPathSpend input', async function () {
-    const tx = utxoLib.testutil.constructTxnBuilder(
-      [{ scriptType: 'p2wsh', value: BigInt(1000) }],
-      [{ scriptType: 'p2sh', value: BigInt(900) }],
-      basecoin.network,
-      rootWalletKey,
-      'unsigned'
-    );
-    const scope = nocks({ txHex: tx.buildIncomplete().toHex() });
-    const result = await wallet.sendMany({ recipients, customSigningFunction });
-
-    assertHasProperty(result, 'ok', true);
-    sinon.assert.calledOnce(customSigningFunction as sinon.SinonStub);
-    scope.done();
-  });
 });

modules/abstract-utxo/test/unit/signTransaction.ts

@@ -6,7 +6,7 @@ import nock = require('nock');
 import { testutil } from '@bitgo/utxo-lib';
 import { common, Triple } from '@bitgo/sdk-core';
 
-import { getReplayProtectionPubkeys } from '../../src';
+import { getReplayProtectionPubkeys, ErrorDeprecatedTxFormat } from '../../src';
 import type { Unspent } from '../../src/unspent';
 
 import { getUtxoWallet, getDefaultWalletKeys, getUtxoCoin, keychainsBase58, defaultBitGo } from './util';
@@ -170,7 +170,7 @@ describe('signTransaction', function () {
     }
   });
 
-  it('customSigningFunction flow - Network Tx', async function () {
+  it('customSigningFunction flow - Network Tx should reject legacy format', async function () {
     const inputs: testutil.TxnInput<bigint>[] = testutil.txnInputScriptTypes
       .filter((v) => v !== 'p2shP2pk')
       .map((scriptType) => ({
@@ -182,9 +182,10 @@ describe('signTransaction', function () {
     const txBuilder = testutil.constructTxnBuilder(inputs, outputs, coin.network, rootWalletKeys, 'unsigned');
     const unspents = inputs.map((v, i) => testutil.toTxnUnspent(v, i, coin.network, rootWalletKeys));
 
-    for (const v of [false, true]) {
-      await signTransaction(txBuilder.buildIncomplete(), v, unspents);
-    }
+    // Legacy format transactions are now deprecated and should throw ErrorDeprecatedTxFormat
+    await assert.rejects(async () => {
+      await signTransaction(txBuilder.buildIncomplete(), false, unspents);
+    }, ErrorDeprecatedTxFormat);
   });
 
   it('fails on PSBT cache miss', async function () {

modules/abstract-utxo/test/unit/transaction.ts

@@ -271,10 +271,7 @@ function run<TNumber extends number | bigint = number>(
         testPsbtValidSignatures(tx, signedBy);
         return;
       }
-      const unspents =
-        txFormat === 'psbt'
-          ? getUnspentsForPsbt().map((u) => ({ ...u, value: bitgo.toTNumber(u.value, amountType) as TNumber }))
-          : getUnspents();
+      const unspents = getUnspents();
       const prevOutputs = unspents.map(
         (u): utxolib.TxOutput<TNumber> => ({
           script: Buffer.from(wasmAddress.toOutputScriptWithCoin(u.address, coin.name)),
@@ -418,9 +415,7 @@ function run<TNumber extends number | bigint = number>(
 }
 
 function runTestForCoin(coin: AbstractUtxoCoin) {
-  (['legacy', 'psbt'] as const).forEach((txFormat) => {
-    run(coin, getScriptTypes(coin, txFormat), txFormat, { decodeWith: 'wasm-utxo' });
-  });
+  run(coin, getScriptTypes(coin, 'psbt'), 'psbt', { decodeWith: 'wasm-utxo' });
 }
 
 describe('Transaction Suite', function () {