diff --git a/migration/1779381590531-RouteScryptEurViaUsdt.js b/migration/1779381590531-RouteScryptEurViaUsdt.js new file mode 100644 index 0000000000..2586cf4700 --- /dev/null +++ b/migration/1779381590531-RouteScryptEurViaUsdt.js @@ -0,0 +1,46 @@ +// Stop buying BTC directly on Scrypt. Convert incoming EUR on Scrypt to USDT instead +// and route USDT to Binance for further BTC acquisition (same pattern as CHF / Rule 312). +// +// Reverts the routing introduced by migration 1774100000000-AddScryptSellIfDeficitAction.js: +// 1. Re-points Rule 313 (Scrypt/EUR redundancy) from Action 261 back to Action 233 +// (Scrypt sell USDT) — already in use by Rule 312 (CHF) and known good. +// 2. Removes Action 261 (Scrypt sell-if-deficit BTC), which is no longer referenced. +// +// Verified on 2026-05-21: no other action references 261 via onSuccessId/onFailId. +// Rule 314 (Scrypt/BTC withdraw) is kept as cleanup path for any residual BTC on Scrypt. +module.exports = class RouteScryptEurViaUsdt1779381590531 { + name = 'RouteScryptEurViaUsdt1779381590531'; + + async up(queryRunner) { + // Skip if rule 313 or action 233 don't exist in this environment + const [rule] = await queryRunner.query(`SELECT "id" FROM "liquidity_management_rule" WHERE "id" = 313`); + if (!rule) return; + + const [fallbackAction] = await queryRunner.query( + `SELECT "id" FROM "liquidity_management_action" WHERE "id" = 233`, + ); + if (!fallbackAction) return; + + // 1. Point Rule 313 (Scrypt/EUR redundancy) at the USDT sell action + await queryRunner.query(`UPDATE "liquidity_management_rule" SET "redundancyStartActionId" = 233 WHERE "id" = 313`); + + // 2. Remove the now-unreferenced sell-if-deficit BTC action + await queryRunner.query( + `DELETE FROM "liquidity_management_action" WHERE "id" = 261 AND "system" = 'Scrypt' AND "command" = 'sell-if-deficit'`, + ); + } + + async down(queryRunner) { + // Restore Action 261 (Scrypt sell-if-deficit BTC) and re-point Rule 313 at it. + const [fallbackAction] = await queryRunner.query( + `SELECT "id" FROM "liquidity_management_action" WHERE "id" = 233`, + ); + if (!fallbackAction) return; + + await queryRunner.query( + `INSERT INTO "liquidity_management_action" ("id", "system", "command", "tag", "params", "onSuccessId", "onFailId") VALUES (261, 'Scrypt', 'sell-if-deficit', 'SCRYPT BTC', '{"tradeAsset":"BTC","checkAssetId":113}', NULL, 233)`, + ); + + await queryRunner.query(`UPDATE "liquidity_management_rule" SET "redundancyStartActionId" = 261 WHERE "id" = 313`); + } +}; diff --git a/migration/1779812989037-FixNullableUniqueIndexes.js b/migration/1779812989037-FixNullableUniqueIndexes.js new file mode 100644 index 0000000000..bd62eec495 --- /dev/null +++ b/migration/1779812989037-FixNullableUniqueIndexes.js @@ -0,0 +1,36 @@ +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + * @typedef {import('typeorm').QueryRunner} QueryRunner + */ + +/** + * @class + * @implements {MigrationInterface} + */ +module.exports = class FixNullableUniqueIndexes1779812989037 { + name = 'FixNullableUniqueIndexes1779812989037' + + /** + * @param {QueryRunner} queryRunner + */ + async up(queryRunner) { + // kyc_step: recreate unique index with NULLS NOT DISTINCT so NULL type is treated as equal + await queryRunner.query(`DROP INDEX "IDX_3a1150791476264753a67212a1"`); + await queryRunner.query(`CREATE UNIQUE INDEX "IDX_3a1150791476264753a67212a1" ON "kyc_step" ("userDataId", "name", "type", "sequenceNumber") NULLS NOT DISTINCT`); + + // user_data: recreate unique index with NULLS NOT DISTINCT so NULL nationality is treated as equal + await queryRunner.query(`DROP INDEX "IDX_99da8fce0c522a35d93b9499f4"`); + await queryRunner.query(`CREATE UNIQUE INDEX "IDX_99da8fce0c522a35d93b9499f4" ON "user_data" ("identDocumentId", "nationalityId", "accountType", "kycType") NULLS NOT DISTINCT WHERE "identDocumentId" IS NOT NULL AND "accountType" IS NOT NULL AND "kycType" IS NOT NULL`); + } + + /** + * @param {QueryRunner} queryRunner + */ + async down(queryRunner) { + await queryRunner.query(`DROP INDEX "IDX_99da8fce0c522a35d93b9499f4"`); + await queryRunner.query(`CREATE UNIQUE INDEX "IDX_99da8fce0c522a35d93b9499f4" ON "user_data" ("identDocumentId", "nationalityId", "accountType", "kycType") WHERE "identDocumentId" IS NOT NULL AND "accountType" IS NOT NULL AND "kycType" IS NOT NULL`); + + await queryRunner.query(`DROP INDEX "IDX_3a1150791476264753a67212a1"`); + await queryRunner.query(`CREATE UNIQUE INDEX "IDX_3a1150791476264753a67212a1" ON "kyc_step" ("userDataId", "name", "type", "sequenceNumber")`); + } +} diff --git a/migration/1780071506610-AddPayoutOrderRetryTracking.js b/migration/1780071506610-AddPayoutOrderRetryTracking.js new file mode 100644 index 0000000000..9a0427ac13 --- /dev/null +++ b/migration/1780071506610-AddPayoutOrderRetryTracking.js @@ -0,0 +1,30 @@ +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + * @typedef {import('typeorm').QueryRunner} QueryRunner + */ + +/** + * @class + * @implements {MigrationInterface} + */ +module.exports = class AddPayoutOrderRetryTracking1780071506610 { + name = 'AddPayoutOrderRetryTracking1780071506610' + + /** + * @param {QueryRunner} queryRunner + */ + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "payout_order" ADD "retryCount" integer NOT NULL DEFAULT 0`); + await queryRunner.query(`ALTER TABLE "payout_order" ADD "lastError" character varying(2048)`); + await queryRunner.query(`ALTER TABLE "payout_order" ADD "lastAttemptDate" TIMESTAMP`); + } + + /** + * @param {QueryRunner} queryRunner + */ + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "payout_order" DROP COLUMN "lastAttemptDate"`); + await queryRunner.query(`ALTER TABLE "payout_order" DROP COLUMN "lastError"`); + await queryRunner.query(`ALTER TABLE "payout_order" DROP COLUMN "retryCount"`); + } +} diff --git a/src/shared/utils/__tests__/migration-psql-check.spec.ts b/src/shared/utils/__tests__/migration-psql-check.spec.ts new file mode 100644 index 0000000000..6215c606e8 --- /dev/null +++ b/src/shared/utils/__tests__/migration-psql-check.spec.ts @@ -0,0 +1,66 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +/** + * Scans migration files for MSSQL-specific syntax that won't work on PostgreSQL. + * + * Since the PSQL migration (#3620), all new migrations must use PostgreSQL syntax. + * This test catches common MSSQL patterns that would fail silently or break on deploy. + */ +describe('Migration PostgreSQL Compatibility', () => { + const migrationDir = path.join(__dirname, '..', '..', '..', '..', 'migration'); + + // Migrations written before the PSQL migration are exempt + const psqlMigrationTimestamp = 1779802432879; // AddForeignKeyIndexes (last known PSQL migration) + + const mssqlPatterns: { pattern: RegExp; description: string }[] = [ + { pattern: /"dbo"\./i, description: 'MSSQL schema prefix "dbo."' }, + { pattern: /IDENTITY_INSERT/i, description: 'MSSQL IDENTITY_INSERT' }, + { pattern: /\bTOP\s+\d+/i, description: 'MSSQL TOP N (use LIMIT instead)' }, + { pattern: /\bNVARCHAR\b/i, description: 'MSSQL NVARCHAR (use VARCHAR or TEXT instead)' }, + { pattern: /\bDATETIME2\b/i, description: 'MSSQL DATETIME2 (use TIMESTAMP instead)' }, + { pattern: /\bGETDATE\s*\(\)/i, description: 'MSSQL GETDATE() (use NOW() instead)' }, + { pattern: /\bIDENTITY\s*\(\s*\d+\s*,\s*\d+\s*\)/i, description: 'MSSQL IDENTITY(1,1) (use SERIAL instead)' }, + { pattern: /\[(\w+)\]/g, description: 'MSSQL bracket quoting [column] (use "column" instead)' }, + ]; + + const getMigrationFiles = (): { name: string; content: string; timestamp: number }[] => { + if (!fs.existsSync(migrationDir)) return []; + + return fs + .readdirSync(migrationDir) + .filter((f) => f.endsWith('.js') && /^\d+/.test(f)) + .map((f) => { + const timestamp = parseInt(f.split('-')[0], 10); + return { + name: f, + content: fs.readFileSync(path.join(migrationDir, f), 'utf-8'), + timestamp, + }; + }) + .filter((f) => f.timestamp > psqlMigrationTimestamp); + }; + + it('should not contain MSSQL syntax in new migrations', () => { + const files = getMigrationFiles(); + const issues: string[] = []; + + for (const file of files) { + for (const { pattern, description } of mssqlPatterns) { + // Reset lastIndex for global patterns + pattern.lastIndex = 0; + + if (pattern.test(file.content)) { + issues.push(` ${file.name}: ${description}`); + } + } + } + + if (issues.length > 0) { + throw new Error( + `Found MSSQL syntax in post-PSQL migrations:\n${issues.join('\n')}\n\n` + + `All migrations after the PSQL migration must use PostgreSQL syntax.`, + ); + } + }); +}); diff --git a/src/subdomains/core/liquidity-management/adapters/actions/scrypt.adapter.ts b/src/subdomains/core/liquidity-management/adapters/actions/scrypt.adapter.ts index 207160051c..12032dee0c 100644 --- a/src/subdomains/core/liquidity-management/adapters/actions/scrypt.adapter.ts +++ b/src/subdomains/core/liquidity-management/adapters/actions/scrypt.adapter.ts @@ -1,4 +1,4 @@ -import { Inject, Injectable, forwardRef } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; import { ScryptOrderInfo, ScryptOrderSide, ScryptTransactionStatus } from 'src/integration/exchange/dto/scrypt.dto'; import { TradeChangedException } from 'src/integration/exchange/exceptions/trade-changed.exception'; @@ -8,28 +8,20 @@ import { Asset } from 'src/shared/models/asset/asset.entity'; import { AssetService } from 'src/shared/models/asset/asset.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Util } from 'src/shared/utils/util'; -import { BuyCryptoService } from 'src/subdomains/core/buy-crypto/process/services/buy-crypto.service'; import { DexService } from 'src/subdomains/supporting/dex/services/dex.service'; -import { - PriceCurrency, - PriceValidity, - PricingService, -} from 'src/subdomains/supporting/pricing/services/pricing.service'; +import { PriceValidity, PricingService } from 'src/subdomains/supporting/pricing/services/pricing.service'; import { LiquidityManagementOrder } from '../../entities/liquidity-management-order.entity'; import { LiquidityManagementSystem } from '../../enums'; import { OrderFailedException } from '../../exceptions/order-failed.exception'; import { OrderNotProcessableException } from '../../exceptions/order-not-processable.exception'; import { Command, CorrelationId } from '../../interfaces'; -import { LiquidityBalanceRepository } from '../../repositories/liquidity-balance.repository'; import { LiquidityManagementOrderRepository } from '../../repositories/liquidity-management-order.repository'; -import { LiquidityManagementRuleRepository } from '../../repositories/liquidity-management-rule.repository'; import { LiquidityActionAdapter } from './base/liquidity-action.adapter'; export enum ScryptAdapterCommands { WITHDRAW = 'withdraw', SELL = 'sell', BUY = 'buy', - SELL_IF_DEFICIT = 'sell-if-deficit', } @Injectable() @@ -44,16 +36,12 @@ export class ScryptAdapter extends LiquidityActionAdapter { private readonly orderRepo: LiquidityManagementOrderRepository, private readonly pricingService: PricingService, private readonly assetService: AssetService, - private readonly ruleRepo: LiquidityManagementRuleRepository, - private readonly balanceRepo: LiquidityBalanceRepository, - @Inject(forwardRef(() => BuyCryptoService)) private readonly buyCryptoService: BuyCryptoService, ) { super(LiquidityManagementSystem.SCRYPT); this.commands.set(ScryptAdapterCommands.WITHDRAW, this.withdraw.bind(this)); this.commands.set(ScryptAdapterCommands.SELL, this.sell.bind(this)); this.commands.set(ScryptAdapterCommands.BUY, this.buy.bind(this)); - this.commands.set(ScryptAdapterCommands.SELL_IF_DEFICIT, this.sellIfDeficit.bind(this)); } async checkCompletion(order: LiquidityManagementOrder): Promise { @@ -67,9 +55,6 @@ export class ScryptAdapter extends LiquidityActionAdapter { case ScryptAdapterCommands.BUY: return this.checkBuyCompletion(order); - case ScryptAdapterCommands.SELL_IF_DEFICIT: - return this.checkSellCompletion(order); - default: return false; } @@ -84,9 +69,6 @@ export class ScryptAdapter extends LiquidityActionAdapter { case ScryptAdapterCommands.BUY: return this.validateTradeParams(params); - case ScryptAdapterCommands.SELL_IF_DEFICIT: - return this.validateSellIfDeficitParams(params); - default: throw new Error(`Command ${command} not supported by ScryptAdapter`); } @@ -128,6 +110,14 @@ export class ScryptAdapter extends LiquidityActionAdapter { private async sell(order: LiquidityManagementOrder): Promise { const { tradeAsset, maxPriceDeviation } = this.parseTradeParams(order.action.paramMap); + // Structural guard: Scrypt BTC/EUR (and other Scrypt BTC pairs) have materially worse spreads + // than Scrypt USDT pairs. BTC acquisition on Scrypt is no longer supported — route via Binance USDT. + if (tradeAsset === 'BTC') { + throw new OrderNotProcessableException( + 'Scrypt: buying BTC is no longer supported (tradeAsset=BTC). Route via Binance USDT instead.', + ); + } + const targetAsset = order.pipeline.rule.targetAsset; const tradeAssetEntity = await this.assetService.getAssetByUniqueName(`Scrypt/${tradeAsset}`); @@ -151,6 +141,15 @@ export class ScryptAdapter extends LiquidityActionAdapter { const { tradeAsset, maxPriceDeviation } = this.parseTradeParams(order.action.paramMap); const targetAssetEntity = order.pipeline.rule.targetAsset; + + // Structural guard: Scrypt BTC pairs have materially worse spreads than USDT pairs. + // BTC acquisition on Scrypt is no longer supported — route via Binance USDT instead. + if (targetAssetEntity.dexName === 'BTC') { + throw new OrderNotProcessableException( + 'Scrypt: buying BTC is no longer supported (targetAsset=BTC). Route via Binance USDT instead.', + ); + } + const tradeAssetEntity = await this.assetService.getAssetByUniqueName(`Scrypt/${tradeAsset}`); const price = await this.getAndCheckTradePrice(tradeAssetEntity, targetAssetEntity, maxPriceDeviation); @@ -184,57 +183,6 @@ export class ScryptAdapter extends LiquidityActionAdapter { } } - private async sellIfDeficit(order: LiquidityManagementOrder): Promise { - const { tradeAsset, checkAssetId, maxPriceDeviation } = this.parseSellIfDeficitParams(order.action.paramMap); - - // Check if the referenced asset has a deficit or pending liquidity demand - const checkRule = await this.ruleRepo.findOneBy({ targetAsset: { id: checkAssetId } }); - if (!checkRule) { - throw new OrderNotProcessableException(`No rule found for asset ${checkAssetId}`); - } - - const checkAsset = checkRule.targetAsset; - const checkBalance = await this.balanceRepo.findOneBy({ asset: { id: checkAssetId } }); - - // Convert pending demand from CHF to check asset - const pendingDemandChf = await this.buyCryptoService.getPendingLiquidityDemandChf(checkAssetId); - const chfPrice = await this.pricingService.getPrice(PriceCurrency.CHF, checkAsset, PriceValidity.VALID_ONLY); - const pendingDemand = chfPrice.convert(pendingDemandChf, 8); - - const effectiveBalance = (checkBalance?.amount ?? 0) - pendingDemand; - const hasDeficit = effectiveBalance < (checkRule.minimal ?? 0); - - if (!hasDeficit) { - throw new OrderNotProcessableException( - `No deficit for asset ${checkAssetId} (balance: ${checkBalance?.amount}, pending: ${pendingDemand}, effective: ${effectiveBalance}, minimal: ${checkRule.minimal})`, - ); - } - - // Calculate how much of the trade asset is needed to reach optimal (accounting for pending demand) - const deficitAmount = (checkRule.optimal ?? 0) - effectiveBalance; - if (deficitAmount <= 0) { - throw new OrderNotProcessableException(`No deficit to optimal for asset ${checkAssetId}`); - } - - const targetAsset = order.pipeline.rule.targetAsset; - const tradeAssetEntity = await this.assetService.getAssetByUniqueName(`Scrypt/${tradeAsset}`); - - const price = await this.getAndCheckTradePrice(targetAsset, tradeAssetEntity, maxPriceDeviation); - const availableBalance = await this.scryptService.getAvailableBalance(targetAsset.dexName); - - // price = targetAsset per tradeAsset (e.g., EUR per BTC) - const sellAmount = Util.floor(deficitAmount * price, 6); - const amount = Util.floor(Math.min(sellAmount, order.maxAmount, availableBalance), 6); - - if (amount <= 0) { - throw new OrderNotProcessableException( - `Scrypt: insufficient amount for sell-if-deficit (needed: ${sellAmount}, available: ${availableBalance})`, - ); - } - - return this.executeSell(order, amount, targetAsset.dexName, tradeAsset); - } - // --- COMPLETION CHECKS --- // private async checkWithdrawCompletion(order: LiquidityManagementOrder): Promise { @@ -389,30 +337,6 @@ export class ScryptAdapter extends LiquidityActionAdapter { return { tradeAsset, maxPriceDeviation }; } - private validateSellIfDeficitParams(params: Record): boolean { - try { - this.parseSellIfDeficitParams(params); - return true; - } catch { - return false; - } - } - - private parseSellIfDeficitParams(params: Record): { - tradeAsset: string; - checkAssetId: number; - maxPriceDeviation?: number; - } { - const { tradeAsset, maxPriceDeviation } = this.parseTradeParams(params); - const checkAssetId = params.checkAssetId as number | undefined; - - if (!checkAssetId) { - throw new Error('Params provided to ScryptAdapter sell-if-deficit command are missing checkAssetId.'); - } - - return { tradeAsset, checkAssetId, maxPriceDeviation }; - } - // --- HELPER METHODS --- // private async executeSell( diff --git a/src/subdomains/generic/kyc/dto/mapper/kyc-info.mapper.ts b/src/subdomains/generic/kyc/dto/mapper/kyc-info.mapper.ts index ddf86420b4..21508f2845 100644 --- a/src/subdomains/generic/kyc/dto/mapper/kyc-info.mapper.ts +++ b/src/subdomains/generic/kyc/dto/mapper/kyc-info.mapper.ts @@ -3,7 +3,7 @@ import { Util } from 'src/shared/utils/util'; import { Wallet } from 'src/subdomains/generic/user/models/wallet/wallet.entity'; import { UserData } from '../../../user/models/user-data/user-data.entity'; import { KycStep } from '../../entities/kyc-step.entity'; -import { KycStepName } from '../../enums/kyc-step-name.enum'; +import { KycStepName, KycStepRepeatable } from '../../enums/kyc-step-name.enum'; import { KycStepType, getKycStepIndex, getKycTypeIndex, requiredKycSteps } from '../../enums/kyc.enum'; import { ReviewStatus } from '../../enums/review-status.enum'; import { KycLevelDto, KycProcessStatus, KycSessionDto } from '../output/kyc-info.dto'; @@ -110,8 +110,10 @@ export class KycInfoMapper { }, new Map()); const visibleSteps = Array.from(groupedSteps.values()).map((steps) => { - const completedSteps = steps.filter((s) => s.isCompleted); - return Util.maxObj(completedSteps.length ? completedSteps : steps, 'sequenceNumber'); + const completedAndInProgressSteps = steps.filter( + (s) => s.isCompleted || (s.isInProgress && KycStepRepeatable.includes(s.name)), + ); + return Util.maxObj(completedAndInProgressSteps.length ? completedAndInProgressSteps : steps, 'sequenceNumber'); }); return visibleSteps.sort((a, b) => { diff --git a/src/subdomains/generic/kyc/enums/kyc-step-name.enum.ts b/src/subdomains/generic/kyc/enums/kyc-step-name.enum.ts index c6fa2f417a..60d66627eb 100644 --- a/src/subdomains/generic/kyc/enums/kyc-step-name.enum.ts +++ b/src/subdomains/generic/kyc/enums/kyc-step-name.enum.ts @@ -31,3 +31,9 @@ export enum KycStepName { } export const KycStepCancelable = [KycStepName.ADDRESS_CHANGE, KycStepName.PHONE_CHANGE, KycStepName.NAME_CHANGE]; +export const KycStepRepeatable = [ + KycStepName.ADDRESS_CHANGE, + KycStepName.PHONE_CHANGE, + KycStepName.NAME_CHANGE, + KycStepName.CONTACT_DATA, +]; diff --git a/src/subdomains/supporting/payin/entities/crypto-input.entity.ts b/src/subdomains/supporting/payin/entities/crypto-input.entity.ts index b7a6bead64..5d9410c52d 100644 --- a/src/subdomains/supporting/payin/entities/crypto-input.entity.ts +++ b/src/subdomains/supporting/payin/entities/crypto-input.entity.ts @@ -228,6 +228,15 @@ export class CryptoInput extends IEntity { return this; } + resetPreparation(): this { + this.prepareTxId = null; + this.forwardFeeAmount = null; + this.forwardFeeAmountChf = null; + this.status = PayInStatus.ACKNOWLEDGED; + + return this; + } + designateForward(forwardAddress: BlockchainAddress): this { this.destinationAddress = forwardAddress; diff --git a/src/subdomains/supporting/payin/strategies/send/impl/base/evm.strategy.ts b/src/subdomains/supporting/payin/strategies/send/impl/base/evm.strategy.ts index dd55ae9696..6ce314480f 100644 --- a/src/subdomains/supporting/payin/strategies/send/impl/base/evm.strategy.ts +++ b/src/subdomains/supporting/payin/strategies/send/impl/base/evm.strategy.ts @@ -37,6 +37,18 @@ export abstract class EvmStrategy extends SendStrategy { for (const payInGroup of [...groups.values()]) { try { if (payInGroup.status === PayInStatus.PREPARING) { + // Reset individual stale pay-ins whose prepare tx was never mined + for (const payIn of payInGroup.payIns) { + if (payIn.updated < Util.hoursBefore(1)) { + this.logger.warn(`Resetting stale Preparing input ${payIn.id} — prepare tx not found after 1h`); + payIn.resetPreparation(); + await this.payInRepo.save(payIn); + } + } + + // Re-check: if any pay-ins were reset, skip this group (they'll be re-grouped next cycle) + if (payInGroup.payIns.some((p) => p.status === PayInStatus.ACKNOWLEDGED)) continue; + const isReady = await this.checkPreparation(payInGroup); if (isReady) { diff --git a/src/subdomains/supporting/payout/entities/__mocks__/payout-order.entity.mock.ts b/src/subdomains/supporting/payout/entities/__mocks__/payout-order.entity.mock.ts index b7b3337e6e..9ed43af7bd 100644 --- a/src/subdomains/supporting/payout/entities/__mocks__/payout-order.entity.mock.ts +++ b/src/subdomains/supporting/payout/entities/__mocks__/payout-order.entity.mock.ts @@ -7,8 +7,19 @@ export function createDefaultPayoutOrder(): PayoutOrder { } export function createCustomPayoutOrder(customValues: Partial): PayoutOrder { - const { id, context, correlationId, chain, asset, amount, destinationAddress, status, transferTxId, payoutTxId } = - customValues; + const { + id, + context, + correlationId, + chain, + asset, + amount, + destinationAddress, + status, + transferTxId, + payoutTxId, + retryCount, + } = customValues; const keys = Object.keys(customValues); const entity = new PayoutOrder(); @@ -23,6 +34,7 @@ export function createCustomPayoutOrder(customValues: Partial): Pay entity.status = keys.includes('status') ? status : PayoutOrderStatus.CREATED; entity.transferTxId = keys.includes('transferTxId') ? transferTxId : 'TTX_01'; entity.payoutTxId = keys.includes('payoutTxId') ? payoutTxId : 'PTX_01'; + entity.retryCount = keys.includes('retryCount') ? retryCount : 0; return entity; } diff --git a/src/subdomains/supporting/payout/entities/payout-order.entity.ts b/src/subdomains/supporting/payout/entities/payout-order.entity.ts index 7a4b9c0d8a..fd4204cbee 100644 --- a/src/subdomains/supporting/payout/entities/payout-order.entity.ts +++ b/src/subdomains/supporting/payout/entities/payout-order.entity.ts @@ -70,6 +70,15 @@ export class PayoutOrder extends IEntity { @Column({ type: 'float', nullable: true }) payoutFeeAmountChf?: number; + @Column({ type: 'int', default: 0 }) + retryCount: number; + + @Column({ length: 2048, nullable: true }) + lastError?: string; + + @Column({ type: 'timestamp', nullable: true }) + lastAttemptDate?: Date; + pendingPreparation(transferTxId: string): this { this.transferTxId = transferTxId; this.status = PayoutOrderStatus.PREPARATION_PENDING; @@ -143,6 +152,22 @@ export class PayoutOrder extends IEntity { return this; } + recordPayoutFailure(message: string): this { + this.retryCount = (this.retryCount ?? 0) + 1; + this.lastError = message?.substring(0, 2048); + this.lastAttemptDate = new Date(); + + return this; + } + + resetPayoutRetry(): this { + this.retryCount = 0; + this.lastError = null; + this.lastAttemptDate = null; + + return this; + } + //*** GETTERS ***// get payoutFee(): { asset: Asset; amount: number } { diff --git a/src/subdomains/supporting/payout/exceptions/invalid-payout-amount.exception.ts b/src/subdomains/supporting/payout/exceptions/invalid-payout-amount.exception.ts new file mode 100644 index 0000000000..84d8a39aeb --- /dev/null +++ b/src/subdomains/supporting/payout/exceptions/invalid-payout-amount.exception.ts @@ -0,0 +1,6 @@ +export class InvalidPayoutAmountException extends Error { + constructor(message: string) { + super(message); + this.name = 'InvalidPayoutAmountException'; + } +} diff --git a/src/subdomains/supporting/payout/services/__tests__/payout-bitcoin.service.spec.ts b/src/subdomains/supporting/payout/services/__tests__/payout-bitcoin.service.spec.ts new file mode 100644 index 0000000000..d39fab7137 --- /dev/null +++ b/src/subdomains/supporting/payout/services/__tests__/payout-bitcoin.service.spec.ts @@ -0,0 +1,138 @@ +/** + * Unit Tests for PayoutBitcoinService + * + * Cover amount sanitization, validation and defensive fee-rate rounding + * that protect the BTC payout pipeline from Bitcoin Core's strict + * ParseFixedPoint amount/fee_rate rejection ("Invalid amount", error -3). + */ + +import { BitcoinClient } from 'src/integration/blockchain/bitcoin/node/bitcoin-client'; +import { BitcoinFeeService } from 'src/integration/blockchain/bitcoin/services/bitcoin-fee.service'; +import { BitcoinService } from 'src/integration/blockchain/bitcoin/services/bitcoin.service'; +import { PayoutOrderContext } from '../../entities/payout-order.entity'; +import { InvalidPayoutAmountException } from '../../exceptions/invalid-payout-amount.exception'; +import { PayoutGroup } from '../base/payout-bitcoin-based.service'; +import { PayoutBitcoinService } from '../payout-bitcoin.service'; + +describe('PayoutBitcoinService', () => { + let service: PayoutBitcoinService; + let mockClient: jest.Mocked; + let mockFeeService: jest.Mocked; + let sendManySpy: jest.Mock; + + beforeEach(() => { + sendManySpy = jest.fn().mockResolvedValue('TX_HASH_01'); + + mockClient = { + sendMany: sendManySpy, + getInfo: jest.fn(), + getTx: jest.fn(), + } as unknown as jest.Mocked; + + const mockBitcoinService = { + getDefaultClient: jest.fn().mockReturnValue(mockClient), + } as unknown as jest.Mocked; + + mockFeeService = { + getSendFeeRate: jest.fn(), + } as unknown as jest.Mocked; + + service = new PayoutBitcoinService(mockBitcoinService, mockFeeService); + }); + + describe('sendUtxoToMany()', () => { + it('should quantize amounts to 8 decimals (strip JS float artifacts)', async () => { + // 1.000000003 has more than 8 decimal places and is observably distinct from 1 + // in IEEE 754 — Bitcoin Core would reject it as "Invalid amount". This is a + // stronger probe than 0.1 + 0.2 because the un-rounded raw value cannot + // collapse to the asserted post-round value by coincidence. + const rawAmount = 1.000000003; + const payout: PayoutGroup = [{ addressTo: 'ADDR_01', amount: rawAmount }]; + mockFeeService.getSendFeeRate.mockResolvedValueOnce(5); + + await service.sendUtxoToMany(PayoutOrderContext.BUY_CRYPTO, payout); + + expect(sendManySpy).toHaveBeenCalledTimes(1); + const [calledPayout] = sendManySpy.mock.calls[0]; + expect(calledPayout[0].amount).toBe(1); + expect(calledPayout[0].amount).not.toBe(rawAmount); + // Verify the post-round value has at most 8 decimals (the Bitcoin Core ceiling). + expect(Number.isInteger(calledPayout[0].amount * 1e8)).toBe(true); + }); + + it('should round fee rate to 3 decimals as defense-in-depth even if fee service regresses', async () => { + const payout: PayoutGroup = [{ addressTo: 'ADDR_01', amount: 0.5 }]; + // Simulate a value that slipped through the fee service un-rounded + mockFeeService.getSendFeeRate.mockResolvedValueOnce(3.8699999999999997); + + await service.sendUtxoToMany(PayoutOrderContext.BUY_CRYPTO, payout); + + expect(sendManySpy).toHaveBeenCalledWith([{ addressTo: 'ADDR_01', amount: 0.5 }], 3.87); + }); + + it('should reject NaN amounts with structured error before RPC call', async () => { + const payout: PayoutGroup = [{ addressTo: 'ADDR_BAD', amount: NaN }]; + mockFeeService.getSendFeeRate.mockResolvedValueOnce(5); + + await expect(service.sendUtxoToMany(PayoutOrderContext.BUY_CRYPTO, payout)).rejects.toThrow( + InvalidPayoutAmountException, + ); + expect(sendManySpy).not.toHaveBeenCalled(); + }); + + it('should reject zero amounts with structured error before RPC call', async () => { + const payout: PayoutGroup = [{ addressTo: 'ADDR_BAD', amount: 0 }]; + mockFeeService.getSendFeeRate.mockResolvedValueOnce(5); + + await expect(service.sendUtxoToMany(PayoutOrderContext.BUY_CRYPTO, payout)).rejects.toThrow( + InvalidPayoutAmountException, + ); + expect(sendManySpy).not.toHaveBeenCalled(); + }); + + it('should reject negative amounts with structured error before RPC call', async () => { + const payout: PayoutGroup = [{ addressTo: 'ADDR_BAD', amount: -1 }]; + mockFeeService.getSendFeeRate.mockResolvedValueOnce(5); + + await expect(service.sendUtxoToMany(PayoutOrderContext.BUY_CRYPTO, payout)).rejects.toThrow( + InvalidPayoutAmountException, + ); + expect(sendManySpy).not.toHaveBeenCalled(); + }); + + it('should reject infinite amounts with structured error before RPC call', async () => { + const payout: PayoutGroup = [{ addressTo: 'ADDR_BAD', amount: Infinity }]; + mockFeeService.getSendFeeRate.mockResolvedValueOnce(5); + + await expect(service.sendUtxoToMany(PayoutOrderContext.BUY_CRYPTO, payout)).rejects.toThrow( + InvalidPayoutAmountException, + ); + expect(sendManySpy).not.toHaveBeenCalled(); + }); + + it('should sanitize every entry in a multi-recipient payout group', async () => { + const payout: PayoutGroup = [ + { addressTo: 'ADDR_01', amount: 0.1 + 0.2 }, + { addressTo: 'ADDR_02', amount: 1.000000003 }, + ]; + mockFeeService.getSendFeeRate.mockResolvedValueOnce(5); + + await service.sendUtxoToMany(PayoutOrderContext.BUY_CRYPTO, payout); + + const [calledPayout] = sendManySpy.mock.calls[0]; + expect(calledPayout).toEqual([ + { addressTo: 'ADDR_01', amount: 0.3 }, + { addressTo: 'ADDR_02', amount: 1 }, + ]); + }); + + it('should propagate the RPC tx id on success', async () => { + const payout: PayoutGroup = [{ addressTo: 'ADDR_01', amount: 0.5 }]; + mockFeeService.getSendFeeRate.mockResolvedValueOnce(5); + + const result = await service.sendUtxoToMany(PayoutOrderContext.BUY_CRYPTO, payout); + + expect(result).toBe('TX_HASH_01'); + }); + }); +}); diff --git a/src/subdomains/supporting/payout/services/payout-bitcoin.service.ts b/src/subdomains/supporting/payout/services/payout-bitcoin.service.ts index 6f376c498e..28ba7bc8ff 100644 --- a/src/subdomains/supporting/payout/services/payout-bitcoin.service.ts +++ b/src/subdomains/supporting/payout/services/payout-bitcoin.service.ts @@ -2,9 +2,17 @@ import { Injectable } from '@nestjs/common'; import { BitcoinClient } from 'src/integration/blockchain/bitcoin/node/bitcoin-client'; import { BitcoinFeeService } from 'src/integration/blockchain/bitcoin/services/bitcoin-fee.service'; import { BitcoinNodeType, BitcoinService } from 'src/integration/blockchain/bitcoin/services/bitcoin.service'; +import { Util } from 'src/shared/utils/util'; import { PayoutOrderContext } from '../entities/payout-order.entity'; +import { InvalidPayoutAmountException } from '../exceptions/invalid-payout-amount.exception'; import { PayoutBitcoinBasedService, PayoutGroup } from './base/payout-bitcoin-based.service'; +// Bitcoin Core's send/sendmany RPC parses amount fields with ParseFixedPoint(decimals=8) +// and fee_rate with decimals=3. Values with more precision (e.g. 0.30000000000000004 +// from JS floating-point arithmetic) are rejected with "Invalid amount" (error code -3). +const BTC_AMOUNT_DECIMALS = 8; +const BTC_FEE_RATE_DECIMALS = 3; + @Injectable() export class PayoutBitcoinService extends PayoutBitcoinBasedService { private readonly client: BitcoinClient; @@ -27,8 +35,10 @@ export class PayoutBitcoinService extends PayoutBitcoinBasedService { } async sendUtxoToMany(_context: PayoutOrderContext, payout: PayoutGroup): Promise { - const feeRate = await this.getCurrentFeeRate(); - return this.client.sendMany(payout, feeRate); + const sanitizedPayout = this.sanitizePayoutAmounts(payout); + const feeRate = Util.round(await this.getCurrentFeeRate(), BTC_FEE_RATE_DECIMALS); + + return this.client.sendMany(sanitizedPayout, feeRate); } async getPayoutCompletionData(_context: any, payoutTxId: string): Promise<[boolean, number]> { @@ -45,4 +55,19 @@ export class PayoutBitcoinService extends PayoutBitcoinBasedService { async getCurrentFeeRate(): Promise { return this.feeService.getSendFeeRate(); } + + // Quantize each amount to 8 decimals before serializing to the RPC. Even though + // BitcoinBasedStrategy.aggregatePayout already rounds once, downstream fee + // adjustments and fixRoundingMismatch can re-introduce float artifacts. Reject + // NaN, infinite or non-positive amounts here so a bad input fails fast with a + // structured error instead of an opaque "Invalid amount" from the RPC. + private sanitizePayoutAmounts(payout: PayoutGroup): PayoutGroup { + return payout.map(({ addressTo, amount }) => { + if (!Number.isFinite(amount) || amount <= 0) { + throw new InvalidPayoutAmountException(`Invalid BTC payout amount for ${addressTo}: ${amount}`); + } + + return { addressTo, amount: Util.round(amount, BTC_AMOUNT_DECIMALS) }; + }); + } } diff --git a/src/subdomains/supporting/payout/strategies/payout/__tests__/payout-bitcoin-based.strategy.spec.ts b/src/subdomains/supporting/payout/strategies/payout/__tests__/payout-bitcoin-based.strategy.spec.ts index 51489ed933..0b1ff1e655 100644 --- a/src/subdomains/supporting/payout/strategies/payout/__tests__/payout-bitcoin-based.strategy.spec.ts +++ b/src/subdomains/supporting/payout/strategies/payout/__tests__/payout-bitcoin-based.strategy.spec.ts @@ -232,6 +232,111 @@ describe('PayoutBitcoinBasedStrategy', () => { }); }); + describe('#trackPayoutFailure(...)', () => { + it('increments retryCount, persists lastError and lastAttemptDate on every order', async () => { + const orders = [createCustomPayoutOrder({ id: 10 }), createCustomPayoutOrder({ id: 11 })]; + + await strategy.trackPayoutFailureWrapper(orders, new Error('Bitcoin RPC send failed: Invalid amount')); + + expect(orders[0].retryCount).toBe(1); + expect(orders[1].retryCount).toBe(1); + expect(orders[0].lastError).toBe('Bitcoin RPC send failed: Invalid amount'); + expect(orders[0].lastAttemptDate).toBeInstanceOf(Date); + expect(repoSaveSpy).toBeCalledTimes(2); + }); + + it('does not alert before the 5-attempt threshold', async () => { + const orders = [createCustomPayoutOrder({ id: 10, retryCount: 3 })]; + + await strategy.trackPayoutFailureWrapper(orders, new Error('Bitcoin RPC send failed: Invalid amount')); + + expect(orders[0].retryCount).toBe(4); + expect(sendErrorMailSpy).not.toBeCalled(); + }); + + it('fires the operator alert on the 5th attempt with 1h debounce (no suppressRecurring)', async () => { + const orders = [createCustomPayoutOrder({ id: 10, retryCount: 4 })]; + + await strategy.trackPayoutFailureWrapper(orders, new Error('Bitcoin RPC send failed: Invalid amount')); + + expect(orders[0].retryCount).toBe(5); + expect(sendErrorMailSpy).toBeCalledTimes(1); + expect(sendErrorMailSpy).toBeCalledWith( + expect.objectContaining({ + type: 'ErrorMonitoring', + context: 'Payout', + // Notification.isSuppressed short-circuits on `suppressRecurring`, so + // setting it would silence the debounce. Debounce alone gives the + // desired "1 alert per group per hour" semantic. + options: { debounce: 3600000 }, + correlationId: expect.stringContaining('PayoutOrderRecurringFailure'), + input: expect.objectContaining({ isLiqMail: true }), + }), + ); + }); + + it('builds a stable correlationId by sorting ids (PG row order is not guaranteed)', async () => { + const orders = [ + createCustomPayoutOrder({ id: 20, retryCount: 4 }), + createCustomPayoutOrder({ id: 11, retryCount: 4 }), + ]; + + await strategy.trackPayoutFailureWrapper(orders, new Error('Bitcoin RPC send failed: Invalid amount')); + + expect(sendErrorMailSpy).toBeCalledWith( + expect.objectContaining({ + correlationId: expect.stringMatching(/PayoutOrderRecurringFailure&BuyCrypto&11-20$/), + }), + ); + }); + + it('alerts on any single order over threshold (max semantic, not min)', async () => { + // Existing failing order at retryCount=4 (about to trip), plus a fresh + // order at retryCount=0 that joined this round. min would silence the + // alert (0 < 5); max keeps the operator informed. + const orders = [ + createCustomPayoutOrder({ id: 10, retryCount: 4 }), + createCustomPayoutOrder({ id: 11, retryCount: 0 }), + ]; + + await strategy.trackPayoutFailureWrapper(orders, new Error('Bitcoin RPC send failed: Invalid amount')); + + expect(orders[0].retryCount).toBe(5); + expect(orders[1].retryCount).toBe(1); + expect(sendErrorMailSpy).toBeCalledTimes(1); + }); + + it('returns early without saving or alerting on an empty orders array', async () => { + await strategy.trackPayoutFailureWrapper([], new Error('whatever')); + + expect(repoSaveSpy).not.toBeCalled(); + expect(sendErrorMailSpy).not.toBeCalled(); + }); + + it('truncates very long error messages to 2048 chars', async () => { + const longError = 'X'.repeat(5000); + const orders = [createCustomPayoutOrder({ id: 10 })]; + + await strategy.trackPayoutFailureWrapper(orders, new Error(longError)); + + expect(orders[0].lastError?.length).toBe(2048); + }); + }); + + describe('#resetPayoutRetry(...)', () => { + it('clears retryCount, lastError and lastAttemptDate on a previously failing order', () => { + const order = createCustomPayoutOrder({ id: 10, retryCount: 7 }); + order.lastError = 'Bitcoin RPC send failed: Invalid amount'; + order.lastAttemptDate = new Date(); + + order.resetPayoutRetry(); + + expect(order.retryCount).toBe(0); + expect(order.lastError).toBeNull(); + expect(order.lastAttemptDate).toBeNull(); + }); + }); + describe('#sendNonRecoverableErrorMailWrapper(...)', () => { it('combines custom message with error message', async () => { await strategy.sendNonRecoverableErrorMailWrapper( @@ -316,6 +421,10 @@ class PayoutBitcoinBasedStrategyWrapper extends BitcoinBasedStrategy { return this.sendNonRecoverableErrorMail(order, message, e); } + trackPayoutFailureWrapper(orders: PayoutOrder[], error: Error) { + return this.trackPayoutFailure(orders, error); + } + estimateFee(): Promise { throw new Error('Method not implemented.'); } diff --git a/src/subdomains/supporting/payout/strategies/payout/impl/base/bitcoin-based.strategy.ts b/src/subdomains/supporting/payout/strategies/payout/impl/base/bitcoin-based.strategy.ts index 83a7831027..396d78b6ac 100644 --- a/src/subdomains/supporting/payout/strategies/payout/impl/base/bitcoin-based.strategy.ts +++ b/src/subdomains/supporting/payout/strategies/payout/impl/base/bitcoin-based.strategy.ts @@ -15,6 +15,12 @@ import { PayoutOrder, PayoutOrderContext } from '../../../../entities/payout-ord import { PayoutOrderRepository } from '../../../../repositories/payout-order.repository'; import { PayoutStrategy } from './payout.strategy'; +// Operator-alert threshold for recurring payout RPC failures. With the ~30s payout +// cron interval, 5 attempts maps to ~2.5 min of silent retry-loop before the first +// notification fires. A 1h `debounce` on the Notification then throttles follow-ups +// so the operator inbox stays clean during long incidents. +const RECURRING_PAYOUT_FAILURE_THRESHOLD = 5; + export abstract class BitcoinBasedStrategy extends PayoutStrategy { protected abstract readonly logger: DfxLogger; @@ -145,6 +151,8 @@ export abstract class BitcoinBasedStrategy extends PayoutStrategy { e, ); + await this.trackPayoutFailure(orders, e); + if (e.message.includes('timeout')) throw e; await this.rollbackPayoutDesignation(orders); @@ -154,6 +162,7 @@ export abstract class BitcoinBasedStrategy extends PayoutStrategy { for (const order of orders) { try { + order.resetPayoutRetry(); const paidOrder = order.pendingPayout(payoutTxId); await this.payoutOrderRepo.save(paidOrder); } catch (e) { @@ -208,6 +217,51 @@ export abstract class BitcoinBasedStrategy extends PayoutStrategy { }); } + // Persistent retry tracking + threshold-based operator alert. Without this, an + // RPC error like "Invalid amount" silently loops every 30 s with no escalation + // (real incident 2026-05-29: 43 min stalled BTC payout, see PR #3729 context). + // Uses Math.max so a single stuck order escalates even when the group later + // gains fresh orders — the underlying RPC error fails the whole sendmany call, + // so any order over threshold is a signal worth surfacing. + protected async trackPayoutFailure(orders: PayoutOrder[], error: Error): Promise { + if (!orders.length) return; + + const message = error?.message ?? 'Unknown payout error'; + for (const order of orders) { + order.recordPayoutFailure(message); + await this.payoutOrderRepo.save(order); + } + + const maxRetryCount = Math.max(...orders.map((o) => o.retryCount)); + if (maxRetryCount >= RECURRING_PAYOUT_FAILURE_THRESHOLD) { + await this.sendRecurringPayoutFailureAlert(orders, message); + } + } + + protected async sendRecurringPayoutFailureAlert(orders: PayoutOrder[], lastError: string): Promise { + // Sort ids so the correlationId is stable across retries regardless of how + // Postgres returns the rows (findBy has no ORDER BY) — otherwise the 1h + // debounce on Notification keys on a moving correlationId and never matches. + const stableIds = orders.map((o) => o.id).sort((a, b) => a - b); + const maxRetry = Math.max(...orders.map((o) => o.retryCount)); + + await this.notificationService.sendMail({ + type: MailType.ERROR_MONITORING, + context: MailContext.PAYOUT, + input: { + subject: `Recurring ${orders[0].asset.name} payout failure: order(s) ${stableIds.join(', ')}`, + errors: [`Retry count: ${maxRetry}`, `Last error: ${lastError}`], + isLiqMail: true, + }, + // debounce only (no suppressRecurring): Notification.isSuppressed evaluates + // `suppressRecurring || isDebounced` and short-circuits — combining both + // would silence every retry forever. With debounce alone the operator gets + // one alert per hour per group while the incident continues. + options: { debounce: 3600000 }, + correlationId: `PayoutOrderRecurringFailure&${orders[0].context}&${stableIds.join('-')}`, + }); + } + //*** HELPER METHODS ***// private validateIfOrdersOfSameAsset(orders: PayoutOrder[]): boolean { diff --git a/src/subdomains/supporting/pricing/services/asset-prices-job.service.ts b/src/subdomains/supporting/pricing/services/asset-prices-job.service.ts index 1ef554191a..1bb7257838 100644 --- a/src/subdomains/supporting/pricing/services/asset-prices-job.service.ts +++ b/src/subdomains/supporting/pricing/services/asset-prices-job.service.ts @@ -38,7 +38,8 @@ export class AssetPricesJobService { updates.push(asset.updatePrice(usdPrice.convert(1), chfPrice.convert(1), eurPrice.convert(1))); - if ([AssetType.COIN, AssetType.TOKEN].includes(asset.type)) await this.saveAssetPrices(asset); + if ([AssetType.COIN, AssetType.TOKEN, AssetType.CUSTODY].includes(asset.type)) + await this.saveAssetPrices(asset); } catch (e) { const level = e instanceof PriceInvalidException ? LogLevel.INFO : LogLevel.ERROR; this.logger.log(level, `Failed to update price of asset ${asset.uniqueName}:`, e);