From c652a4028ab7a6b42e75ba513ff2fce4de3bb18f Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 8 Jun 2026 17:28:17 +0200 Subject: [PATCH] fix(log): reconcile FinancialDataLog snapshot validity against operating result The total-balance metric is built by live-polling many balance sources (exchanges, chain nodes, OTC custody). A failed or partial read can be persisted as 0 and a glitchy source can report an implausibly large value, so the logged total drifts far from reality in both directions (observed swings from ~0 to >1,000,000 CHF) and the legacy `valid` guard let these through via its unconditional 15-minute escape. A snapshot's total only legitimately moves by operating result + FX. Capture the month-to-date operating result (changeLog.total) in each snapshot and mark a snapshot valid only when its total-balance delta reconciles with the operating delta within financeLogTotalBalanceChangeLimit (FX/rounding absorbed). Glitches that break this reconciliation are flagged invalid and excluded from the valid-filtered series. The month-to-date reset is handled by falling back to the legacy escape when no same-month delta is available. This only sets the `valid` flag; no balance value is altered. --- .../log/__tests__/log-job.service.spec.ts | 40 +++++++++++++++++++ src/subdomains/supporting/log/dto/log.dto.ts | 4 ++ .../supporting/log/log-job.service.ts | 40 +++++++++++++++++-- 3 files changed, 80 insertions(+), 4 deletions(-) diff --git a/src/subdomains/supporting/log/__tests__/log-job.service.spec.ts b/src/subdomains/supporting/log/__tests__/log-job.service.spec.ts index 1d5a7d7c10..61915f3bb1 100644 --- a/src/subdomains/supporting/log/__tests__/log-job.service.spec.ts +++ b/src/subdomains/supporting/log/__tests__/log-job.service.spec.ts @@ -24,6 +24,7 @@ import { createCustomBankTx } from '../../bank-tx/bank-tx/__mocks__/bank-tx.enti import { BankService } from '../../bank/bank/bank.service'; import { PayInService } from '../../payin/services/payin.service'; import { PayoutService } from '../../payout/services/payout.service'; +import { FinanceLog } from '../dto/log.dto'; import { LogJobService } from '../log-job.service'; import { LogService } from '../log.service'; @@ -532,4 +533,43 @@ describe('LogJobService', () => { expect(service.getUnmatchedSenders(senderTx, receiverTx)).toEqual([]); }); + + // --- isReconciledSnapshot --- // + + const lastFinanceLog = (totalBalanceChf: number, changesTotal?: number): FinanceLog => ({ + assets: {}, + tradings: {}, + balancesByFinancialType: {}, + balancesTotal: { plusBalanceChf: 0, minusBalanceChf: 0, totalBalanceChf }, + changesTotal, + }); + + it('should accept a total change explained by the operating result, even beyond the limit', () => { + // total +8000 explained by operating result +8000 (10000 -> 18000) => unexplained 0 + expect(service.isReconciledSnapshot(108000, 18000, lastFinanceLog(100000, 10000), new Date())).toBe(true); + }); + + it('should accept small FX/rounding noise within the limit', () => { + // total +1000, operating result unchanged => unexplained 1000 <= 5000 + expect(service.isReconciledSnapshot(101000, 10000, lastFinanceLog(100000, 10000), new Date())).toBe(true); + }); + + it('should reject an unreconciled jump beyond the limit, without a time-based escape', () => { + // total -20000 with operating result +100 => unexplained ~20100, stays invalid even for an old last log + expect(service.isReconciledSnapshot(80000, 10100, lastFinanceLog(100000, 10000), Util.hoursBefore(2))).toBe(false); + }); + + it('should fall back to the legacy time escape only when reconciliation data is missing', () => { + // no changesTotal on last log: large jump accepted after 15 min, rejected while recent + expect(service.isReconciledSnapshot(80000, 10100, lastFinanceLog(100000), Util.hoursBefore(2))).toBe(true); + expect(service.isReconciledSnapshot(80000, 10100, lastFinanceLog(100000), new Date())).toBe(false); + }); + + it('should handle the month-to-date reset of the operating result without false invalids', () => { + // new month: changesTotal reset to ~0 while the last valid snapshot carried +16000; balances carry over (Δtotal ~0) + // => no reconcilable delta, legacy path accepts the near-unchanged total + expect(service.isReconciledSnapshot(100050, 50, lastFinanceLog(100000, 16000), new Date())).toBe(true); + // a genuine glitch at the same reset boundary (large Δtotal, recent) is still rejected + expect(service.isReconciledSnapshot(80000, 50, lastFinanceLog(100000, 16000), new Date())).toBe(false); + }); }); diff --git a/src/subdomains/supporting/log/dto/log.dto.ts b/src/subdomains/supporting/log/dto/log.dto.ts index 1d934070d7..dca79cc99b 100644 --- a/src/subdomains/supporting/log/dto/log.dto.ts +++ b/src/subdomains/supporting/log/dto/log.dto.ts @@ -14,6 +14,10 @@ export interface FinanceLog { tradings: TradingLog; balancesByFinancialType: BalancesByFinancialType; balancesTotal: BalancesTotal; + // Month-to-date operating result (= ChangeLog.total) captured at log time. Used to reconcile the + // total-balance delta between snapshots: a valid snapshot's total only moves by operating result + FX. + // Optional because snapshots written before this field existed do not carry it. + changesTotal?: number; } /** diff --git a/src/subdomains/supporting/log/log-job.service.ts b/src/subdomains/supporting/log/log-job.service.ts index f1e992d053..cee33d360f 100644 --- a/src/subdomains/supporting/log/log-job.service.ts +++ b/src/subdomains/supporting/log/log-job.service.ts @@ -125,7 +125,7 @@ export class LogJobService { await this.processService.setSafetyModeActive(totalBalanceChf < minTotalBalanceChf); const lastLog = await this.logService.maxEntity('LogService', 'FinancialDataLog', LogSeverity.INFO, true); - const lastTotalBalance = (JSON.parse(lastLog.message) as FinanceLog).balancesTotal.totalBalanceChf; + const lastFinanceLog = JSON.parse(lastLog.message) as FinanceLog; await this.logService.create({ system: 'LogService', @@ -140,10 +140,9 @@ export class LogJobService { minusBalanceChf: this.getJsonValue(minusBalanceChf, AmountType.FIAT, true), totalBalanceChf: this.getJsonValue(totalBalanceChf, AmountType.FIAT, true), }, + changesTotal: this.getJsonValue(changeLog.total, AmountType.FIAT, true), }), - valid: - Math.abs(totalBalanceChf - lastTotalBalance) <= Config.financeLogTotalBalanceChangeLimit || - Util.minutesDiff(lastLog.created) > 15, + valid: this.isReconciledSnapshot(totalBalanceChf, changeLog.total, lastFinanceLog, lastLog.created), category: null, }); @@ -163,6 +162,39 @@ export class LogJobService { // --- LOG METHODS --- // + /** + * A snapshot is valid only if its total-balance change reconciles with the operating result. + * + * `totalBalanceChf` moves only by operating result (changeLog) + FX. `changeLog.total` is month-to-date, so the + * expected delta since the last valid snapshot is its change; FX and rounding are absorbed by + * `financeLogTotalBalanceChangeLimit`. A read glitch (an asset misread as 0 or an implausibly large value) breaks + * this reconciliation and is flagged invalid, so it does not pollute the valid-filtered series. + * + * Falls back to the legacy time-based escape when reconciliation data is unavailable: either the last snapshot + * predates `changesTotal`, or `changeLog.total` has reset (it is month-to-date and drops to ~0 at month start, so a + * value below the last snapshot's yields no meaningful delta). Once a same-month delta is available, an unreconciled + * jump is never silently accepted after 15 minutes. + */ + public isReconciledSnapshot( + totalBalanceChf: number, + changesTotal: number, + lastFinanceLog: FinanceLog, + lastLogCreated: Date, + ): boolean { + const lastTotalBalance = lastFinanceLog.balancesTotal.totalBalanceChf; + const canReconcile = lastFinanceLog.changesTotal != null && changesTotal >= lastFinanceLog.changesTotal; + const expectedChange = canReconcile ? changesTotal - lastFinanceLog.changesTotal : undefined; + const unexplainedChange = + expectedChange != null + ? Math.abs(totalBalanceChf - lastTotalBalance - expectedChange) + : Math.abs(totalBalanceChf - lastTotalBalance); + + return ( + unexplainedChange <= Config.financeLogTotalBalanceChangeLimit || + (expectedChange == null && Util.minutesDiff(lastLogCreated) > 15) + ); + } + private getBalancesByFinancialType(assets: Asset[], assetLog: AssetLog): BalancesByFinancialType { const financialTypeMap = Util.groupBy( assets.filter((a) => a.financialType),