diff --git a/migration/1779221816705-AddLogFinancialIndex.js b/migration/1779221816705-AddLogFinancialIndex.js new file mode 100644 index 0000000000..516e57a757 --- /dev/null +++ b/migration/1779221816705-AddLogFinancialIndex.js @@ -0,0 +1,33 @@ +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + * @typedef {import('typeorm').QueryRunner} QueryRunner + */ + +/** + * @class + * @implements {MigrationInterface} + */ +module.exports = class AddLogFinancialIndex1779221816705 { + name = 'AddLogFinancialIndex1779221816705'; + + /** + * @param {QueryRunner} queryRunner + */ + async up(queryRunner) { + // Composite index to accelerate dashboard-financial queries: + // - getLatestFinancialLog (system/subsystem/severity, ORDER BY id DESC) + // - getFinancialLogs / getFinancialChangesLogs (range scans on created with daily/minute bucketing) + // - log-job.service.maxEntity lookups (every minute on the same predicates) + // The log table grows by ~525k rows/year (one cron entry per minute), so a non-covered scan is expensive. + await queryRunner.query( + `CREATE INDEX "IDX_7765c3f5f663a0c6d250d28255" ON "log" ("subsystem", "severity", "created")`, + ); + } + + /** + * @param {QueryRunner} queryRunner + */ + async down(queryRunner) { + await queryRunner.query(`DROP INDEX "IDX_7765c3f5f663a0c6d250d28255" ON "log"`); + } +}; diff --git a/migration/1779221847925-AddLogFinancialAggregates.js b/migration/1779221847925-AddLogFinancialAggregates.js new file mode 100644 index 0000000000..5f764dd81b --- /dev/null +++ b/migration/1779221847925-AddLogFinancialAggregates.js @@ -0,0 +1,36 @@ +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + * @typedef {import('typeorm').QueryRunner} QueryRunner + */ + +/** + * @class + * @implements {MigrationInterface} + */ +module.exports = class AddLogFinancialAggregates1779221847925 { + name = 'AddLogFinancialAggregates1779221847925'; + + /** + * @param {QueryRunner} queryRunner + */ + async up(queryRunner) { + // Denormalised aggregates for the FinancialDataLog cron rows, consumed by the dashboard chart + // endpoint. Nullable so older rows continue to work via a JSON.parse fallback in the service. + await queryRunner.query(`ALTER TABLE "log" ADD "totalBalanceChf" float`); + await queryRunner.query(`ALTER TABLE "log" ADD "plusBalanceChf" float`); + await queryRunner.query(`ALTER TABLE "log" ADD "minusBalanceChf" float`); + await queryRunner.query(`ALTER TABLE "log" ADD "btcPriceChf" float`); + await queryRunner.query(`ALTER TABLE "log" ADD "balancesByTypeJson" nvarchar(4000)`); + } + + /** + * @param {QueryRunner} queryRunner + */ + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "log" DROP COLUMN "balancesByTypeJson"`); + await queryRunner.query(`ALTER TABLE "log" DROP COLUMN "btcPriceChf"`); + await queryRunner.query(`ALTER TABLE "log" DROP COLUMN "minusBalanceChf"`); + await queryRunner.query(`ALTER TABLE "log" DROP COLUMN "plusBalanceChf"`); + await queryRunner.query(`ALTER TABLE "log" DROP COLUMN "totalBalanceChf"`); + } +}; diff --git a/package-lock.json b/package-lock.json index 233b36f79e..29dc76912c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -72,6 +72,7 @@ "class-transformer": "^0.5.1", "class-validator": "^0.14.2", "coingecko-api-v3": "^0.0.25", + "compression": "^1.8.1", "cors": "^2.8.5", "ebics-client": "^5.0.0", "ethers": "^5.8.0", @@ -120,6 +121,7 @@ "@nestjs/cli": "^9.5.0", "@nestjs/schematics": "^9.2.0", "@nestjs/testing": "^9.4.3", + "@types/compression": "^1.8.1", "@types/express": "^4.17.25", "@types/express-useragent": "^1.0.5", "@types/google-libphonenumber": "^7.4.30", @@ -9591,6 +9593,17 @@ "@types/responselike": "^1.0.0" } }, + "node_modules/@types/compression": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/@types/compression/-/compression-1.8.1.tgz", + "integrity": "sha512-kCFuWS0ebDbmxs0AXYn6e2r2nrGAb5KwQhknjSPSPgJcGd8+HVSILlUyFhGqML2gk39HcG7D1ydW9/qpYkN00Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/express": "*", + "@types/node": "*" + } + }, "node_modules/@types/connect": { "version": "3.4.38", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", @@ -13883,6 +13896,80 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/compressible": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", + "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", + "license": "MIT", + "dependencies": { + "mime-db": ">= 1.43.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/compression": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/compression/-/compression-1.8.1.tgz", + "integrity": "sha512-9mAqGPHLakhCLeNyxPkK4xVo746zQ/czLH1Ky+vkitMnWfWZps8r0qXuwhwizagCRttsL4lfG4pIOvaWLpAP0w==", + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "compressible": "~2.0.18", + "debug": "2.6.9", + "negotiator": "~0.6.4", + "on-headers": "~1.1.0", + "safe-buffer": "5.2.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/compression/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/compression/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/compression/node_modules/negotiator": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.4.tgz", + "integrity": "sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/compression/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -21919,7 +22006,6 @@ "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", "license": "MIT", - "peer": true, "engines": { "node": ">= 0.6" } diff --git a/package.json b/package.json index e7706a1d6e..95e7394d78 100644 --- a/package.json +++ b/package.json @@ -89,6 +89,7 @@ "class-transformer": "^0.5.1", "class-validator": "^0.14.2", "coingecko-api-v3": "^0.0.25", + "compression": "^1.8.1", "cors": "^2.8.5", "ebics-client": "^5.0.0", "ethers": "^5.8.0", @@ -137,6 +138,7 @@ "@nestjs/cli": "^9.5.0", "@nestjs/schematics": "^9.2.0", "@nestjs/testing": "^9.4.3", + "@types/compression": "^1.8.1", "@types/express": "^4.17.25", "@types/express-useragent": "^1.0.5", "@types/google-libphonenumber": "^7.4.30", diff --git a/src/main.ts b/src/main.ts index c043c0cb9e..dfe3275882 100644 --- a/src/main.ts +++ b/src/main.ts @@ -5,6 +5,7 @@ import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; import * as AppInsights from 'applicationinsights'; import { spawnSync } from 'child_process'; import { useContainer } from 'class-validator'; +import compression from 'compression'; import cors from 'cors'; import { json, raw, text } from 'express'; import helmet from 'helmet'; @@ -62,6 +63,7 @@ async function bootstrap() { app.use(morgan('dev')); app.use(helmet()); + app.use(compression()); app.use( cors({ exposedHeaders: ['content-disposition'], diff --git a/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts b/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts new file mode 100644 index 0000000000..8480151fda --- /dev/null +++ b/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts @@ -0,0 +1,135 @@ +import { createMock } from '@golevelup/ts-jest'; +import { Test, TestingModule } from '@nestjs/testing'; +import { AssetService } from 'src/shared/models/asset/asset.service'; +import { RefRewardService } from 'src/subdomains/core/referral/reward/services/ref-reward.service'; +import { createCustomLog } from '../../log/__mocks__/log.entity.mock'; +import { LogService } from '../../log/log.service'; +import { DashboardFinancialService } from '../dashboard-financial.service'; + +describe('DashboardFinancialService', () => { + let service: DashboardFinancialService; + + let logService: LogService; + let assetService: AssetService; + let refRewardService: RefRewardService; + + beforeEach(async () => { + logService = createMock(); + assetService = createMock(); + refRewardService = createMock(); + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + DashboardFinancialService, + { provide: LogService, useValue: logService }, + { provide: AssetService, useValue: assetService }, + { provide: RefRewardService, useValue: refRewardService }, + ], + }).compile(); + + service = module.get(DashboardFinancialService); + }); + + describe('getFinancialLog (chart fast path)', () => { + it('uses denormalised aggregate columns without parsing the message JSON', async () => { + const log = createCustomLog({ + id: 42, + created: new Date('2026-05-19T12:00:00Z'), + message: '{"this":"must not be parsed"}', + totalBalanceChf: 1_000_000, + plusBalanceChf: 1_200_000, + minusBalanceChf: 200_000, + btcPriceChf: 90_000, + balancesByTypeJson: JSON.stringify({ EUR: { plusBalanceChf: 50, minusBalanceChf: 10 } }), + }); + + jest.spyOn(logService, 'getFinancialLogs').mockResolvedValue([log]); + jest.spyOn(assetService, 'getBtcCoin').mockResolvedValue({ id: 1 } as never); + + const result = await service.getFinancialLog(new Date(), false); + + expect(result.entries).toHaveLength(1); + expect(result.entries[0]).toEqual({ + timestamp: log.created, + totalBalanceChf: 1_000_000, + plusBalanceChf: 1_200_000, + minusBalanceChf: 200_000, + btcPriceChf: 90_000, + balancesByType: { EUR: { plusBalanceChf: 50, minusBalanceChf: 10 } }, + }); + }); + + it('falls back to JSON.parse for legacy rows without denormalised columns', async () => { + const message = JSON.stringify({ + balancesByFinancialType: { EUR: { plusBalanceChf: 100, minusBalanceChf: 20 } }, + balancesTotal: { plusBalanceChf: 100, minusBalanceChf: 20, totalBalanceChf: 80 }, + assets: { 1: { priceChf: 90_000 } }, + }); + + const legacyLog = createCustomLog({ + id: 1, + created: new Date('2024-01-01T00:00:00Z'), + message, + totalBalanceChf: null, + plusBalanceChf: null, + minusBalanceChf: null, + btcPriceChf: null, + balancesByTypeJson: null, + }); + + jest.spyOn(logService, 'getFinancialLogs').mockResolvedValue([legacyLog]); + jest.spyOn(assetService, 'getBtcCoin').mockResolvedValue({ id: 1 } as never); + + const result = await service.getFinancialLog(new Date(), false); + + expect(result.entries).toHaveLength(1); + expect(result.entries[0]).toMatchObject({ + totalBalanceChf: 80, + plusBalanceChf: 100, + minusBalanceChf: 20, + btcPriceChf: 90_000, + balancesByType: { EUR: { plusBalanceChf: 100, minusBalanceChf: 20 } }, + }); + }); + + it('caches results within the TTL window', async () => { + const log = createCustomLog({ + id: 1, + created: new Date(), + totalBalanceChf: 1, + plusBalanceChf: 1, + minusBalanceChf: 0, + btcPriceChf: 0, + }); + + const spy = jest.spyOn(logService, 'getFinancialLogs').mockResolvedValue([log]); + jest.spyOn(assetService, 'getBtcCoin').mockResolvedValue({ id: 1 } as never); + + const from = new Date(); + await service.getFinancialLog(from, false); + await service.getFinancialLog(from, false); + + expect(spy).toHaveBeenCalledTimes(1); + }); + + it('uses different cache entries for different query parameters', async () => { + const log = createCustomLog({ + id: 1, + created: new Date(), + totalBalanceChf: 1, + plusBalanceChf: 1, + minusBalanceChf: 0, + btcPriceChf: 0, + }); + + const spy = jest.spyOn(logService, 'getFinancialLogs').mockResolvedValue([log]); + jest.spyOn(assetService, 'getBtcCoin').mockResolvedValue({ id: 1 } as never); + + await service.getFinancialLog(new Date('2026-01-01'), false); + await service.getFinancialLog(new Date('2026-01-02'), false); + await service.getFinancialLog(new Date('2026-01-01'), true); + + expect(spy).toHaveBeenCalledTimes(3); + }); + }); +}); diff --git a/src/subdomains/supporting/dashboard/dashboard-financial.service.ts b/src/subdomains/supporting/dashboard/dashboard-financial.service.ts index 4099ddac96..140336ab6e 100644 --- a/src/subdomains/supporting/dashboard/dashboard-financial.service.ts +++ b/src/subdomains/supporting/dashboard/dashboard-financial.service.ts @@ -1,5 +1,6 @@ import { Injectable } from '@nestjs/common'; import { AssetService } from 'src/shared/models/asset/asset.service'; +import { AsyncCache, CacheItemResetPeriod } from 'src/shared/utils/async-cache'; import { RefRewardService } from '../../core/referral/reward/services/ref-reward.service'; import { Log } from '../log/log.entity'; import { LogService } from '../log/log.service'; @@ -16,6 +17,20 @@ import { @Injectable() export class DashboardFinancialService { + // The underlying data is written by a cron every minute; a 30s TTL hides repeated dashboard + // refreshes (multiple admins, polling, page revisits) without ever serving data older than the + // next cron tick on average. + private readonly latestBalanceCache = new AsyncCache( + CacheItemResetPeriod.EVERY_30_SECONDS, + ); + private readonly latestChangesCache = new AsyncCache( + CacheItemResetPeriod.EVERY_30_SECONDS, + ); + private readonly financialLogCache = new AsyncCache(CacheItemResetPeriod.EVERY_30_SECONDS); + private readonly financialChangesCache = new AsyncCache( + CacheItemResetPeriod.EVERY_30_SECONDS, + ); + constructor( private readonly logService: LogService, private readonly assetService: AssetService, @@ -23,6 +38,11 @@ export class DashboardFinancialService { ) {} async getFinancialLog(from?: Date, dailySample?: boolean): Promise { + const cacheKey = `from=${from?.toISOString() ?? ''}|dailySample=${dailySample ?? ''}`; + return this.financialLogCache.get(cacheKey, () => this.loadFinancialLog(from, dailySample)); + } + + private async loadFinancialLog(from?: Date, dailySample?: boolean): Promise { const [logs, btcAsset] = await Promise.all([ this.logService.getFinancialLogs(from, dailySample), this.assetService.getBtcCoin(), @@ -41,12 +61,19 @@ export class DashboardFinancialService { } async getLatestFinancialChanges(): Promise { - const latest = await this.logService.getLatestFinancialChangesLog(); - if (!latest) return undefined; - return this.mapChangesLogToEntry(latest); + return this.latestChangesCache.get('latest', async () => { + const latest = await this.logService.getLatestFinancialChangesLog(); + if (!latest) return undefined; + return this.mapChangesLogToEntry(latest); + }); } async getFinancialChanges(from?: Date, dailySample?: boolean): Promise { + const cacheKey = `from=${from?.toISOString() ?? ''}|dailySample=${dailySample ?? ''}`; + return this.financialChangesCache.get(cacheKey, () => this.loadFinancialChanges(from, dailySample)); + } + + private async loadFinancialChanges(from?: Date, dailySample?: boolean): Promise { const logs = await this.logService.getFinancialChangesLogs(from, dailySample); const entries = logs @@ -103,6 +130,10 @@ export class DashboardFinancialService { } async getLatestBalance(): Promise { + return this.latestBalanceCache.get('latest', () => this.loadLatestBalance()); + } + + private async loadLatestBalance(): Promise { const latest = await this.logService.getLatestFinancialLog(); if (!latest) return undefined; @@ -225,6 +256,19 @@ export class DashboardFinancialService { } private mapLogToEntry(log: Log, btcAssetId?: number): FinancialLogEntryDto | undefined { + // Fast path: use denormalised aggregate columns when present (populated for new rows by the + // FinancialDataLog cron). Falls back to JSON.parse of `message` for legacy rows pre-migration. + if (log.totalBalanceChf != null && log.plusBalanceChf != null && log.minusBalanceChf != null) { + return { + timestamp: log.created, + totalBalanceChf: log.totalBalanceChf, + plusBalanceChf: log.plusBalanceChf, + minusBalanceChf: log.minusBalanceChf, + btcPriceChf: log.btcPriceChf ?? 0, + balancesByType: log.balancesByType, + }; + } + try { const financeLog: FinanceLog = JSON.parse(log.message); diff --git a/src/subdomains/supporting/log/__tests__/log.repository.spec.ts b/src/subdomains/supporting/log/__tests__/log.repository.spec.ts new file mode 100644 index 0000000000..6dc07c8243 --- /dev/null +++ b/src/subdomains/supporting/log/__tests__/log.repository.spec.ts @@ -0,0 +1,29 @@ +import { Util } from 'src/shared/utils/util'; +import { getSampleIntervalMinutes } from '../log.util'; + +describe('getSampleIntervalMinutes', () => { + it('returns null when dailySample is true (caller picks daily bucket)', () => { + expect(getSampleIntervalMinutes(undefined, true)).toBeNull(); + expect(getSampleIntervalMinutes(Util.daysBefore(3), true)).toBeNull(); + }); + + it('returns null when no `from` is provided (no range to bucket)', () => { + expect(getSampleIntervalMinutes(undefined, false)).toBeNull(); + expect(getSampleIntervalMinutes(undefined, undefined)).toBeNull(); + }); + + it('returns null for the 24h live view (full per-minute resolution)', () => { + expect(getSampleIntervalMinutes(Util.hoursBefore(24), false)).toBeNull(); + expect(getSampleIntervalMinutes(Util.hoursBefore(1), false)).toBeNull(); + }); + + it('returns 5-minute buckets for 3-day and 1-week ranges', () => { + expect(getSampleIntervalMinutes(Util.daysBefore(3), false)).toBe(5); + expect(getSampleIntervalMinutes(Util.daysBefore(7), false)).toBe(5); + }); + + it('returns null beyond 1 week so callers do not silently drop data', () => { + expect(getSampleIntervalMinutes(Util.daysBefore(14), false)).toBeNull(); + expect(getSampleIntervalMinutes(Util.daysBefore(30), false)).toBeNull(); + }); +}); diff --git a/src/subdomains/supporting/log/dto/create-log.dto.ts b/src/subdomains/supporting/log/dto/create-log.dto.ts index bb4926a99e..ab23576ba2 100644 --- a/src/subdomains/supporting/log/dto/create-log.dto.ts +++ b/src/subdomains/supporting/log/dto/create-log.dto.ts @@ -1,5 +1,5 @@ -import { IsBoolean, IsEnum, IsNotEmpty, IsOptional, IsString } from 'class-validator'; -import { LogSeverity } from '../log.entity'; +import { IsBoolean, IsEnum, IsNotEmpty, IsNumber, IsObject, IsOptional, IsString } from 'class-validator'; +import { BalancesByTypeMap, LogSeverity } from '../log.entity'; export class CreateLogDto { @IsNotEmpty() @@ -25,6 +25,26 @@ export class CreateLogDto { @IsOptional() @IsBoolean() valid: boolean; + + @IsOptional() + @IsNumber() + totalBalanceChf?: number; + + @IsOptional() + @IsNumber() + plusBalanceChf?: number; + + @IsOptional() + @IsNumber() + minusBalanceChf?: number; + + @IsOptional() + @IsNumber() + btcPriceChf?: number; + + @IsOptional() + @IsObject() + balancesByType?: BalancesByTypeMap; } export class UpdateLogDto { diff --git a/src/subdomains/supporting/log/log-job.service.ts b/src/subdomains/supporting/log/log-job.service.ts index bb39225982..59af03c326 100644 --- a/src/subdomains/supporting/log/log-job.service.ts +++ b/src/subdomains/supporting/log/log-job.service.ts @@ -54,7 +54,7 @@ import { ManualLogPosition, TradingLog, } from './dto/log.dto'; -import { LogSeverity } from './log.entity'; +import { BalancesByTypeMap, LogSeverity } from './log.entity'; import { LogService } from './log.service'; @Injectable() @@ -126,6 +126,23 @@ export class LogJobService { const lastLog = await this.logService.maxEntity('LogService', 'FinancialDataLog', LogSeverity.INFO, true); const lastTotalBalance = (JSON.parse(lastLog.message) as FinanceLog).balancesTotal.totalBalanceChf; + // Aggregate values needed by the dashboard chart endpoint. Mirrored to dedicated columns so + // the chart query can read them directly instead of JSON.parse'ing the nvarchar(MAX) message. + const roundedTotalBalanceChf = this.getJsonValue(totalBalanceChf, AmountType.FIAT, true); + const roundedPlusBalanceChf = this.getJsonValue(plusBalanceChf, AmountType.FIAT, true); + const roundedMinusBalanceChf = this.getJsonValue(minusBalanceChf, AmountType.FIAT, true); + const btcAsset = assets.find((a) => a.name === 'BTC' && (a.blockchain as string) === Blockchain.BITCOIN); + const btcPriceChf = btcAsset ? (assetLog[btcAsset.id]?.priceChf ?? 0) : 0; + + // Compact per-type aggregate snapshot (chart consumes only plus/minus CHF per type). + const balancesByType: BalancesByTypeMap = {}; + for (const [type, data] of Object.entries(balancesByFinancialType)) { + balancesByType[type] = { + plusBalanceChf: data.plusBalanceChf, + minusBalanceChf: data.minusBalanceChf, + }; + } + await this.logService.create({ system: 'LogService', subsystem: 'FinancialDataLog', @@ -135,15 +152,20 @@ export class LogJobService { tradings: tradingLog, balancesByFinancialType, balancesTotal: { - plusBalanceChf: this.getJsonValue(plusBalanceChf, AmountType.FIAT, true), - minusBalanceChf: this.getJsonValue(minusBalanceChf, AmountType.FIAT, true), - totalBalanceChf: this.getJsonValue(totalBalanceChf, AmountType.FIAT, true), + plusBalanceChf: roundedPlusBalanceChf, + minusBalanceChf: roundedMinusBalanceChf, + totalBalanceChf: roundedTotalBalanceChf, }, }), valid: Math.abs(totalBalanceChf - lastTotalBalance) <= Config.financeLogTotalBalanceChangeLimit || Util.minutesDiff(lastLog.created) > 15, category: null, + totalBalanceChf: roundedTotalBalanceChf, + plusBalanceChf: roundedPlusBalanceChf, + minusBalanceChf: roundedMinusBalanceChf, + btcPriceChf, + balancesByType, }); await this.logService.create({ diff --git a/src/subdomains/supporting/log/log.entity.ts b/src/subdomains/supporting/log/log.entity.ts index 40d5b0c992..a5a7733676 100644 --- a/src/subdomains/supporting/log/log.entity.ts +++ b/src/subdomains/supporting/log/log.entity.ts @@ -7,6 +7,8 @@ export enum LogSeverity { ERROR = 'Error', } +export type BalancesByTypeMap = Record; + @Entity() export class Log extends IEntity { @Column({ length: 256 }) @@ -26,4 +28,33 @@ export class Log extends IEntity { @Column({ nullable: true }) valid?: boolean; + + // Denormalised aggregates for FinancialDataLog, used by the dashboard chart endpoint to avoid + // parsing the nvarchar(MAX) message JSON for every row. Nullable so legacy rows fall back to JSON. + @Column({ type: 'float', nullable: true }) + totalBalanceChf?: number; + + @Column({ type: 'float', nullable: true }) + plusBalanceChf?: number; + + @Column({ type: 'float', nullable: true }) + minusBalanceChf?: number; + + @Column({ type: 'float', nullable: true }) + btcPriceChf?: number; + + // Compact JSON snapshot of the per-financialType plus/minus aggregates (orders of magnitude + // smaller than the full message), kept separate so the chart endpoint avoids the full parse. + // Access via the typed `balancesByType` getter/setter — never read/write this raw string from + // business logic. + @Column({ length: 4000, nullable: true }) + balancesByTypeJson?: string; + + get balancesByType(): BalancesByTypeMap { + return this.balancesByTypeJson ? JSON.parse(this.balancesByTypeJson) : {}; + } + + set balancesByType(balances: BalancesByTypeMap) { + this.balancesByTypeJson = JSON.stringify(balances); + } } diff --git a/src/subdomains/supporting/log/log.repository.ts b/src/subdomains/supporting/log/log.repository.ts index ae63fcd63a..bbb253581b 100644 --- a/src/subdomains/supporting/log/log.repository.ts +++ b/src/subdomains/supporting/log/log.repository.ts @@ -4,6 +4,7 @@ import { Util } from 'src/shared/utils/util'; import { EntityManager, FindOptionsWhere, LessThanOrEqual, MoreThanOrEqual } from 'typeorm'; import { LogCleanupSetting } from './dto/create-log.dto'; import { Log, LogSeverity } from './log.entity'; +import { getSampleIntervalMinutes } from './log.util'; @Injectable() export class LogRepository extends BaseRepository { @@ -67,13 +68,26 @@ export class LogRepository extends BaseRepository { } async getFinancialChangesLogs(from?: Date, dailySample?: boolean): Promise { + return this.getSampledFinancialLogs('FinancialChangesLog', from, dailySample); + } + + async getFinancialLogs(from?: Date, dailySample?: boolean): Promise { + return this.getSampledFinancialLogs('FinancialDataLog', from, dailySample); + } + + private async getSampledFinancialLogs(subsystem: string, from?: Date, dailySample?: boolean): Promise { if (dailySample) { - const subQuery = this.createQueryBuilder('subLog') + let subQuery = this.createQueryBuilder('subLog') .select('MAX(subLog.id)', 'max_id') .where('subLog.system = :system', { system: 'LogService' }) - .andWhere('subLog.subsystem = :subsystem', { subsystem: 'FinancialChangesLog' }) - .andWhere('subLog.severity = :severity', { severity: LogSeverity.INFO }) - .groupBy('CAST(subLog.created AS DATE)'); + .andWhere('subLog.subsystem = :subsystem', { subsystem }) + .andWhere('subLog.severity = :severity', { severity: LogSeverity.INFO }); + + if (from) { + subQuery = subQuery.andWhere('subLog.created >= :from', { from }); + } + + subQuery = subQuery.groupBy('CAST(subLog.created AS DATE)'); let query = this.createQueryBuilder('log') .where(`log.id IN (${subQuery.getQuery()})`) @@ -87,27 +101,24 @@ export class LogRepository extends BaseRepository { return query.getMany(); } - const where: FindOptionsWhere = { - system: 'LogService', - subsystem: 'FinancialChangesLog', - severity: LogSeverity.INFO, - }; - - if (from) { - where.created = MoreThanOrEqual(from); - } - - return this.find({ where, order: { created: 'ASC' } }); - } + const bucketMinutes = getSampleIntervalMinutes(from, dailySample); - async getFinancialLogs(from?: Date, dailySample?: boolean): Promise { - if (dailySample) { - const subQuery = this.createQueryBuilder('subLog') + if (bucketMinutes != null) { + // DB-side N-minute bucketing: pick the latest id per bucket, then fetch those rows. + // Mirrors the dailySample shape but uses DATEADD/DATEDIFF for sub-day buckets. + let subQuery = this.createQueryBuilder('subLog') .select('MAX(subLog.id)', 'max_id') .where('subLog.system = :system', { system: 'LogService' }) - .andWhere('subLog.subsystem = :subsystem', { subsystem: 'FinancialDataLog' }) - .andWhere('subLog.severity = :severity', { severity: LogSeverity.INFO }) - .groupBy('CAST(subLog.created AS DATE)'); + .andWhere('subLog.subsystem = :subsystem', { subsystem }) + .andWhere('subLog.severity = :severity', { severity: LogSeverity.INFO }); + + if (from) { + subQuery = subQuery.andWhere('subLog.created >= :from', { from }); + } + + subQuery = subQuery.groupBy( + `DATEADD(MINUTE, (DATEDIFF(MINUTE, 0, subLog.created) / ${bucketMinutes}) * ${bucketMinutes}, 0)`, + ); let query = this.createQueryBuilder('log') .where(`log.id IN (${subQuery.getQuery()})`) @@ -123,7 +134,7 @@ export class LogRepository extends BaseRepository { const where: FindOptionsWhere = { system: 'LogService', - subsystem: 'FinancialDataLog', + subsystem, severity: LogSeverity.INFO, }; diff --git a/src/subdomains/supporting/log/log.service.ts b/src/subdomains/supporting/log/log.service.ts index d63cefcd9a..47f01bf483 100644 --- a/src/subdomains/supporting/log/log.service.ts +++ b/src/subdomains/supporting/log/log.service.ts @@ -28,7 +28,9 @@ export class LogService { if (dto.message === maxEntity?.message && dto.valid === maxEntity?.valid && dto.category === maxEntity?.category) return maxEntity; - const newEntity = this.logRepo.create(dto); + const { balancesByType, ...rest } = dto; + const newEntity = this.logRepo.create(rest); + if (balancesByType) newEntity.balancesByType = balancesByType; return this.logRepo.save(newEntity); } diff --git a/src/subdomains/supporting/log/log.util.ts b/src/subdomains/supporting/log/log.util.ts new file mode 100644 index 0000000000..9188d25494 --- /dev/null +++ b/src/subdomains/supporting/log/log.util.ts @@ -0,0 +1,23 @@ +/** + * Returns the bucket size (minutes) for DB-side sampling of FinancialDataLog rows when the caller + * did not request a daily sample. + * + * - `null` → no bucketing, return every row (covers the 24h live view, ~1440 rows) + * - positive integer → 1 row per N-minute bucket (covers 3D/week ranges to keep payloads small) + * + * The cron writes a new row every minute, so without bucketing a 3-day range returns ~4320 rows. + * A 5-minute bucket compresses that to ~864 rows without losing visible detail at chart resolution. + */ +export function getSampleIntervalMinutes(from?: Date, dailySample?: boolean): number | null { + if (dailySample) return null; + if (!from) return null; + + const rangeHours = (Date.now() - from.getTime()) / (1000 * 60 * 60); + + if (rangeHours <= 26) return null; // 24h live view: full resolution + if (rangeHours <= 24 * 7) return 5; // 3 days, 1 week: 5-minute buckets + + // Beyond 1 week without dailySample we keep per-minute resolution; callers typically pass + // dailySample=true for longer ranges, but we don't want to drop data silently here. + return null; +}