Faster payments page (#1124)

* Don't fetch inputs unless requested

This avoids loading the tx inputs where they are unlikely needed. Can still be requested by passing`includeInputs=true` to the query.

The following api endpoints are affected:
 - /api/payments
 - /api/paybutton/transactions/[id]

None of these needs the inputs by default.

* Synchronous functions don't need async

This just add overhead, especially when called in loops.

* Improve performance of fetchDistinctPaymentYearsByUser

This is called from the very niche endpoint /api/transaction/years.

The amount > 0 was intended to only check for payments, but in practice this doesn't matter. We can just return the years of acticity for this userId and show no payments for the years it was not used on paybutton. This dramatically speeds up the query (from ~9s to ~100ms on my machine).

* Add a new isPayment flag to Transaction and use it as an index

This makes the payment requests much faster.

* Fetch paginated payments and counts in parallel

This speeds up loading of the payments page for some filters.

Author: Fabcien

pages/api/paybutton/transactions/[id].ts

@@ -1,4 +1,4 @@
-import { RESPONSE_MESSAGES, TX_PAGE_SIZE_LIMIT } from 'constants/index'
+import { RESPONSE_MESSAGES, TX_PAGE_SIZE_LIMIT, DEFAULT_TX_PAGE_SIZE } from 'constants/index'
 import { fetchTransactionsByPaybuttonIdWithPagination } from 'services/transactionService'
 import * as paybuttonService from 'services/paybuttonService'
 import { setSession } from 'utils/setSession'
@@ -13,6 +13,7 @@ export default async (req: any, res: any): Promise<void> => {
     const pageSize = (req.query.pageSize === '' || req.query.pageSize === undefined) ? DEFAULT_TX_PAGE_SIZE : Number(req.query.pageSize)
     const orderBy = (req.query.orderBy === '' || req.query.orderBy === undefined) ? undefined : req.query.orderBy as string
     const orderDesc: boolean = !!(req.query.orderDesc === '' || req.query.orderDesc === undefined || req.query.orderDesc === 'true')
+    const includeInputs: boolean = req.query.includeInputs === 'true'
 
     if (isNaN(page) || isNaN(pageSize)) {
       throw new Error(RESPONSE_MESSAGES.PAGE_SIZE_AND_PAGE_SHOULD_BE_NUMBERS_400.message)
@@ -27,7 +28,7 @@ export default async (req: any, res: any): Promise<void> => {
         throw new Error(RESPONSE_MESSAGES.RESOURCE_DOES_NOT_BELONG_TO_USER_400.message)
       }
 
-      const transactions = await fetchTransactionsByPaybuttonIdWithPagination(paybuttonId, page, pageSize, orderDesc, orderBy)
+      const transactions = await fetchTransactionsByPaybuttonIdWithPagination(paybuttonId, page, pageSize, orderDesc, orderBy, undefined, includeInputs)
 
       res.status(200).json({ transactions })
     } catch (err: any) {

pages/api/payments/index.ts

@@ -28,6 +28,7 @@ export default async (req: any, res: any): Promise<void> => {
     if (typeof req.query.endDate === 'string' && req.query.endDate !== '') {
       endDate = req.query.endDate as string
     }
+    const includeInputs = req.query.includeInputs === 'true'
     const userReqTimezone = req.headers.timezone as string
     const userPreferredTimezone = user?.preferredTimezone
     let timezone = userPreferredTimezone !== '' ? userPreferredTimezone : userReqTimezone
@@ -47,7 +48,8 @@ export default async (req: any, res: any): Promise<void> => {
       buttonIds,
       years,
       startDate,
-      endDate
+      endDate,
+      includeInputs
     )
     res.status(200).json(resJSON)
   }

pages/payments/index.tsx

@@ -224,16 +224,16 @@ export default function Payments ({ user, userId, organization }: PaybuttonsProp
         paymentsCountUrl += `${paymentsCountUrl.includes('?') ? '&' : '?'}endDate=${endDate}`
       }
 
-      const paymentsResponse = await fetch(url, {
-        headers: {
-          Timezone: timezone
-        }
-      })
-
-      const paymentsCountResponse = await fetch(
-        paymentsCountUrl,
-        { headers: { Timezone: timezone } }
-      )
+      const [paymentsResponse, paymentsCountResponse] = await Promise.all([
+        fetch(url, {
+          headers: {
+            Timezone: timezone
+          }
+        }),
+        fetch(paymentsCountUrl, {
+          headers: { Timezone: timezone }
+        })
+      ])
 
       if (!paymentsResponse.ok || !paymentsCountResponse.ok) {
         console.log('paymentsResponse status', paymentsResponse.status)
@@ -243,8 +243,10 @@ export default function Payments ({ user, userId, organization }: PaybuttonsProp
         throw new Error('Failed to fetch payments or count')
       }
 
-      const totalCount = await paymentsCountResponse.json()
-      const payments = await paymentsResponse.json()
+      const [payments, totalCount] = await Promise.all([
+        paymentsResponse.json(),
+        paymentsCountResponse.json()
+      ])
 
       return { data: payments, totalCount }
     } catch (error) {

prisma-local/migrations/20260228000000_add_is_payment_flag/migration.sql

@@ -0,0 +1,8 @@
+-- Add isPayment column
+ALTER TABLE `Transaction` ADD COLUMN `isPayment` BOOLEAN NOT NULL DEFAULT FALSE;
+
+-- Populate isPayment for existing data
+UPDATE `Transaction` SET `isPayment` = TRUE WHERE `amount` > 0;
+
+-- Add composite index for addressId + isPayment queries
+CREATE INDEX `Transaction_addressId_isPayment_idx` ON `Transaction`(`addressId`, `isPayment`);

prisma-local/schema.prisma

@@ -72,6 +72,7 @@ model Transaction {
   amount    Decimal                @db.Decimal(24, 8)
   confirmed Boolean                @default(false)
   orphaned  Boolean                @default(false)
+  isPayment Boolean                @default(false)
   timestamp Int
   addressId String
   opReturn String                @db.LongText @default("")
@@ -85,6 +86,7 @@ model Transaction {
 
   @@unique([hash, addressId], name: "Transaction_hash_addressId_unique_constraint")
   @@index([addressId, timestamp], map: "Transaction_addressId_timestamp_idx")
+  @@index([addressId, isPayment])
 }
 
 model TransactionInput {

redis/paymentCache.ts

@@ -69,7 +69,7 @@ interface GroupedPaymentsAndInfoObject {
   info: AddressPaymentInfo
 }
 
-export const generatePaymentFromTx = async (tx: TransactionsWithPaybuttonsAndPrices): Promise<Payment> => {
+export const generatePaymentFromTx = (tx: TransactionsWithPaybuttonsAndPrices): Payment => {
   const values = getTransactionValue(tx)
   let buttonDisplayDataList: Array<{ name: string, id: string}> = []
   if (tx.address.paybuttons !== undefined) {
@@ -97,7 +97,7 @@ export const generatePaymentFromTx = async (tx: TransactionsWithPaybuttonsAndPri
   }
 }
 
-export const generatePaymentFromTxWithInvoices = async (tx: TransactionWithAddressAndPricesAndInvoices, userId?: string): Promise<Payment> => {
+export const generatePaymentFromTxWithInvoices = (tx: TransactionWithAddressAndPricesAndInvoices, userId?: string): Payment => {
   const values = getTransactionValue(tx)
   let buttonDisplayDataList: Array<{ name: string, id: string}> = []
   if (tx.address.paybuttons !== undefined) {
@@ -141,7 +141,7 @@ export const generateAndCacheGroupedPaymentsAndInfoForAddress = async (address:
     for (const tx of batch) {
       balance = balance.plus(tx.amount)
       if (tx.amount.gt(0)) {
-        const payment = await generatePaymentFromTx(tx)
+        const payment = generatePaymentFromTx(tx)
         paymentList.push(payment)
         paymentCount++
       }
@@ -235,7 +235,7 @@ const cacheGroupedPaymentsAppend = async (paymentsGroupedByKey: KeyValueT<Paymen
 export const cacheManyTxs = async (txs: TransactionsWithPaybuttonsAndPrices[]): Promise<void> => {
   const zero = new Prisma.Decimal(0)
   for (const tx of txs.filter(tx => tx.amount > zero)) {
-    const payment = await generatePaymentFromTx(tx)
+    const payment = generatePaymentFromTx(tx)
     if (payment.values.usd !== new Prisma.Decimal(0)) {
       const paymentsGroupedByKey = getPaymentsByWeek(tx.address.address, [payment])
       void await cacheGroupedPaymentsAppend(paymentsGroupedByKey)

services/chronikService.ts

@@ -298,6 +298,7 @@ export class ChronikBlockchainClient {
       timestamp: transaction.block !== undefined ? transaction.block.timestamp : transaction.timeFirstSeen,
       addressId: address.id,
       confirmed: transaction.block !== undefined,
+      isPayment: amount > 0,
       opReturn,
       inputs: {
         create: inputAddresses

services/transactionService.ts

@@ -183,7 +183,8 @@ export async function fetchTransactionsByAddressListWithPagination (
   pageSize: number,
   orderBy?: string,
   orderDesc = true,
-  networkIdsListFilter?: number[]
+  networkIdsListFilter?: number[],
+  includeInputs = false
 ): Promise<TransactionsWithPaybuttonsAndPrices[]> {
   const orderDescString: Prisma.SortOrder = orderDesc ? 'desc' : 'asc'
 
@@ -209,6 +210,14 @@ export async function fetchTransactionsByAddressListWithPagination (
     }
   }
 
+  // Build include conditionally - exclude inputs by default unless explicitly requested
+  const include = includeInputs
+    ? includePaybuttonsAndPricesAndInvoices
+    : (() => {
+        const { inputs, ...rest } = includePaybuttonsAndPricesAndInvoices
+        return rest
+      })()
+
   return await prisma.transaction.findMany({
     where: {
       addressId: {
@@ -220,11 +229,11 @@ export async function fetchTransactionsByAddressListWithPagination (
         }
       }
     },
-    include: includePaybuttonsAndPricesAndInvoices,
+    include,
     orderBy: orderByQuery,
     skip: page * pageSize,
     take: pageSize
-  })
+  }) as unknown as TransactionsWithPaybuttonsAndPrices[]
 }
 
 export async function fetchTxCountByAddressString (addressString: string): Promise<number> {
@@ -570,6 +579,7 @@ export async function createManyTransactions (
     timestamp: tx.timestamp,
     addressId: tx.addressId,
     confirmed: tx.confirmed ?? false,
+    isPayment: tx.amount > 0,
     opReturn: tx.opReturn ?? '',
     orphaned: false
   }))
@@ -823,15 +833,19 @@ export async function fetchTransactionsByPaybuttonIdWithPagination (
   pageSize: number,
   orderDesc: boolean,
   orderBy?: string,
-  networkIds?: number[]): Promise<TransactionsWithPaybuttonsAndPrices[]> {
+  networkIds?: number[],
+  includeInputs = false
+): Promise<TransactionsWithPaybuttonsAndPrices[]> {
   const addressIdList = await fetchAddressesByPaybuttonId(paybuttonId)
   const transactions = await fetchTransactionsByAddressListWithPagination(
     addressIdList,
     page,
     pageSize,
     orderBy,
     orderDesc,
-    networkIds)
+    networkIds,
+    includeInputs
+  )
 
   return transactions
 }
@@ -935,7 +949,7 @@ export async function getPaymentsByUserIdOrderedByButtonName (
     LEFT JOIN \`PricesOnTransactions\` pt ON t.\`id\` = pt.\`transactionId\`
     LEFT JOIN \`Price\` pb ON pt.\`priceId\` = pb.\`id\`
     LEFT JOIN \`Invoice\` i ON i.\`transactionId\` = t.\`id\`
-    WHERE t.\`amount\` > 0
+    WHERE t.\`isPayment\` = TRUE
     AND EXISTS (
       SELECT 1
       FROM \`AddressesOnUserProfiles\` au
@@ -1005,7 +1019,8 @@ export async function fetchAllPaymentsByUserIdWithPagination (
   buttonIds?: string[],
   years?: string[],
   startDate?: string,
-  endDate?: string
+  endDate?: string,
+  includeInputs = false
 ): Promise<Payment[]> {
   const orderDescString: Prisma.SortOrder = orderDesc ? 'desc' : 'asc'
 
@@ -1042,7 +1057,7 @@ export async function fetchAllPaymentsByUserIdWithPagination (
     address: {
       userProfiles: { some: { userId } }
     },
-    amount: { gt: 0 }
+    isPayment: true
   }
 
   if (startDate !== undefined && endDate !== undefined && startDate !== '' && endDate !== '') {
@@ -1061,9 +1076,17 @@ export async function fetchAllPaymentsByUserIdWithPagination (
     }
   }
 
+  // Build include conditionally - exclude inputs by default unless explicitly requested
+  const include = includeInputs
+    ? includePaybuttonsAndPricesAndInvoices
+    : (() => {
+        const { inputs, ...rest } = includePaybuttonsAndPricesAndInvoices
+        return rest
+      })()
+
   const transactions = await prisma.transaction.findMany({
     where,
-    include: includePaybuttonsAndPricesAndInvoices,
+    include,
     orderBy: orderByQuery,
     skip: page * Number(pageSize),
     take: Number(pageSize)
@@ -1073,7 +1096,7 @@ export async function fetchAllPaymentsByUserIdWithPagination (
   for (let index = 0; index < transactions.length; index++) {
     const tx = transactions[index]
     if (Number(tx.amount) > 0) {
-      const payment = await generatePaymentFromTxWithInvoices(tx, userId)
+      const payment = generatePaymentFromTxWithInvoices(tx as unknown as TransactionWithAddressAndPricesAndInvoices, userId)
       transformedData.push(payment)
     }
   }
@@ -1160,9 +1183,7 @@ export async function fetchAllPaymentsByUserId (
         in: networkIds ?? Object.values(NETWORK_IDS)
       }
     },
-    amount: {
-      gt: 0
-    }
+    isPayment: true
   }
 
   if (buttonIds !== undefined && buttonIds.length > 0) {
@@ -1216,7 +1237,7 @@ export const getFilteredTransactionCount = async (
         some: { userId }
       }
     },
-    amount: { gt: 0 }
+    isPayment: true
   }
   if (buttonIds !== undefined && buttonIds.length > 0) {
     where.address!.paybuttons = {
@@ -1243,8 +1264,7 @@ export const fetchDistinctPaymentYearsByUser = async (userId: string): Promise<n
     FROM Transaction t
     JOIN Address a ON a.id = t.addressId
     JOIN AddressesOnUserProfiles ap ON ap.addressId = a.id
-    WHERE ap.userId = ${userId} AND
-    t.amount > 0
+    WHERE ap.userId = ${userId}
     ORDER BY year ASC
   `