Merge pull request #887 from PayButton/fix/slow-page-loads

[#540] fix: slow page loads

Author: Klakurka

pages/api/address/transactions/count/[address].ts

@@ -1,5 +1,5 @@
 import { NextApiRequest, NextApiResponse } from 'next'
-import { fetchTxCount } from 'services/transactionService'
+import { fetchTxCountByAddressString } from 'services/transactionService'
 import { RESPONSE_MESSAGES } from 'constants/index'
 import { parseAddress } from 'utils/validators'
 
@@ -10,7 +10,7 @@ export default async (req: NextApiRequest, res: NextApiResponse): Promise<void>
         throw new Error(RESPONSE_MESSAGES.ADDRESS_NOT_PROVIDED_400.message)
       }
       const address = parseAddress(req.query.address as string)
-      const count = await fetchTxCount(address)
+      const count = await fetchTxCountByAddressString(address)
       res.status(200).send(count)
     } catch (err: any) {
       switch (err.message) {

redis/dashboardCache.ts

@@ -1,10 +1,9 @@
 import { redis } from './clientInstance'
-import { getPaymentList } from 'redis/paymentCache'
-import { ChartData, PeriodData, DashboardData, Payment, ButtonData, PaymentDataByButton, ChartColor } from './types'
+import { getCachedWeekKeysForUser, getPaymentsForWeekKey, getPaymentStream } from 'redis/paymentCache'
+import { ChartData, DashboardData, Payment, ButtonData, PaymentDataByButton, ChartColor, PeriodData, ButtonDisplayData } from './types'
 import { Prisma } from '@prisma/client'
 import moment, { DurationInputArg2 } from 'moment'
 import { XEC_NETWORK_ID, BCH_NETWORK_ID } from 'constants/index'
-import { fetchPaybuttonArrayByUserId } from 'services/paybuttonService'
 import { QuoteValues } from 'services/priceService'
 
 // USERID:dashboard
@@ -16,30 +15,6 @@ const getChartLabels = function (n: number, periodString: string, formatString =
   return [...new Array(n)].map((_, idx) => moment().startOf('day').subtract(idx, periodString as DurationInputArg2).format(formatString)).reverse()
 }
 
-interface RevenuePaymentData {
-  revenue: QuoteValues[]
-  payments: number[]
-}
-
-const getChartRevenuePaymentData = function (n: number, periodString: string, paymentList: Payment[]): RevenuePaymentData {
-  const revenueArray: QuoteValues[] = []
-  const paymentsArray: number[] = []
-  const _ = [...new Array(n)]
-  _.forEach((_, idx) => {
-    const lowerThreshold = moment().subtract(idx, periodString as DurationInputArg2).startOf(periodString === 'months' ? 'month' : 'day')
-    const upperThreshold = moment().subtract(idx, periodString as DurationInputArg2).endOf(periodString === 'months' ? 'month' : 'day')
-    const periodPaymentList = filterLastPayments(lowerThreshold, upperThreshold, paymentList)
-    const revenue = sumPaymentsValue(periodPaymentList)
-    const paymentCount = periodPaymentList.length
-    revenueArray.push(revenue)
-    paymentsArray.push(paymentCount)
-  })
-  return {
-    revenue: revenueArray.reverse(),
-    payments: paymentsArray.reverse()
-  }
-}
-
 const filterLastPayments = function (lowerThreshold: moment.Moment, upperThreshold: moment.Moment, paymentList: Payment[]): Payment[] {
   return paymentList.filter((p) => {
     const tMoment = moment(p.timestamp * 1000)
@@ -64,27 +39,24 @@ const getChartData = function (n: number, periodString: string, dataArray: numbe
   }
 }
 
-const getPeriodData = function (n: number, periodString: string, paymentList: Payment[], borderColor: ChartColor, formatString = 'M/D'): PeriodData {
-  const revenuePaymentData = getChartRevenuePaymentData(n, periodString, paymentList)
-  const revenue = getChartData(n, periodString, revenuePaymentData.revenue, borderColor.revenue, formatString)
-  const payments = getChartData(n, periodString, revenuePaymentData.payments, borderColor.payments, formatString)
-  const buttons = getButtonPaymentData(n, periodString, paymentList)
-  const totalRevenue = (revenue.datasets[0].data as QuoteValues[]).reduce(sumQuoteValues, { usd: new Prisma.Decimal(0), cad: new Prisma.Decimal(0) })
-  const totalPayments = (payments.datasets[0].data as any).reduce((a: number, b: number) => a + b, 0)
+function getOldestDateKey (keys: string[]): string {
+  const keyDatePairs = keys.map(k => [k, k.split(':').slice(-2).map(Number)] as [string, [number, number]])
+  keyDatePairs.sort((a, b) => {
+    const [aYear, aWeek] = a[1]
+    const [bYear, bWeek] = b[1]
 
-  return {
-    revenue,
-    payments,
-    totalRevenue,
-    totalPayments,
-    buttons
-  }
+    // compare year first, then week
+    return (aYear !== bYear ? aYear - bYear : aWeek - bWeek)
+  })
+  return keyDatePairs[0][0]
 }
 
-const getNumberOfMonths = function (paymentList: Payment[]): number {
-  if (paymentList.length === 0) return 0
-  const oldestTimestamp = Math.min(...paymentList.map(p => p.timestamp)
-  )
+const getNumberOfMonths = async function (userId: string): Promise<number> {
+  const weekKeys = await getCachedWeekKeysForUser(userId)
+  if (weekKeys.length === 0) return 0
+  const oldestKey = getOldestDateKey(weekKeys)
+  const oldestPayments = await getPaymentsForWeekKey(oldestKey)
+  const oldestTimestamp = Math.min(...oldestPayments.map(p => p.timestamp))
   const oldestDate = moment(oldestTimestamp * 1000)
   const today = moment()
   const floatDiff = today.diff(oldestDate, 'months', true)
@@ -147,32 +119,243 @@ export const sumPaymentsValue = function (paymentList: Payment[]): QuoteValues {
   return ret
 }
 
-export const getUserDashboardData = async function (userId: string): Promise<DashboardData> {
-  let dashboardData = await getCachedDashboardData(userId)
-  if (dashboardData === null) {
-    const buttons = await fetchPaybuttonArrayByUserId(userId)
-    const paymentList = await getPaymentList(userId)
-
-    const totalRevenue = sumPaymentsValue(paymentList)
-    const nMonthsTotal = getNumberOfMonths(paymentList)
-
-    const thirtyDays: PeriodData = getPeriodData(30, 'days', paymentList, { revenue: '#66fe91', payments: '#669cfe' })
-    const sevenDays: PeriodData = getPeriodData(7, 'days', paymentList, { revenue: '#66fe91', payments: '#669cfe' })
-    const year: PeriodData = getPeriodData(12, 'months', paymentList, { revenue: '#66fe91', payments: '#669cfe' }, 'MMM')
-    const all: PeriodData = getPeriodData(nMonthsTotal, 'months', paymentList, { revenue: '#66fe91', payments: '#669cfe' }, 'MMM YYYY')
-
-    dashboardData = {
-      thirtyDays,
-      sevenDays,
-      year,
-      all,
-      paymentList,
-      total: {
-        revenue: totalRevenue,
-        payments: paymentList.length,
-        buttons: buttons.length
+const generateDashboardDataFromStream = async function (
+  paymentStream: AsyncGenerator<Payment>,
+  nMonthsTotal: number,
+  borderColor: ChartColor
+): Promise<DashboardData> {
+  // Initialize accumulators for periods
+  const revenueAccumulators = createRevenueAccumulators(nMonthsTotal)
+  const paymentCounters = createPaymentCounters(nMonthsTotal)
+  const buttonDataAccumulators = createButtonDataAccumulators()
+
+  const today = moment().startOf('day')
+  const monthStart = moment().startOf('month')
+  const thresholds = createThresholds(today, monthStart, nMonthsTotal)
+
+  // Process all payments
+  for await (const payment of paymentStream) {
+    const paymentTime = moment(payment.timestamp * 1000)
+
+    // Process button data and assign to relevant periods
+    payment.buttonDisplayDataList.forEach((button) => {
+      processButtonData(button, payment, paymentTime, buttonDataAccumulators, thresholds)
+    })
+
+    // Accumulate period data
+    const periods = ['thirtyDays', 'sevenDays', 'year', 'all'] as const
+    periods.forEach((period) => {
+      if (paymentTime.isSameOrAfter(thresholds[period])) {
+        const index =
+          period === 'thirtyDays' || period === 'sevenDays'
+            ? today.diff(paymentTime, 'days')
+            : monthStart.diff(paymentTime, 'months')
+        if (index < revenueAccumulators[period].length) {
+          revenueAccumulators[period][index] = sumQuoteValues(revenueAccumulators[period][index], payment.values)
+          paymentCounters[period][index] += 1
+        }
       }
+    })
+  }
+
+  reverseAccumulators(revenueAccumulators, paymentCounters)
+
+  // Generate PeriodData for each period
+  const thirtyDays = createPeriodData(
+    30,
+    'days',
+    revenueAccumulators.thirtyDays,
+    paymentCounters.thirtyDays,
+    buttonDataAccumulators.thirtyDays,
+    borderColor.revenue,
+    borderColor.payments
+  )
+
+  const sevenDays = createPeriodData(
+    7,
+    'days',
+    revenueAccumulators.sevenDays,
+    paymentCounters.sevenDays,
+    buttonDataAccumulators.sevenDays,
+    borderColor.revenue,
+    borderColor.payments
+  )
+
+  const year = createPeriodData(
+    12,
+    'months',
+    revenueAccumulators.year,
+    paymentCounters.year,
+    buttonDataAccumulators.year,
+    borderColor.revenue,
+    borderColor.payments,
+    'MMM'
+  )
+
+  const all = createPeriodData(
+    nMonthsTotal,
+    'months',
+    revenueAccumulators.all,
+    paymentCounters.all,
+    buttonDataAccumulators.all,
+    borderColor.revenue,
+    borderColor.payments,
+    'MMM YYYY'
+  )
+
+  return {
+    thirtyDays,
+    sevenDays,
+    year,
+    all,
+    total: {
+      revenue: all.totalRevenue,
+      payments: all.totalPayments,
+      buttons: Object.keys(buttonDataAccumulators.all).length
     }
+  }
+}
+
+interface PeriodRevenueAccumulators {
+  thirtyDays: QuoteValues[]
+  sevenDays: QuoteValues[]
+  year: QuoteValues[]
+  all: QuoteValues[]
+}
+
+function createRevenueAccumulators (nMonthsTotal: number): PeriodRevenueAccumulators {
+  return {
+    thirtyDays: Array(30).fill({ usd: new Prisma.Decimal(0), cad: new Prisma.Decimal(0) }),
+    sevenDays: Array(7).fill({ usd: new Prisma.Decimal(0), cad: new Prisma.Decimal(0) }),
+    year: Array(12).fill({ usd: new Prisma.Decimal(0), cad: new Prisma.Decimal(0) }),
+    all: Array(nMonthsTotal).fill({ usd: new Prisma.Decimal(0), cad: new Prisma.Decimal(0) })
+  }
+}
+
+interface PeriodPaymentCounters {
+  thirtyDays: number[]
+  sevenDays: number[]
+  year: number[]
+  all: number[]
+}
+
+function createPaymentCounters (nMonthsTotal: number): PeriodPaymentCounters {
+  return {
+    thirtyDays: Array(30).fill(0),
+    sevenDays: Array(7).fill(0),
+    year: Array(12).fill(0),
+    all: Array(nMonthsTotal).fill(0)
+  }
+}
+
+interface PeriodButtonDataAccumulators {
+  thirtyDays: PaymentDataByButton
+  sevenDays: PaymentDataByButton
+  year: PaymentDataByButton
+  all: PaymentDataByButton
+}
+
+function createButtonDataAccumulators (): PeriodButtonDataAccumulators {
+  return {
+    thirtyDays: {},
+    sevenDays: {},
+    year: {},
+    all: {}
+  }
+}
+
+interface PeriodThresholds {
+  thirtyDays: moment.Moment
+  sevenDays: moment.Moment
+  year: moment.Moment
+  all: moment.Moment
+}
+
+function createThresholds (today: moment.Moment, monthStart: moment.Moment, nMonthsTotal: number): PeriodThresholds {
+  return {
+    thirtyDays: today.clone().subtract(30, 'days'),
+    sevenDays: today.clone().subtract(7, 'days'),
+    year: monthStart.clone().subtract(12, 'months'),
+    all: monthStart.clone().subtract(nMonthsTotal, 'months')
+  }
+}
+
+function processButtonData (
+  button: ButtonDisplayData,
+  payment: Payment,
+  paymentTime: moment.Moment,
+  buttonDataAccumulators: ReturnType<typeof createButtonDataAccumulators>,
+  thresholds: ReturnType<typeof createThresholds>
+): void {
+  const periods = ['thirtyDays', 'sevenDays', 'year', 'all'] as const
+  periods.forEach((period) => {
+    if (paymentTime.isSameOrAfter(thresholds[period])) {
+      if (buttonDataAccumulators[period][button.id] === undefined) {
+        buttonDataAccumulators[period][button.id] = {
+          displayData: {
+            ...button,
+            isXec: payment.networkId === XEC_NETWORK_ID,
+            isBch: payment.networkId === BCH_NETWORK_ID,
+            lastPayment: payment.timestamp
+          },
+          total: {
+            revenue: payment.values,
+            payments: 1
+          }
+        }
+      } else {
+        const buttonData = buttonDataAccumulators[period][button.id]
+        buttonData.total.revenue = sumQuoteValues(buttonData.total.revenue, payment.values)
+        buttonData.total.payments += 1
+        buttonData.displayData.lastPayment = Math.max(
+          buttonData.displayData.lastPayment ?? 0,
+          payment.timestamp
+        )
+      }
+    }
+  })
+}
+
+function reverseAccumulators (
+  revenueAccumulators: ReturnType<typeof createRevenueAccumulators>,
+  paymentCounters: ReturnType<typeof createPaymentCounters>
+): void {
+  Object.keys(revenueAccumulators).forEach((key) => {
+    revenueAccumulators[key as keyof typeof revenueAccumulators].reverse()
+    paymentCounters[key as keyof typeof paymentCounters].reverse()
+  })
+}
+
+function createPeriodData (
+  periodLength: number,
+  periodUnit: string,
+  revenueData: QuoteValues[],
+  paymentData: number[],
+  buttonData: PaymentDataByButton,
+  revenueColor: string,
+  paymentColor: string,
+  labelFormat = 'M/D'
+): PeriodData {
+  return {
+    revenue: getChartData(periodLength, periodUnit, revenueData, revenueColor, labelFormat),
+    payments: getChartData(periodLength, periodUnit, paymentData, paymentColor, labelFormat),
+    totalRevenue: revenueData.reduce(sumQuoteValues, { usd: new Prisma.Decimal(0), cad: new Prisma.Decimal(0) }),
+    totalPayments: paymentData.reduce((a, b) => a + b, 0),
+    buttons: buttonData
+  }
+}
+
+export const getUserDashboardData = async function (userId: string): Promise<DashboardData> {
+  const dashboardData = await getCachedDashboardData(userId)
+  if (dashboardData === null) {
+    const nMonthsTotal = await getNumberOfMonths(userId)
+    const paymentStream = getPaymentStream(userId)
+
+    const dashboardData = await generateDashboardDataFromStream(
+      paymentStream,
+      nMonthsTotal,
+      { revenue: '#66fe91', payments: '#669cfe' }
+    )
     await cacheDashboardData(userId, dashboardData) // WIP SET THIS NULL ON UPDATE BUTTONS & WS
     return dashboardData
   }
@@ -182,7 +365,6 @@ export const getUserDashboardData = async function (userId: string): Promise<Das
 export const cacheDashboardData = async (userId: string, dashboardData: DashboardData): Promise<void> => {
   const key = getDashboardSummaryKey(userId)
   const {
-    paymentList,
     ...cachable
   } = dashboardData
   await redis.set(key, JSON.stringify(cachable))