Add support for table row aggregation and update table version (#2240)

Fixes OPS-4029

Author: bigfluffycookie

compose.yaml

@@ -3,7 +3,7 @@ services:
   tables:
     container_name: tables
     restart: unless-stopped
-    image: public.ecr.aws/openops/openops-tables:0.2.17
+    image: public.ecr.aws/openops/openops-tables:0.2.18
     environment:
       BASEROW_PUBLIC_URL: ${OPS_OPENOPS_TABLES_PUBLIC_URL}
       BASEROW_PRIVATE_URL: http://localhost:3001

deploy/docker-compose/docker-compose.yml

@@ -23,7 +23,7 @@ services:
     environment:
       OPS_COMPONENT: app
       OPS_VERSION: ${OPS_VERSION:-latest}
-      OPS_OPENOPS_TABLES_VERSION: 0.2.17
+      OPS_OPENOPS_TABLES_VERSION: 0.2.18
       OPS_ANALYTICS_VERSION: 0.14.7
     depends_on:
       openops-tables:
@@ -49,7 +49,7 @@ services:
     depends_on:
       - openops-app
   openops-tables:
-    image: public.ecr.aws/openops/openops-tables:0.2.17
+    image: public.ecr.aws/openops/openops-tables:0.2.18
     restart: unless-stopped
     environment:
       BASEROW_PUBLIC_URL: ${OPS_OPENOPS_TABLES_PUBLIC_URL}

packages/openops/src/lib/openops-tables/rows.ts

@@ -257,6 +257,66 @@ function getEqualityFilterType(
   return ViewFilterTypesEnum.equal;
 }
 
+export type AggregationSpec =
+  | { type: 'count' }
+  | { type: 'sum'; field: string }
+  | { type: 'distinct_values'; field: string };
+
+export interface TableFilter {
+  fieldName: string;
+  type: 'not_in';
+  value: string[];
+}
+
+export interface BatchTableAggregationsParams {
+  tokenOrResolver: TokenOrResolver;
+  tableIds: number[];
+  filters?: TableFilter[];
+  aggregations: AggregationSpec[];
+}
+
+export type TableAggregationResult = {
+  count?: number;
+  [key: string]: number | string[] | undefined;
+};
+
+export type BatchTableAggregationsResult = Record<
+  string,
+  TableAggregationResult
+>;
+
+export async function batchTableAggregations(
+  params: BatchTableAggregationsParams,
+): Promise<BatchTableAggregationsResult> {
+  const url = 'api/database/rows/batch-aggregations/';
+
+  return executeWithConcurrencyLimit(
+    async () => {
+      const authenticationHeader = createAxiosHeaders(params.tokenOrResolver);
+      return await makeOpenOpsTablesPost<BatchTableAggregationsResult>(
+        url,
+        {
+          table_ids: params.tableIds,
+          filters: (params.filters ?? []).map((filter) => ({
+            field: filter.fieldName,
+            type: filter.type,
+            value: filter.value,
+          })),
+          aggregations: params.aggregations,
+        },
+        authenticationHeader,
+      );
+    },
+    (error) => {
+      logger.error('Error while posting batch table aggregations:', {
+        error,
+        url,
+        tableIds: params.tableIds,
+      });
+    },
+  );
+}
+
 export async function batchDeleteRows(
   params: BatchDeleteRowsParams,
 ): Promise<void> {

packages/openops/test/openops-tables/rows.test.ts

@@ -55,6 +55,7 @@ import { axiosTablesRetryConfig } from '../../src/lib/openops-tables/requests-he
 import {
   addRow,
   batchDeleteRows,
+  batchTableAggregations,
   deleteRow,
   getRowByPrimaryKeyValue,
   getRows,
@@ -356,6 +357,156 @@ describe('batchDeleteRows', () => {
   });
 });
 
+describe('batchTableAggregations', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  test('posts to correct url with count aggregation', async () => {
+    makeOpenOpsTablesPostMock.mockResolvedValue({ '1': { count: 5 } });
+    createAxiosHeadersMock.mockReturnValue('some header');
+
+    const result = await batchTableAggregations({
+      tokenOrResolver: 'token',
+      tableIds: [1],
+      aggregations: [{ type: 'count' }],
+    });
+
+    expect(result).toStrictEqual({ '1': { count: 5 } });
+    expect(acquireMock).toBeCalledTimes(1);
+    expect(releaseMock).toBeCalledTimes(1);
+    expect(makeOpenOpsTablesPostMock).toBeCalledTimes(1);
+    expect(makeOpenOpsTablesPostMock).toHaveBeenCalledWith(
+      'api/database/rows/batch-aggregations/',
+      { table_ids: [1], filters: [], aggregations: [{ type: 'count' }] },
+      'some header',
+    );
+    expect(createAxiosHeadersMock).toHaveBeenCalledWith('token');
+  });
+
+  test('posts to correct url with sum aggregation', async () => {
+    makeOpenOpsTablesPostMock.mockResolvedValue({
+      '2': { 'sum__Estimated savings USD per month': 27880 },
+    });
+    createAxiosHeadersMock.mockReturnValue('some header');
+
+    const result = await batchTableAggregations({
+      tokenOrResolver: 'token',
+      tableIds: [2],
+      aggregations: [{ type: 'sum', field: 'Estimated savings USD per month' }],
+    });
+
+    expect(result).toStrictEqual({
+      '2': { 'sum__Estimated savings USD per month': 27880 },
+    });
+    expect(makeOpenOpsTablesPostMock).toHaveBeenCalledWith(
+      'api/database/rows/batch-aggregations/',
+      {
+        table_ids: [2],
+        filters: [],
+        aggregations: [
+          { type: 'sum', field: 'Estimated savings USD per month' },
+        ],
+      },
+      'some header',
+    );
+  });
+
+  test('posts multiple aggregations for multiple tables', async () => {
+    makeOpenOpsTablesPostMock.mockResolvedValue({
+      '1': { count: 10, sum__Cost: 500 },
+      '2': { count: 3, sum__Cost: 120 },
+    });
+    createAxiosHeadersMock.mockReturnValue('some header');
+
+    const result = await batchTableAggregations({
+      tokenOrResolver: 'token',
+      tableIds: [1, 2],
+      aggregations: [{ type: 'count' }, { type: 'sum', field: 'Cost' }],
+    });
+
+    expect(result).toStrictEqual({
+      '1': { count: 10, sum__Cost: 500 },
+      '2': { count: 3, sum__Cost: 120 },
+    });
+    expect(makeOpenOpsTablesPostMock).toHaveBeenCalledWith(
+      'api/database/rows/batch-aggregations/',
+      {
+        table_ids: [1, 2],
+        filters: [],
+        aggregations: [{ type: 'count' }, { type: 'sum', field: 'Cost' }],
+      },
+      'some header',
+    );
+  });
+
+  test('passes filters in request body', async () => {
+    makeOpenOpsTablesPostMock.mockResolvedValue({ '1': { count: 2 } });
+    createAxiosHeadersMock.mockReturnValue('some header');
+
+    await batchTableAggregations({
+      tokenOrResolver: 'token',
+      tableIds: [1],
+      filters: [
+        {
+          fieldName: 'Status',
+          type: 'not_in',
+          value: ['Resolved', 'Dismissed'],
+        },
+      ],
+      aggregations: [{ type: 'count' }],
+    });
+
+    expect(makeOpenOpsTablesPostMock).toHaveBeenCalledWith(
+      'api/database/rows/batch-aggregations/',
+      {
+        table_ids: [1],
+        filters: [
+          { field: 'Status', type: 'not_in', value: ['Resolved', 'Dismissed'] },
+        ],
+        aggregations: [{ type: 'count' }],
+      },
+      'some header',
+    );
+  });
+
+  test('defaults filters to empty array when not provided', async () => {
+    makeOpenOpsTablesPostMock.mockResolvedValue({ '1': { count: 0 } });
+    createAxiosHeadersMock.mockReturnValue('some header');
+
+    await batchTableAggregations({
+      tokenOrResolver: 'token',
+      tableIds: [1],
+      aggregations: [{ type: 'count' }],
+    });
+
+    expect(makeOpenOpsTablesPostMock).toHaveBeenCalledWith(
+      'api/database/rows/batch-aggregations/',
+      expect.objectContaining({ filters: [] }),
+      'some header',
+    );
+  });
+
+  test('logs error and rethrows when post fails', async () => {
+    const error = new Error('network error');
+    makeOpenOpsTablesPostMock.mockRejectedValue(error);
+    createAxiosHeadersMock.mockReturnValue('some header');
+
+    await expect(
+      batchTableAggregations({
+        tokenOrResolver: 'token',
+        tableIds: [1],
+        aggregations: [{ type: 'count' }],
+      }),
+    ).rejects.toThrow('network error');
+
+    expect(logger.error).toHaveBeenCalledWith(
+      'Error while posting batch table aggregations:',
+      expect.objectContaining({ error, tableIds: [1] }),
+    );
+  });
+});
+
 describe('getRowByPrimaryKeyValue', () => {
   beforeEach(() => {
     jest.clearAllMocks();