perf: reduce duplicate push audit storage

.github/workflows/ci.yml

@@ -72,12 +72,12 @@ jobs:
         env:
           RUN_MONGO_TESTS: 'true'
           GIT_PROXY_MONGO_CONNECTION_STRING: mongodb://localhost:27017/git-proxy-test
-        run: npm run test:integration
+        run: npm run test:integration -- --coverage --coverage.reporter=lcovonly --coverage.reportsDirectory=coverage/integration
 
       - name: Upload test coverage report
         uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
         with:
-          files: ./coverage/lcov.info,./coverage-cli/lcov.info
+          files: ./coverage/lcov.info,./coverage/integration/lcov.info,./coverage-cli/lcov.info
           token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Build frontend
@@ -140,7 +140,13 @@ jobs:
         env:
           RUN_POSTGRES_TESTS: 'true'
           GIT_PROXY_POSTGRES_CONNECTION_STRING: postgresql://postgres:postgres@localhost:5432/git_proxy_test
-        run: npm run test:integration:postgres
+        run: npm run test:integration:postgres -- --coverage --coverage.reporter=lcovonly --coverage.reportsDirectory=coverage/postgres
+
+      - name: Upload PostgreSQL coverage report
+        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
+        with:
+          files: ./coverage/postgres/lcov.info
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   # Windows build - single combination for development support
   build-windows:

src/db/file/pushes.ts

@@ -20,6 +20,8 @@ import { activityPrimaryStatusFromFlags } from '../../activity/activityPrimarySt
 import { canonicalRemoteUrl } from '../../activity/canonicalRemoteUrl';
 import { Action } from '../../proxy/actions/Action';
 import { toClass } from '../helper';
+import { pushListProjection } from '../pushProjection';
+import { compactPush, restorePush } from '../pushStorage';
 import {
   PushQuery,
   RepoActivityTabCounts,
@@ -150,12 +152,6 @@ const defaultPushQuery: Partial<PushQuery> = {
   type: 'push',
 };
 
-// List queries drop `steps` and `diff` from the returned document: both hold the
-// full diff (largest part of a push row), and the mongo and postgres list
-// projections exclude them as well. The push-detail path (`getPush`) still returns
-// the whole document.
-const pushListProjection = { steps: 0, diff: 0 } as const;
-
 export const getPushes = (query: Partial<PushQuery>): Promise<Action[]> => {
   if (!query) query = defaultPushQuery;
   return new Promise((resolve, reject) => {
@@ -213,7 +209,7 @@ export const getPush = async (id: string): Promise<Action | null> => {
         if (!doc) {
           resolve(null);
         } else {
-          resolve(toClass(doc, Action.prototype));
+          resolve(restorePush(doc));
         }
       }
     });
@@ -237,7 +233,7 @@ export const deletePush = async (id: string): Promise<void> => {
 export const writeAudit = async (action: Action): Promise<void> => {
   return new Promise((resolve, reject) => {
     const options = { multi: false, upsert: true };
-    db.update({ id: action.id }, action, options, (err) => {
+    db.update({ id: action.id }, compactPush(action), options, (err) => {
       // ignore for code coverage as neDB rarely returns errors even for an invalid query
       /* istanbul ignore if */
       if (err) {

src/db/mongo/pushes.ts

@@ -18,7 +18,8 @@ import { activityPrimaryStatusFromFlags } from '../../activity/activityPrimarySt
 import { canonicalRemoteUrl } from '../../activity/canonicalRemoteUrl';
 import { connect, findDocuments, findOneDocument } from './helper';
 import { Action } from '../../proxy/actions';
-import { toClass } from '../helper';
+import { pushListProjection } from '../pushProjection';
+import { compactPush, restorePush } from '../pushStorage';
 import {
   PushQuery,
   RepoActivityTabCounts,
@@ -120,33 +121,6 @@ const defaultPushQuery: Partial<PushQuery> = {
   type: 'push',
 };
 
-/** Fields returned for push list / activity UIs (shared by getPushes and getPushesForUserProfile). */
-const pushListProjection = {
-  _id: 0,
-  id: 1,
-  allowPush: 1,
-  attestation: 1,
-  authorised: 1,
-  blocked: 1,
-  blockedMessage: 1,
-  branch: 1,
-  canceled: 1,
-  commitData: 1,
-  commitFrom: 1,
-  commitTo: 1,
-  error: 1,
-  method: 1,
-  project: 1,
-  rejected: 1,
-  rejection: 1,
-  repo: 1,
-  repoName: 1,
-  timestamp: 1,
-  type: 1,
-  url: 1,
-  userEmail: 1,
-} as const;
-
 export const getPushes = async (
   query: Partial<PushQuery> = defaultPushQuery,
 ): Promise<Action[]> => {
@@ -169,7 +143,7 @@ export const getPushesForUserProfile = async (
 
 export const getPush = async (id: string): Promise<Action | null> => {
   const doc = await findOneDocument<Action>(collectionName, { id });
-  return doc ? (toClass(doc, Action.prototype) as Action) : null;
+  return doc ? restorePush(doc) : null;
 };
 
 export const deletePush = async function (id: string): Promise<void> {
@@ -178,14 +152,16 @@ export const deletePush = async function (id: string): Promise<void> {
 };
 
 export const writeAudit = async (action: Action): Promise<void> => {
-  const data = JSON.parse(JSON.stringify(action));
+  const data = JSON.parse(JSON.stringify(compactPush(action)));
   const options = { upsert: true };
   const collection = await connect(collectionName);
   delete data._id;
   if (typeof data.id !== 'string') {
     throw new Error('Invalid id');
   }
-  await collection.updateOne({ id: data.id }, { $set: data }, options);
+  const unset: Record<string, ''> =
+    data._lastStepIndex === undefined ? { _lastStepIndex: '' } : { lastStep: '' };
+  await collection.updateOne({ id: data.id }, { $set: data, $unset: unset }, options);
 };
 
 export const authorise = async (

src/db/postgres/pushes.ts

@@ -18,7 +18,8 @@ import { activityPrimaryStatusFromFlags } from '../../activity/activityPrimarySt
 import { canonicalRemoteUrl } from '../../activity/canonicalRemoteUrl';
 import { Action } from '../../proxy/actions';
 import { CompletedAttestation, Rejection } from '../../proxy/processors/types';
-import { toClass } from '../helper';
+import { pushListProjection } from '../pushProjection';
+import { compactPush, restorePush } from '../pushStorage';
 import {
   emptyRepoActivityTabCounts,
   PushQuery,
@@ -47,8 +48,14 @@ const FILTER_COLUMNS: Record<string, string> = {
   type: 'type',
 };
 
-const rowToAction = (row: { data: unknown }): Action =>
-  toClass(row.data, Action.prototype) as Action;
+const rowToAction = (row: { data: unknown }): Action => restorePush(row.data);
+
+const listFields = Object.entries(pushListProjection)
+  .filter(([, included]) => included)
+  .map(([field]) => `'${field}'`)
+  .join(', ');
+const listData = `(SELECT COALESCE(jsonb_object_agg(field, data->field), '{}'::jsonb)
+  FROM unnest(ARRAY[${listFields}]) AS field WHERE data ? field)`;
 
 function bumpCount(
   m: Map<string, RepoActivityTabCounts>,
@@ -163,16 +170,12 @@ export const getPushesForUserProfile = async (
   }
 
   const result = await query<{ data: unknown }>(
-    `SELECT data - 'steps' - 'diff' AS data FROM pushes WHERE type = 'push' AND ${predicate} ORDER BY timestamp DESC`,
+    `SELECT ${listData} AS data FROM pushes WHERE type = 'push' AND ${predicate} ORDER BY timestamp DESC`,
     values,
   );
   return result.rows.map(rowToAction);
 };
 
-// List queries drop `steps` and `diff` from the returned document: both hold the
-// full diff (largest part of a push row) and the mongo backend's list projection
-// excludes them as well. The push-detail path (`getPush`) still returns the whole
-// document.
 export const getPushes = async (q: Partial<PushQuery> = defaultPushQuery): Promise<Action[]> => {
   const clauses: string[] = [];
   const values: unknown[] = [];
@@ -185,7 +188,7 @@ export const getPushes = async (q: Partial<PushQuery> = defaultPushQuery): Promi
 
   const where = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
   const result = await query<{ data: unknown }>(
-    `SELECT data - 'steps' - 'diff' AS data FROM pushes ${where} ORDER BY timestamp DESC`,
+    `SELECT ${listData} AS data FROM pushes ${where} ORDER BY timestamp DESC`,
     values,
   );
   return result.rows.map(rowToAction);
@@ -208,7 +211,7 @@ const buildAuditUpsert = (action: Action): { text: string; values: unknown[] } =
 
   // Round-trip through JSON to drop class identity / mongo-specific _id fields
   // before persisting (mirrors mongo's `JSON.parse(JSON.stringify(action))`).
-  const data = JSON.parse(JSON.stringify(action));
+  const data = JSON.parse(JSON.stringify(compactPush(action)));
   delete data._id;
 
   return {

src/db/pushProjection.ts

@@ -0,0 +1,56 @@
+/**
+ * Copyright 2026 GitProxy Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Action } from '../proxy/actions';
+
+/**
+ * List and profile responses contain metadata only. Keep every diff representation
+ * (steps, lastStep, and the top-level diff) out of this allowlist.
+ * Detail reads and audit writes retain the complete record.
+ */
+export const pushListProjection = {
+  _id: 0,
+  actionType: 1,
+  allowPush: 1,
+  attestation: 1,
+  authorised: 1,
+  autoApproved: 1,
+  autoRejected: 1,
+  blocked: 1,
+  blockedMessage: 1,
+  branch: 1,
+  canceled: 1,
+  commitData: 1,
+  commitFrom: 1,
+  commitTo: 1,
+  error: 1,
+  id: 1,
+  legacyId: 1,
+  method: 1,
+  project: 1,
+  pusherVerified: 1,
+  rejected: 1,
+  rejection: 1,
+  repo: 1,
+  repoName: 1,
+  tagData: 1,
+  tags: 1,
+  timestamp: 1,
+  type: 1,
+  url: 1,
+  user: 1,
+  userEmail: 1,
+} as const satisfies Partial<Record<keyof Action | '_id', 0 | 1>>;

src/db/pushStorage.ts

@@ -0,0 +1,60 @@
+/**
+ * Copyright 2026 GitProxy Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { isEqual } from 'lodash';
+import { Action } from '../proxy/actions';
+import { toClass } from './helper';
+
+// Internal storage markers that may exist on persisted records but must never
+// reach callers of the database layer.
+type StoredPush = Action & { _lastStepIndex?: number; _activity?: unknown };
+
+/**
+ * Prepares a push for storage. Works on a shallow copy, so the caller's Action
+ * is not modified. Any internal markers already on the input are dropped
+ * first. When `lastStep` is identical to the final entry in `steps`, the
+ * duplicate copy is replaced with `_lastStepIndex`, a reference to that entry.
+ * A `lastStep` that differs from the final step is stored as-is.
+ */
+export const compactPush = (action: Action) => {
+  const stored: Partial<StoredPush> = { ...action };
+  delete stored._lastStepIndex;
+  delete stored._activity;
+  const index = action.steps?.length - 1;
+  if (action.lastStep && index >= 0 && isEqual(action.lastStep, action.steps[index])) {
+    stored._lastStepIndex = index;
+    delete stored.lastStep;
+  }
+  return stored;
+};
+
+/**
+ * Reverses compactPush when reading a stored record back into an Action. If
+ * the record holds a valid `_lastStepIndex` and no `lastStep`, `lastStep` is
+ * rebuilt from that entry in `steps`. Records that store `lastStep` directly
+ * (a distinct last step, or historical records) keep it as stored. Internal
+ * markers are always removed from the returned Action.
+ */
+export const restorePush = (doc: unknown): Action => {
+  const action: StoredPush = toClass(doc, Action.prototype);
+  const index = action._lastStepIndex;
+  if (index !== undefined && Number.isInteger(index) && index >= 0 && !action.lastStep) {
+    action.lastStep = action.steps?.[index];
+  }
+  delete action._lastStepIndex;
+  delete action._activity;
+  return action;
+};

src/proxy/processors/push-action/getDiff.ts

@@ -50,7 +50,7 @@ const exec = async (_req: Request, action: Action): Promise<Action> => {
     const revisionRange = `${commitFrom}..${action.commitTo}`;
     const diff = await git.diff([revisionRange]);
     action.setDiff(diff);
-    step.log(diff);
+    step.log(`Generated diff (${Buffer.byteLength(diff, 'utf8')} bytes)`);
   } catch (error: unknown) {
     const msg = getErrorMessage(error);
     step.setError(msg);

src/proxy/processors/push-action/scanDiff.ts

@@ -181,7 +181,6 @@ const exec = async (_req: Request, action: Action): Promise<Action> => {
   step.log(`Scanning diff: ${commitFrom}:${commitTo}`);
   const diff = action.diff ?? steps.find((s) => s.stepName === 'diff')?.content;
 
-  step.log(diff);
   const diffViolations = getDiffViolations(diff, action.project, step);
 
   if (diffViolations) {

test/db/file/pushStorage.test.ts

@@ -0,0 +1,20 @@
+/**
+ * Copyright 2026 GitProxy Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as pushes from '../../../src/db/file/pushes';
+import { definePushStorageContract } from '../pushStorage.contract';
+
+definePushStorageContract('NeDB push storage', pushes, (id) => pushes.db.findOneAsync({ id }));

test/db/file/pushes.test.ts

@@ -16,6 +16,7 @@
 
 import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
 import * as pushesModule from '../../../src/db/file/pushes';
+import { pushListProjection } from '../../../src/db/pushProjection';
 
 describe('File DB - Pushes', () => {
   beforeEach(() => {
@@ -116,8 +117,8 @@ describe('File DB - Pushes', () => {
       // row, so neither may reach a list response. The detail view still returns
       // the whole document.
       expect(projection).toHaveBeenCalledTimes(2);
-      expect(projection).toHaveBeenNthCalledWith(1, { steps: 0, diff: 0 });
-      expect(projection).toHaveBeenNthCalledWith(2, { steps: 0, diff: 0 });
+      expect(projection).toHaveBeenNthCalledWith(1, pushListProjection);
+      expect(projection).toHaveBeenNthCalledWith(2, pushListProjection);
       expect(pushesModule.db.findOne).toHaveBeenCalledWith({ id: 'p1' }, expect.any(Function));
     });
   });