Merge pull request #310 from ShipSecAI/codex/eng-179-redact-sensitive-logs

fix(logging): redact sensitive tokens in workflow logs

Author: betterclever

backend/src/auth/providers/clerk-auth.provider.ts

@@ -69,8 +69,7 @@ export class ClerkAuthProvider implements AuthProviderStrategy {
 
   private async verifyClerkToken(token: string): Promise<ClerkJwt> {
     try {
-      // Log token preview for debugging (first 20 chars)
-      this.logger.log(`[AUTH] Verifying token (preview: ${token.substring(0, 20)}...)`);
+      this.logger.log('[AUTH] Verifying token');
 
       // Add clock skew tolerance to handle server clock differences
       // Clerk tokens can have iat in the future due to clock skew between servers
@@ -88,7 +87,6 @@ export class ClerkAuthProvider implements AuthProviderStrategy {
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
       this.logger.error(`[AUTH] Clerk token verification failed: ${message}`);
-      this.logger.error(`[AUTH] Token preview: ${token.substring(0, 50)}...`);
       this.logger.error(`[AUTH] Secret key configured: ${this.config.secretKey ? 'yes' : 'no'}`);
       throw new UnauthorizedException('Invalid Clerk token');
     }

backend/src/logging/__tests__/log-ingest.service.spec.ts

@@ -0,0 +1,46 @@
+import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test';
+
+import { LogIngestService } from '../log-ingest.service';
+import type { LogStreamRepository } from '../../trace/log-stream.repository';
+
+describe('LogIngestService', () => {
+  const originalEnv = { ...process.env };
+
+  beforeEach(() => {
+    process.env.LOG_KAFKA_BROKERS = 'localhost:9092';
+    process.env.LOKI_URL = 'http://localhost:3100';
+  });
+
+  afterEach(() => {
+    process.env = { ...originalEnv };
+  });
+
+  it('redacts sensitive data before pushing to Loki', async () => {
+    const repository = {
+      upsertMetadata: mock(async () => undefined),
+    } as unknown as LogStreamRepository;
+
+    const service = new LogIngestService(repository);
+    const push = mock(async () => undefined);
+    (service as any).lokiClient = { push };
+
+    await (service as any).processEntry({
+      runId: 'run-1',
+      nodeRef: 'node-1',
+      stream: 'stdout',
+      message: 'token=abc123 authorization=Bearer super-secret',
+      timestamp: '2026-02-21T00:00:00.000Z',
+      organizationId: 'org-1',
+    });
+
+    expect(push).toHaveBeenCalledTimes(1);
+    const call = push.mock.calls[0] as unknown[] | undefined;
+    expect(call).toBeTruthy();
+    const lines = (call?.[1] ?? []) as { message: string }[];
+    expect(lines).toHaveLength(1);
+    expect(lines[0]?.message).toContain('token=[REDACTED]');
+    expect(lines[0]?.message).toContain('authorization=[REDACTED]');
+    expect(lines[0]?.message).not.toContain('abc123');
+    expect(lines[0]?.message).not.toContain('super-secret');
+  });
+});

backend/src/logging/__tests__/redact-sensitive.spec.ts

@@ -0,0 +1,48 @@
+import { describe, expect, it } from 'bun:test';
+
+import { redactSensitiveData } from '../redact-sensitive';
+
+describe('redactSensitiveData', () => {
+  it('redacts common secret key-value pairs', () => {
+    const input =
+      'authorization=Bearer abcdefghijklmnop token=123456 password=hunter2 api_key=xyz987';
+    const redacted = redactSensitiveData(input);
+
+    expect(redacted).toContain('authorization=[REDACTED]');
+    expect(redacted).toContain('token=[REDACTED]');
+    expect(redacted).toContain('password=[REDACTED]');
+    expect(redacted).toContain('api_key=[REDACTED]');
+  });
+
+  it('redacts JSON-style secret fields', () => {
+    const input = '{"access_token":"abc123","client_secret":"super-secret"}';
+    const redacted = redactSensitiveData(input);
+
+    expect(redacted).toBe('{"access_token":"[REDACTED]","client_secret":"[REDACTED]"}');
+  });
+
+  it('redacts token-like standalone values and URL params', () => {
+    const input =
+      'https://example.com?token=abc123&foo=1 Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.aGVsbG8td29ybGQ.signature ghp_abcdefghijklmnopqrstuvwxyz1234 sk-abcdefghijklmnopqrstuvwxyz123456';
+    const redacted = redactSensitiveData(input);
+
+    expect(redacted).toContain('?token=[REDACTED]&foo=1');
+    expect(redacted).not.toContain('eyJhbGciOiJIUzI1Ni');
+    expect(redacted).not.toContain('ghp_abcdefghijklmnopqrstuvwxyz1234');
+    expect(redacted).not.toContain('sk-abcdefghijklmnopqrstuvwxyz123456');
+  });
+
+  it('redacts github clone URLs with embedded x-access-token credentials', () => {
+    const input =
+      'CLONE_URL=https://x-access-token:ghs_abcdefghijklmnopqrstuvwxyz1234567890@github.com/LuD1161/git-test-repo.git';
+    const redacted = redactSensitiveData(input);
+
+    expect(redacted).toContain('CLONE_URL=https://x-access-token:[REDACTED]@github.com/');
+    expect(redacted).not.toContain('ghs_abcdefghijklmnopqrstuvwxyz1234567890');
+  });
+
+  it('preserves non-sensitive text', () => {
+    const input = 'workflow finished successfully in 245ms';
+    expect(redactSensitiveData(input)).toBe(input);
+  });
+});

backend/src/logging/log-ingest.service.ts

@@ -5,6 +5,7 @@ import { getTopicResolver } from '../common/kafka-topic-resolver';
 import { LogStreamRepository } from '../trace/log-stream.repository';
 import type { KafkaLogEntry } from './log-entry.types';
 import { LokiLogClient } from './loki.client';
+import { redactSensitiveData } from './redact-sensitive';
 
 @Injectable()
 export class LogIngestService implements OnModuleInit, OnModuleDestroy {
@@ -108,13 +109,14 @@ export class LogIngestService implements OnModuleInit, OnModuleDestroy {
   }
 
   private async processEntry(entry: KafkaLogEntry): Promise<void> {
-    if (!entry.message || entry.message.trim().length === 0) {
+    const sanitizedMessage = redactSensitiveData(entry.message ?? '');
+    if (!sanitizedMessage || sanitizedMessage.trim().length === 0) {
       return;
     }
 
     const timestamp = entry.timestamp ? new Date(entry.timestamp) : new Date();
     const labels = this.buildLabels(entry);
-    const lines = this.buildLines(entry.message, timestamp);
+    const lines = this.buildLines(sanitizedMessage, timestamp);
     if (!lines.length) {
       return;
     }

backend/src/logging/redact-sensitive.ts

@@ -0,0 +1,42 @@
+const REDACTED = '[REDACTED]';
+
+// Patterns for high-signal secret forms commonly seen in logs.
+const SECRET_KEY_PATTERN =
+  '(?:access_token|refresh_token|id_token|token|api[_-]?key|apikey|client_secret|secret|password|authorization|x-api-key|private_key|session_token)';
+
+const JSON_SECRET_PAIR_REGEX = new RegExp(
+  `("(${SECRET_KEY_PATTERN})"\\s*:\\s*")([^"\\r\\n]{3,})(")`,
+  'gi',
+);
+const AUTH_SCHEME_ASSIGNMENT_REGEX = /\bauthorization\b\s*([=:])\s*(?:Bearer|Basic)\s+[^\s,;&]+/gi;
+const ASSIGNMENT_SECRET_PAIR_REGEX = new RegExp(
+  `(\\b${SECRET_KEY_PATTERN}\\b\\s*[=:]\\s*)([^\\s,;&@]+)`,
+  'gi',
+);
+const URL_SECRET_PARAM_REGEX = new RegExp(`([?&](?:${SECRET_KEY_PATTERN})=)([^&#\\s]+)`, 'gi');
+const BEARER_REGEX = /\bBearer\s+[A-Za-z0-9._~+/=-]{8,}\b/gi;
+const BASIC_REGEX = /\bBasic\s+[A-Za-z0-9+/=]{8,}\b/gi;
+const JWT_REGEX = /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g;
+const GITHUB_TOKEN_REGEX = /\b(?:gh[pousr]_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,})\b/g;
+const GENERIC_SK_TOKEN_REGEX = /\bsk-[A-Za-z0-9]{20,}\b/g;
+
+export function redactSensitiveData(input: string): string {
+  if (!input) {
+    return input;
+  }
+
+  let output = input;
+
+  output = output.replace(JSON_SECRET_PAIR_REGEX, `$1${REDACTED}$4`);
+  output = output.replace(AUTH_SCHEME_ASSIGNMENT_REGEX, `authorization$1${REDACTED}`);
+  output = output.replace(ASSIGNMENT_SECRET_PAIR_REGEX, `$1${REDACTED}`);
+  output = output.replace(URL_SECRET_PARAM_REGEX, `$1${REDACTED}`);
+
+  output = output.replace(BEARER_REGEX, `Bearer ${REDACTED}`);
+  output = output.replace(BASIC_REGEX, `Basic ${REDACTED}`);
+  output = output.replace(JWT_REGEX, REDACTED);
+  output = output.replace(GITHUB_TOKEN_REGEX, REDACTED);
+  output = output.replace(GENERIC_SK_TOKEN_REGEX, REDACTED);
+
+  return output;
+}

backend/src/trace/__tests__/log-stream.service.spec.ts

@@ -147,4 +147,38 @@ describe('LogStreamService', () => {
     expect(calledUrl).toContain(`start=${firstNs}`);
     expect(calledUrl).toContain(`end=${lastNs}`);
   });
+
+  it('redacts sensitive values returned from Loki', async () => {
+    const nanoTs = (BigInt(record.firstTimestamp.getTime()) * 1000000n).toString();
+
+    // @ts-expect-error override global fetch for test
+    global.fetch = async () =>
+      ({
+        ok: true,
+        json: async () => ({
+          data: {
+            result: [
+              {
+                values: [[nanoTs, 'token=abc123 authorization=Bearer super-secret-value']],
+              },
+            ],
+          },
+        }),
+      }) as Response;
+
+    const repository = {
+      listByRunId: async () => [record],
+    } as unknown as LogStreamRepository;
+    const service = new LogStreamService(repository);
+    const result = await service.fetch('run-123', authContext, {
+      nodeRef: 'node-1',
+      stream: 'stdout',
+    });
+
+    expect(result.logs).toHaveLength(1);
+    expect(result.logs[0]?.message).toContain('token=[REDACTED]');
+    expect(result.logs[0]?.message).toContain('authorization=[REDACTED]');
+    expect(result.logs[0]?.message).not.toContain('abc123');
+    expect(result.logs[0]?.message).not.toContain('super-secret-value');
+  });
 });

backend/src/trace/log-stream.service.ts

@@ -3,6 +3,7 @@ import { ForbiddenException, Injectable, ServiceUnavailableException } from '@ne
 import { LogStreamRepository } from './log-stream.repository';
 import type { WorkflowLogStreamRecord } from '../database/schema';
 import type { AuthContext } from '../auth/types';
+import { redactSensitiveData } from '../logging/redact-sensitive';
 
 interface FetchLogsOptions {
   nodeRef?: string;
@@ -215,7 +216,7 @@ export class LogStreamService {
       for (const [timestamp, message] of result.values ?? []) {
         entries.push({
           timestamp: this.fromNanoseconds(timestamp),
-          message,
+          message: this.sanitizeMessage(message),
         });
       }
     }
@@ -265,7 +266,7 @@ export class LogStreamService {
       for (const [timestamp, message] of result.values ?? []) {
         entries.push({
           timestamp: this.fromNanoseconds(timestamp),
-          message,
+          message: this.sanitizeMessage(message),
           level: streamLabels.level,
           nodeId: streamLabels.node,
         });
@@ -336,7 +337,7 @@ export class LogStreamService {
       for (const [timestamp, message] of result.values ?? []) {
         entries.push({
           timestamp: this.fromNanoseconds(timestamp),
-          message,
+          message: this.sanitizeMessage(message),
           level: streamLabels.level,
           nodeId: streamLabels.node,
         });
@@ -421,6 +422,10 @@ export class LogStreamService {
     return new Date(millis).toISOString();
   }
 
+  private sanitizeMessage(message: string): string {
+    return redactSensitiveData(message);
+  }
+
   private requireOrganizationId(auth: AuthContext | null): string {
     const organizationId = auth?.organizationId;
     if (!organizationId) {