Add env variable to not redact in environments where its set to false

Author: bigfluffycookie

packages/server/shared/src/lib/logger/log-cleaner.ts

@@ -1,6 +1,8 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
 import { ApplicationError } from '@openops/shared';
+import { system } from '../system';
+import { SharedSystemProp } from '../system/system-prop';
 
 export const maxFieldLength = 2048;
 
@@ -23,18 +25,26 @@ const SENSITIVE_FIELD_PATTERNS = SENSITIVE_PATTERNS.map(
     new RegExp(String.raw`"[^"]*${pattern}[^"]*"\s*:\s*"[^"]*"`, 'gi'),
 );
 
+const shouldRedactLogs = (): boolean => {
+  return system.getBoolean(SharedSystemProp.REDACT_LOGS) ?? true;
+};
+
 const isSensitiveField = (key: string): boolean => {
+  if (!shouldRedactLogs()) {
+    return false;
+  }
   const lowerKey = key.toLowerCase();
   return SENSITIVE_PATTERNS.some((pattern) => lowerKey.includes(pattern));
 };
 
 const redactSensitiveFields = (obj: any, visited = new WeakSet()): any => {
   try {
-    if (obj === null || obj === undefined) {
-      return obj;
-    }
-
-    if (typeof obj !== 'object') {
+    if (
+      obj === null ||
+      obj === undefined ||
+      typeof obj !== 'object' ||
+      !shouldRedactLogs()
+    ) {
       return obj;
     }
 
@@ -67,7 +77,7 @@ const redactSensitiveFields = (obj: any, visited = new WeakSet()): any => {
 const redactSensitiveDataInString = (
   value: string | undefined,
 ): string | undefined => {
-  if (!value) {
+  if (!value || !shouldRedactLogs()) {
     return value;
   }
   let result = value;

packages/server/shared/src/lib/system/system-prop.ts

@@ -115,6 +115,7 @@ export enum AppSystemProp {
 export enum SharedSystemProp {
   LOG_LEVEL = 'LOG_LEVEL',
   LOG_PRETTY = 'LOG_PRETTY',
+  REDACT_LOGS = 'REDACT_LOGS',
   ENVIRONMENT = 'ENVIRONMENT',
   CONTAINER_TYPE = 'CONTAINER_TYPE',
   TRIGGER_TIMEOUT_SECONDS = 'TRIGGER_TIMEOUT_SECONDS',

packages/server/shared/src/lib/system/system.ts

@@ -59,6 +59,7 @@ const systemPropDefaultValues: Partial<Record<SystemProp, string>> = {
   [WorkerSystemProps.SCHEDULED_WORKER_CONCURRENCY]: '10',
   [SharedSystemProp.LOG_LEVEL]: 'info',
   [SharedSystemProp.LOG_PRETTY]: 'false',
+  [SharedSystemProp.REDACT_LOGS]: 'true',
   [SharedSystemProp.PACKAGE_ARCHIVE_PATH]: 'cache/archives',
   [SharedSystemProp.BLOCKS_SOURCE]: BlocksSource.FILE,
   [AppSystemProp.QUEUE_MODE]: QueueMode.REDIS,

packages/server/shared/test/log-cleaner.test.ts

@@ -135,6 +135,61 @@ describe('log-cleaner', () => {
   });
 
   describe('sensitive data redaction', () => {
+    const originalEnv = process.env['OPS_REDACT_LOGS'];
+
+    afterEach(() => {
+      if (originalEnv === undefined) {
+        delete process.env['OPS_REDACT_LOGS'];
+      } else {
+        process.env['OPS_REDACT_LOGS'] = originalEnv;
+      }
+    });
+
+    it('should redact password field when redaction is enabled', () => {
+      process.env['OPS_REDACT_LOGS'] = 'true';
+      const logEvent = {
+        event: {
+          email: 'user@example.com',
+          password: 'secret',
+        },
+      };
+
+      const result = cleanLogEvent(logEvent);
+
+      expect(result.event.password).toBe('[REDACTED]');
+      expect(result.event.email).toBe('user@example.com');
+    });
+
+    it('should NOT redact password field when redaction is disabled', () => {
+      process.env['OPS_REDACT_LOGS'] = 'false';
+      const logEvent = {
+        event: {
+          email: 'user@example.com',
+          password: 'secret',
+        },
+      };
+
+      const result = cleanLogEvent(logEvent);
+
+      expect(result.event.password).toBe('secret');
+      expect(result.event.email).toBe('user@example.com');
+    });
+
+    it('should redact password field by default when variable is not set', () => {
+      delete process.env['OPS_REDACT_LOGS'];
+      const logEvent = {
+        event: {
+          email: 'user@example.com',
+          password: 'secret',
+        },
+      };
+
+      const result = cleanLogEvent(logEvent);
+
+      expect(result.event.password).toBe('[REDACTED]');
+      expect(result.event.email).toBe('user@example.com');
+    });
+
     it('should redact password field', () => {
       const logEvent = {
         event: {
@@ -150,6 +205,7 @@ describe('log-cleaner', () => {
     });
 
     it('should redact authorization field in objects', () => {
+      process.env['OPS_REDACT_LOGS'] = 'true';
       const logEvent = {
         event: {
           request: {
@@ -166,6 +222,7 @@ describe('log-cleaner', () => {
     });
 
     it('should redact password in stringified JSON', () => {
+      process.env['OPS_REDACT_LOGS'] = 'true';
       const logEvent = {
         event: {
           body: '{"email":"user@example.com","password":"secret"}',
@@ -179,6 +236,7 @@ describe('log-cleaner', () => {
     });
 
     it('should redact password in nested objects', () => {
+      process.env['OPS_REDACT_LOGS'] = 'true';
       const logEvent = {
         event: {
           request: {
@@ -193,6 +251,41 @@ describe('log-cleaner', () => {
       expect(result.event.request).toContain('[REDACTED]');
       expect(result.event.request).not.toContain('secret');
     });
+
+    it('should NOT redact token usage statistics when redaction is disabled', () => {
+      process.env['OPS_REDACT_LOGS'] = 'false';
+      const logEvent = {
+        message: 'Total token usage for stream',
+        event: {
+          usage:
+            '{"inputTokens":"1234","outputTokens":"5678","totalTokens":"6912","cachedInputTokens":"100"}',
+        },
+      };
+
+      const result = cleanLogEvent(logEvent);
+
+      expect(result.event.usage).toContain('1234');
+      expect(result.event.usage).toContain('5678');
+      expect(result.event.usage).toContain('6912');
+      expect(result.event.usage).toContain('100');
+      expect(result.event.usage).not.toContain('[REDACTED]');
+    });
+
+    it('should redact token usage statistics when redaction is enabled', () => {
+      process.env['OPS_REDACT_LOGS'] = 'true';
+      const logEvent = {
+        message: 'Total token usage for stream',
+        event: {
+          usage:
+            '{"inputTokens":"1234","outputTokens":"5678","totalTokens":"6912","cachedInputTokens":"100"}',
+        },
+      };
+
+      const result = cleanLogEvent(logEvent);
+
+      expect(result.event.usage).toContain('[REDACTED]');
+      expect(result.event.usage).not.toContain('1234');
+    });
   });
 
   describe('error objects', () => {