Expand custom formats guide

Author: code-with-auto

docs/guide/custom-formats.md

@@ -1,8 +1,12 @@
 # Custom Formats
 
-Add custom log format parsers via config files.
+loq supports adding custom log format parsers in two ways:
+1. **Config files** - Quick setup for personal/project use
+2. **Code contribution** - Add built-in parsers for everyone
 
-## Config File Locations
+## Config File Setup
+
+### Config Locations
 
 loq looks for config in these locations (in order):
 
@@ -14,7 +18,9 @@ loq looks for config in these locations (in order):
 6. `~/.loqrc.json`
 7. `~/.config/loq/config.json`
 
-## TypeScript Config
+### TypeScript Config
+
+Best for complex parsing logic:
 
 ```typescript
 // loq.config.ts
@@ -60,7 +66,9 @@ export default {
 };
 ```
 
-## JSON Config
+### JSON Config
+
+Simpler option using regex patterns:
 
 ```json
 {
@@ -80,3 +88,286 @@ export default {
   ]
 }
 ```
+
+## Format Definition
+
+### Required Fields
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `name` | string | Unique identifier for the format |
+| `detect` | RegExp \| string \| function | How to identify this log format |
+| `parse` | object \| function | How to parse log lines |
+
+### Detection Methods
+
+**Regex pattern:**
+```typescript
+detect: /^\[\d{4}-\d{2}-\d{2}/
+```
+
+**String (converted to regex):**
+```json
+"detect": "^\\[\\d{4}"
+```
+
+**Function (most flexible):**
+```typescript
+detect: (line) => {
+  try {
+    const obj = JSON.parse(line);
+    return 'my_special_field' in obj;
+  } catch {
+    return false;
+  }
+}
+```
+
+### Parse Methods
+
+**Pattern-based:**
+```typescript
+parse: {
+  pattern: /^\[(?<timestamp>[^\]]+)\] (?<level>\w+): (?<message>.+)$/,
+  fields: {
+    timestamp: 'timestamp',  // named group
+    level: 'level',
+    message: 'message',
+  },
+}
+```
+
+**Function-based:**
+```typescript
+parse: (line) => {
+  const parts = line.split(' | ');
+  return {
+    timestamp: parts[0],
+    level: parts[1],
+    message: parts[2],
+    fields: {
+      custom_field: parts[3],
+    },
+  };
+}
+```
+
+### LogEntry Structure
+
+Your parser should return this structure:
+
+```typescript
+interface LogEntry {
+  raw: string;        // Original line (added automatically)
+  timestamp?: string; // ISO timestamp or parseable date
+  level?: string;     // error, warn, info, debug, etc.
+  message?: string;   // Main log message
+  fields?: Record<string, unknown>; // Additional fields
+}
+```
+
+## Using Custom Formats
+
+Once configured, loq auto-detects your format:
+
+```bash
+# Auto-detect
+loq app.log
+
+# Force specific format
+loq app.log --format my-app
+
+# Query custom fields
+loq app.log where custom_field=value
+```
+
+## Command Aliases
+
+Define shortcuts for common queries:
+
+```typescript
+aliases: {
+  errors: 'where level=error',
+  slow: 'where response_time>1000',
+  today: 'where timestamp after today',
+  '5xx': 'where status>=500 and status<600',
+}
+```
+
+Usage:
+```bash
+loq app.log errors           # expands to: where level=error
+loq access.log slow          # expands to: where response_time>1000
+loq app.log errors limit 10  # can combine with other options
+```
+
+## Examples
+
+### Rails Log Format
+
+```typescript
+{
+  name: 'rails',
+  detect: /^[DIWEF],\s*\[/,
+  parse: {
+    pattern: /^(?<level>[DIWEF]),\s*\[(?<timestamp>[^\]]+)\]\s*(?<pid>\d+)\s*(?<source>\w+)\s*--\s*:\s*(?<message>.*)$/,
+    fields: {
+      level: 'level',
+      timestamp: 'timestamp',
+      message: 'message',
+      pid: 'pid',
+      source: 'source',
+    },
+  },
+}
+```
+
+### Docker JSON Logs
+
+```typescript
+{
+  name: 'docker-json',
+  detect: (line) => {
+    try {
+      const obj = JSON.parse(line);
+      return 'log' in obj && 'stream' in obj;
+    } catch {
+      return false;
+    }
+  },
+  parse: (line) => {
+    const obj = JSON.parse(line);
+    return {
+      timestamp: obj.time,
+      level: obj.stream === 'stderr' ? 'error' : 'info',
+      message: obj.log.trim(),
+      fields: obj,
+    };
+  },
+}
+```
+
+### Custom App with Metadata
+
+```typescript
+{
+  name: 'my-service',
+  detect: /^\d{4}-\d{2}-\d{2}T.*\|/,
+  parse: (line) => {
+    const [timestamp, level, service, traceId, ...rest] = line.split(' | ');
+    return {
+      timestamp,
+      level: level.toLowerCase(),
+      message: rest.join(' | '),
+      fields: {
+        service,
+        traceId,
+      },
+    };
+  },
+}
+```
+
+## Contributing Built-in Parsers
+
+Want to add a parser for everyone? Follow these steps:
+
+### 1. Create Parser File
+
+```typescript
+// src/parser/formats/myformat.ts
+import type { LogEntry, LogParser } from '../types';
+
+export const myFormatParser: LogParser = {
+  name: 'myformat',
+
+  detect(line: string): boolean {
+    return line.startsWith('MYFORMAT:');
+  },
+
+  parse(line: string): LogEntry | null {
+    const match = line.match(/^MYFORMAT: \[(.+?)\] (\w+) - (.+)$/);
+    if (!match) return null;
+
+    return {
+      raw: line,
+      timestamp: match[1],
+      level: match[2],
+      message: match[3],
+      fields: {},
+    };
+  },
+};
+```
+
+### 2. Register Parser
+
+```typescript
+// src/parser/auto-detect.ts
+import { myFormatParser } from './formats/myformat';
+
+const builtinParsers: LogParser[] = [
+  jsonParser,
+  apacheParser,
+  syslogParser,
+  clfParser,
+  myFormatParser,  // Add here
+];
+```
+
+### 3. Add Tests
+
+```typescript
+// tests/parser/myformat.test.ts
+import { describe, expect, test } from 'bun:test';
+import { myFormatParser } from '../../src/parser/formats/myformat';
+
+describe('myformat parser', () => {
+  test('detects myformat lines', () => {
+    expect(myFormatParser.detect('MYFORMAT: [2024-01-01] INFO - Hello')).toBe(true);
+    expect(myFormatParser.detect('Some other log')).toBe(false);
+  });
+
+  test('parses myformat correctly', () => {
+    const result = myFormatParser.parse('MYFORMAT: [2024-01-01] INFO - Hello world');
+    expect(result).toEqual({
+      raw: 'MYFORMAT: [2024-01-01] INFO - Hello world',
+      timestamp: '2024-01-01',
+      level: 'INFO',
+      message: 'Hello world',
+      fields: {},
+    });
+  });
+
+  test('returns null for invalid lines', () => {
+    expect(myFormatParser.parse('invalid line')).toBeNull();
+  });
+});
+```
+
+### 4. Submit PR
+
+1. Fork the repo
+2. Create a branch: `git checkout -b add-myformat-parser`
+3. Make your changes
+4. Run tests: `bun test`
+5. Push and open a PR
+
+## Debugging Custom Formats
+
+Test your format detection:
+
+```bash
+# See which format is detected
+loq app.log --format auto --verbose
+
+# Force your format to test parsing
+loq app.log --format my-custom-format
+```
+
+Check if fields are parsed correctly:
+
+```bash
+# Output as JSON to see all fields
+loq app.log -o json limit 1
+```