diff --git a/.changeset/annotations-debug-mode.md b/.changeset/annotations-debug-mode.md new file mode 100644 index 0000000..f794de5 --- /dev/null +++ b/.changeset/annotations-debug-mode.md @@ -0,0 +1,18 @@ +--- +"@lytics/playwright-annotations": minor +--- + +Add debug mode for troubleshooting annotation issues using @lytics/kero logger + +Set `PLAYWRIGHT_ANNOTATIONS_DEBUG=true` to enable verbose logging that shows: +- When each annotation helper is called +- What annotations exist at each step +- Validation results and errors + +Example usage: +```bash +PLAYWRIGHT_ANNOTATIONS_DEBUG=true npx playwright test +``` + +This release adds `@lytics/kero` as a dependency for structured logging with pretty output in development and JSON in production. + diff --git a/.gitignore b/.gitignore index c966b03..50f8d69 100644 --- a/.gitignore +++ b/.gitignore @@ -51,4 +51,4 @@ pnpm-debug.log* # OS .DS_Store -Thumbs.db \ No newline at end of file +Thumbs.db.dev-agent/ diff --git a/packages/annotations/package.json b/packages/annotations/package.json index 6041dda..183af5d 100644 --- a/packages/annotations/package.json +++ b/packages/annotations/package.json @@ -51,6 +51,10 @@ }, "devDependencies": { "@playwright/test": "^1.40.0", + "@types/node": "^22.0.0", "typescript": "^5.3.3" + }, + "dependencies": { + "@lytics/kero": "^1.0.0" } } diff --git a/packages/annotations/src/helpers.ts b/packages/annotations/src/helpers.ts index 65d8ed4..6768b65 100644 --- a/packages/annotations/src/helpers.ts +++ b/packages/annotations/src/helpers.ts @@ -1,13 +1,30 @@ +import { createLogger } from '@lytics/kero'; import type { TestInfo } from '@playwright/test'; -import type { TestAnnotations, ContentstackAnnotations } from './types'; +import type { ContentstackAnnotations, TestAnnotations } from './types'; import { validateContentstackAnnotations } from './validators'; +/** + * Logger for playwright-annotations + * + * Enable debug logging with: + * ```bash + * PLAYWRIGHT_ANNOTATIONS_DEBUG=true npx playwright test + * ``` + * + * Or set LOG_LEVEL=debug for more control. + */ +const logger = createLogger({ + preset: process.env.NODE_ENV === 'production' ? 'production' : 'development', + level: process.env.PLAYWRIGHT_ANNOTATIONS_DEBUG ? 'debug' : 'warn', + context: { pkg: 'playwright-annotations' }, +}); + /** * Generic helper: Push any annotations to a test - * + * * This is the base helper that works with any annotation schema. * Teams can use this directly or create schema-specific wrappers. - * + * * @example * ```typescript * // With custom schema @@ -15,7 +32,7 @@ import { validateContentstackAnnotations } from './validators'; * featureArea: string; * ticketId: string; * } - * + * * pushAnnotations(testInfo, { * featureArea: "auth", * ticketId: "JIRA-123" @@ -26,37 +43,45 @@ export function pushAnnotations( testInfo: TestInfo, annotations: T ): void { + logger.debug( + { testTitle: testInfo.title, annotations, existingAnnotations: testInfo.annotations }, + 'pushAnnotations called' + ); + for (const [key, value] of Object.entries(annotations)) { if (value !== undefined) { const description = Array.isArray(value) ? value.join(', ') : value; + logger.debug({ key, description }, 'Adding annotation'); testInfo.annotations.push({ type: key, description, }); } } + + logger.debug({ finalAnnotations: testInfo.annotations }, 'pushAnnotations complete'); } /** * Extract generic annotations from a test case - * + * * @example * ```typescript * const annotations = getAnnotations(test); * // Returns: { [key: string]: string } * ``` */ -export function getAnnotations( - test: { annotations: Array<{ type: string; description?: string }> } -): Partial { +export function getAnnotations(test: { + annotations: Array<{ type: string; description?: string }>; +}): Partial { const result: Record = {}; - + for (const annotation of test.annotations) { if (annotation.description) { result[annotation.type] = annotation.description; } } - + return result as Partial; } @@ -69,9 +94,9 @@ export function getAnnotations( /** * Contentstack helper: Push suite-level annotation - * + * * Use in `beforeEach` at describe level to set the suite for all tests. - * + * * @example * ```typescript * test.describe("My Feature", () => { @@ -81,22 +106,26 @@ export function getAnnotations( * }); * ``` */ -export function pushSuiteAnnotation( - testInfo: TestInfo, - testSuiteName: string -): void { +export function pushSuiteAnnotation(testInfo: TestInfo, testSuiteName: string): void { + logger.debug( + { testTitle: testInfo.title, testSuiteName, existingAnnotations: testInfo.annotations }, + 'pushSuiteAnnotation called' + ); + testInfo.annotations.push({ type: 'testSuiteName', description: testSuiteName, }); + + logger.debug({ finalAnnotations: testInfo.annotations }, 'pushSuiteAnnotation complete'); } /** * Contentstack helper: Push test-level annotations with validation - * + * * Use in individual `test()` blocks. Validates annotations before pushing. * Requires `testSuiteName` to be set first via `pushSuiteAnnotation`. - * + * * @example * ```typescript * test("my test", async ({}, testInfo) => { @@ -106,7 +135,7 @@ export function pushSuiteAnnotation( * }); * }); * ``` - * + * * @throws {Error} If testSuiteName not set or validation fails */ export function pushTestAnnotations( @@ -116,49 +145,69 @@ export function pushTestAnnotations( testCaseId: string; } ): void { + logger.debug( + { testTitle: testInfo.title, annotations, existingAnnotations: testInfo.annotations }, + 'pushTestAnnotations called' + ); + // Get suite name from existing annotations - const testSuiteName = testInfo.annotations.find( - a => a.type === 'testSuiteName' - )?.description; + const testSuiteName = testInfo.annotations.find((a) => a.type === 'testSuiteName')?.description; + + logger.debug( + { + found: !!testSuiteName, + testSuiteName, + allAnnotationTypes: testInfo.annotations.map((a) => a.type), + }, + 'Looking for testSuiteName in existing annotations' + ); if (!testSuiteName) { - throw new Error( + const errorMsg = 'testSuiteName must be set before calling pushTestAnnotations. ' + - 'Use pushSuiteAnnotation in beforeEach first.' - ); + 'Use pushSuiteAnnotation in beforeEach first.'; + logger.error({ existingAnnotations: testInfo.annotations }, 'testSuiteName not found'); + throw new Error(errorMsg); } // Validate complete annotation set - const validation = validateContentstackAnnotations({ + const fullAnnotations = { testSuiteName, journeyId: annotations.journeyId, testCaseId: annotations.testCaseId, - }); + }; + + logger.debug(fullAnnotations, 'Validating annotations'); + + const validation = validateContentstackAnnotations(fullAnnotations); + + logger.debug({ ...validation }, 'Validation result'); if (!validation.valid) { - throw new Error( - `Invalid annotations:\n${validation.errors.join('\n')}` - ); + const errorMsg = `Invalid annotations:\n${validation.errors.join('\n')}`; + logger.error({ errors: validation.errors, annotations: fullAnnotations }, 'Validation failed'); + throw new Error(errorMsg); } if (validation.warnings.length > 0) { - console.warn( - `⚠️ Annotation warnings:\n${validation.warnings.join('\n')}` - ); + logger.warn({ warnings: validation.warnings }, 'Annotation warnings'); } // Push annotations + logger.debug('Pushing journeyId and testCaseId annotations'); testInfo.annotations.push( { type: 'journeyId', description: annotations.journeyId }, { type: 'testCaseId', description: annotations.testCaseId } ); + + logger.debug({ finalAnnotations: testInfo.annotations }, 'pushTestAnnotations complete'); } /** * Contentstack helper: Push all annotations at once - * + * * Alternative to separate suite/test helpers. Validates before pushing. - * + * * @example * ```typescript * test("my test", async ({}, testInfo) => { @@ -169,37 +218,45 @@ export function pushTestAnnotations( * }); * }); * ``` - * + * * @throws {Error} If validation fails */ export function pushContentstackAnnotations( testInfo: TestInfo, annotations: ContentstackAnnotations ): void { + logger.debug( + { testTitle: testInfo.title, annotations, existingAnnotations: testInfo.annotations }, + 'pushContentstackAnnotations called' + ); + + logger.debug({ ...annotations }, 'Validating annotations'); const validation = validateContentstackAnnotations(annotations); + logger.debug({ ...validation }, 'Validation result'); if (!validation.valid) { - throw new Error( - `Invalid annotations:\n${validation.errors.join('\n')}` - ); + const errorMsg = `Invalid annotations:\n${validation.errors.join('\n')}`; + logger.error({ errors: validation.errors, annotations }, 'Validation failed'); + throw new Error(errorMsg); } if (validation.warnings.length > 0) { - console.warn( - `⚠️ Annotation warnings:\n${validation.warnings.join('\n')}` - ); + logger.warn({ warnings: validation.warnings }, 'Annotation warnings'); } + logger.debug('Pushing all annotations'); testInfo.annotations.push( { type: 'testSuiteName', description: annotations.testSuiteName }, { type: 'journeyId', description: annotations.journeyId }, { type: 'testCaseId', description: annotations.testCaseId } ); + + logger.debug({ finalAnnotations: testInfo.annotations }, 'pushContentstackAnnotations complete'); } /** * Contentstack helper: Extract Contentstack annotations from a test case - * + * * @example * ```typescript * const annotations = getContentstackAnnotations(test); @@ -207,15 +264,15 @@ export function pushContentstackAnnotations( * console.log(annotations.journeyId); * } * ``` - * + * * @returns Contentstack annotations or null if incomplete */ -export function getContentstackAnnotations( - test: { annotations: Array<{ type: string; description?: string }> } -): ContentstackAnnotations | null { - const testSuiteName = test.annotations.find(a => a.type === 'testSuiteName')?.description; - const journeyId = test.annotations.find(a => a.type === 'journeyId')?.description; - const testCaseId = test.annotations.find(a => a.type === 'testCaseId')?.description; +export function getContentstackAnnotations(test: { + annotations: Array<{ type: string; description?: string }>; +}): ContentstackAnnotations | null { + const testSuiteName = test.annotations.find((a) => a.type === 'testSuiteName')?.description; + const journeyId = test.annotations.find((a) => a.type === 'journeyId')?.description; + const testCaseId = test.annotations.find((a) => a.type === 'testCaseId')?.description; if (!testSuiteName || !journeyId || !testCaseId) { return null; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bd2e622..cc831ec 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -65,10 +65,17 @@ importers: version: 5.9.3 packages/annotations: + dependencies: + '@lytics/kero': + specifier: ^1.0.0 + version: 1.0.0 devDependencies: '@playwright/test': specifier: ^1.40.0 version: 1.56.1 + '@types/node': + specifier: ^22.0.0 + version: 22.19.1 typescript: specifier: ^5.3.3 version: 5.9.3 @@ -891,6 +898,11 @@ packages: resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==} dev: false + /@lytics/kero@1.0.0: + resolution: {integrity: sha512-3gwBAjsMhcNlvTNNc0yOZ+nuQ1J6N17op9c0jNxvChia2L7XJI0e2Oeb+KBV5euJiEj0CfiFo6iQdjC7bg4wQg==} + engines: {node: '>=22'} + dev: false + /@lytics/playwright-slack@0.3.1: resolution: {integrity: sha512-NEuEc4zP5EfrvnmdcTcWoMPi3D/O8lRrHd+papjx7ymcHDGtEfT857jivP1z3jj2ZJGRTnpirhkqgg1OQqLrzg==} engines: {node: '>=22'} diff --git a/website/content/docs/adapters/firestore.mdx b/website/content/docs/adapters/firestore.mdx index 36cdb23..bcb359a 100644 --- a/website/content/docs/adapters/firestore.mdx +++ b/website/content/docs/adapters/firestore.mdx @@ -303,6 +303,142 @@ When `TRIGGER_TYPE` environment variable is `'pull_request'`, no data is written Write errors are logged but don't throw, allowing other adapters to continue. +## Local Development + +By default, you may want to **disable Firestore writes for local runs** to avoid polluting production data. Here are several patterns for handling this: + +### Pattern 1: Environment-Based Toggle (Recommended) + +Create a custom reporter that conditionally enables Firestore: + +```typescript +// reporter.ts +import { CoreReporter } from '@lytics/playwright-reporter'; +import { FirestoreAdapter } from '@lytics/playwright-adapters/firestore'; +import { FilesystemAdapter } from '@lytics/playwright-adapters/filesystem'; +import type { ResultAdapter } from '@lytics/playwright-reporter'; +import { readFileSync } from 'fs'; + +const { GCP_PROJECT_ID, GOOGLE_APPLICATION_CREDENTIALS, TEST_ENV } = process.env; +const isProduction = TEST_ENV === 'production'; + +// Build adapters list +const adapters: ResultAdapter[] = [ + // Always write to filesystem for local debugging + new FilesystemAdapter({ outputDir: './test-results' }), +]; + +// Only add Firestore in production with valid credentials +if (isProduction && GCP_PROJECT_ID && GOOGLE_APPLICATION_CREDENTIALS) { + try { + // Handle both file path and JSON string + const credentials = GOOGLE_APPLICATION_CREDENTIALS.startsWith('{') + ? GOOGLE_APPLICATION_CREDENTIALS + : readFileSync(GOOGLE_APPLICATION_CREDENTIALS, 'utf8'); + + adapters.push(new FirestoreAdapter({ + projectId: GCP_PROJECT_ID, + credentials, + collections: { + testRuns: 'test_runs', + testCases: 'test_cases', + latestTestCases: 'latest_test_cases', + }, + skipConditions: { + skipPullRequests: true, + }, + })); + console.log('✅ Firestore adapter configured'); + } catch (error) { + console.warn('⚠️ Failed to configure Firestore:', error); + } +} else { + console.log('ℹ️ Firestore disabled (not production or missing credentials)'); +} + +export default class CustomReporter extends CoreReporter { + constructor() { + super({ adapters }); + } +} +``` + +Then in `playwright.config.ts`: + +```typescript +export default defineConfig({ + reporter: [ + ['list'], + ['./reporter.ts'], + ], +}); +``` + +**Local run:** `npx playwright test` → writes to filesystem only + +**Production run:** `TEST_ENV=production npx playwright test` → writes to both + +### Pattern 2: Separate Dev Collections + +Write local runs to separate collections so they don't affect production data: + +```typescript +const env = process.env.TEST_ENV || 'dev'; + +new FirestoreAdapter({ + projectId: 'my-gcp-project', + credentials: process.env.GOOGLE_APPLICATION_CREDENTIALS_JSON, + collections: { + testRuns: `${env}_test_runs`, // dev_test_runs or production_test_runs + testCases: `${env}_test_cases`, + latestTestCases: `${env}_latest_test_cases`, + }, +}) +``` + +**Local run:** `npx playwright test` → writes to `dev_*` collections + +**Production run:** `TEST_ENV=production npx playwright test` → writes to `production_*` collections + +### Pattern 3: Firestore Emulator + +For testing Firestore integration without a real GCP project: + +```bash +# Install and start the emulator +gcloud components install cloud-firestore-emulator +gcloud emulators firestore start --host-port=localhost:8080 +``` + +```bash +# In another terminal, set the emulator host +export FIRESTORE_EMULATOR_HOST="localhost:8080" +npx playwright test +``` + +The `@google-cloud/firestore` client automatically uses the emulator when `FIRESTORE_EMULATOR_HOST` is set. + +### Pattern 4: Explicit Enable Flag + +Add an explicit flag to enable Firestore: + +```typescript +const shouldWriteToFirestore = + process.env.FIRESTORE_ENABLED === 'true' && + process.env.GCP_PROJECT_ID && + process.env.GOOGLE_APPLICATION_CREDENTIALS; + +if (shouldWriteToFirestore) { + adapters.push(new FirestoreAdapter({ /* ... */ })); +} +``` + +**Local run with Firestore:** `FIRESTORE_ENABLED=true npx playwright test` + + + **Recommendation:** Use Pattern 1 (environment-based toggle) for most teams. It's explicit, safe by default, and matches how CI/CD pipelines typically work. + + ## CI/CD Integration ### GitHub Actions diff --git a/website/content/docs/annotations/helpers.mdx b/website/content/docs/annotations/helpers.mdx index 9463d07..527ef01 100644 --- a/website/content/docs/annotations/helpers.mdx +++ b/website/content/docs/annotations/helpers.mdx @@ -283,3 +283,29 @@ test('my test', async ({}, testInfo) => { }); ``` +## Debug Mode + +Enable verbose logging to troubleshoot annotation issues: + +```bash +PLAYWRIGHT_ANNOTATIONS_DEBUG=true npx playwright test +``` + +When enabled, all helper functions log detailed information: + +``` +🔍 [playwright-annotations] pushSuiteAnnotation called { + "testTitle": "user can view access tokens", + "testSuiteName": "ACCOUNT_SECURITY", + "existingAnnotations": [] +} +🔍 [playwright-annotations] pushSuiteAnnotation complete { + "finalAnnotations": [{ "type": "testSuiteName", "description": "ACCOUNT_SECURITY" }] +} +``` + +This helps diagnose: +- Whether `beforeEach` is running before your test +- What annotations exist at each step +- Validation errors and their causes + diff --git a/website/content/docs/annotations/index.mdx b/website/content/docs/annotations/index.mdx index ba84603..8e29698 100644 --- a/website/content/docs/annotations/index.mdx +++ b/website/content/docs/annotations/index.mdx @@ -152,6 +152,44 @@ export default new CoreReporter({ The reporter extracts annotations from tests and includes them in the output, enabling filtering, grouping, and analysis. +## Troubleshooting + +### Debug Mode + +If annotations aren't working as expected, enable debug mode to see detailed logging: + +```bash +PLAYWRIGHT_ANNOTATIONS_DEBUG=true npx playwright test +``` + +This outputs verbose logs showing: +- When each annotation helper is called +- What annotations exist at each step +- Validation results and any errors + +Example output: + +``` +🔍 [playwright-annotations] pushSuiteAnnotation called { + "testTitle": "user can view access tokens", + "testSuiteName": "ACCOUNT_SECURITY", + "existingAnnotations": [] +} +🔍 [playwright-annotations] pushTestAnnotations called { + "testTitle": "user can view access tokens", + "annotations": { "journeyId": "...", "testCaseId": "..." }, + "existingAnnotations": [{ "type": "testSuiteName", "description": "ACCOUNT_SECURITY" }] +} +``` + +### Common Issues + +| Issue | Cause | Solution | +|-------|-------|----------| +| `testSuiteName must be set` | `beforeEach` not running or missing `testInfo` | Ensure `pushSuiteAnnotation` is in `beforeEach` with `testInfo` parameter | +| Validation errors | Naming hierarchy not followed | Check that `testCaseId` starts with `journeyId` | +| Annotations not appearing | Wrong `testInfo` scope | Use `beforeEach` (not `beforeAll`) for suite annotations | + ## Next Steps - [Schema & Types](/docs/annotations/schema) — Type definitions and interfaces