resolve merge conflicts

Author: rcholic

.github/workflows/sync-extension.yml

@@ -25,6 +25,7 @@ jobs:
       uses: actions/checkout@v4
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
+        fetch-depth: 0  # Fetch all history for proper branching
 
     - name: Set up Node.js
       uses: actions/setup-node@v4
@@ -62,7 +63,7 @@ jobs:
         mkdir -p extension-temp
         cd extension-temp
         
-        # Download individual files from release (reliable method)
+        # Download individual files from release (reliable method - no zip)
         echo "📁 Downloading individual files from release..."
         curl -L -H "Authorization: token ${{ secrets.SENTIENCE_CHROME_TOKEN }}" \
           "https://api.github.com/repos/$REPO/releases/tags/$TAG" | \
@@ -148,6 +149,11 @@ jobs:
           cp extension-temp/extension-package/pkg/*.d.ts src/extension/pkg/ 2>/dev/null || echo "⚠️ Type definitions not found"
         fi
         
+        # Verify copied files
+        echo "📋 Copied files:"
+        ls -la src/extension/
+        ls -la src/extension/pkg/ 2>/dev/null || echo "⚠️ pkg directory not created"
+        
     - name: Check for changes
       if: steps.release.outputs.skip != 'true'
       id: changes
@@ -167,7 +173,9 @@ jobs:
       if: steps.release.outputs.skip != 'true' && steps.changes.outputs.changed == 'true'
       uses: peter-evans/create-pull-request@v5
       with:
-        token: ${{ secrets.GITHUB_TOKEN }}
+        # Use PR_TOKEN if available (for repos with org restrictions), otherwise use GITHUB_TOKEN
+        # To use PAT: create secret named PR_TOKEN with a Personal Access Token that has 'repo' scope
+        token: ${{ secrets.PR_TOKEN }}
         commit-message: "chore: sync extension files from sentience-chrome ${{ steps.release.outputs.tag }}"
         title: "Sync Extension: ${{ steps.release.outputs.tag }}"
         body: |
@@ -177,7 +185,10 @@ jobs:
           - Extension manifest and scripts
           - WASM binary and bindings
           
-          **Source:** [sentience-chrome release ${{ steps.release.outputs.tag }}](${{ secrets.SENTIENCE_CHROME_REPO }}/releases/tag/${{ steps.release.outputs.tag }})
+          **Source:** [sentience-chrome release ${{ steps.release.outputs.tag }}](https://github.com/${{ secrets.SENTIENCE_CHROME_REPO }}/releases/tag/${{ steps.release.outputs.tag }})
         branch: sync-extension-${{ steps.release.outputs.tag }}
         delete-branch: true
+        labels: |
+          automated
+          extension-sync
 

README.md

@@ -78,6 +78,14 @@ npm run build
 - `snapshot(browser, options)` - Capture page state
 - TypeScript types for type safety
 
+### Content Reading & Screenshots
+- `read(browser, options)` - Read page content as text or markdown
+  - Enhanced markdown conversion using `turndown` (better than extension's lightweight conversion)
+  - Supports `enhance_markdown` option to use improved conversion
+- `screenshot(browser, options)` - Capture standalone screenshot
+  - Returns base64-encoded data URL
+  - Supports PNG and JPEG formats with quality control
+
 ### Day 4: Query Engine
 - `query(snapshot, selector)` - Find elements matching selector
 - `find(snapshot, selector)` - Find single best match
@@ -128,6 +136,50 @@ See `examples/` directory:
 - `wait-and-click.ts` - Wait and actions
 - `read-markdown.ts` - Reading page content and converting to markdown
 
+### Content Reading Example
+
+```typescript
+import { SentienceBrowser, read } from './src';
+
+const browser = new SentienceBrowser();
+await browser.start();
+
+await browser.getPage().goto('https://example.com');
+await browser.getPage().waitForLoadState('networkidle');
+
+// Read as enhanced markdown (better quality)
+const result = await read(browser, { 
+  format: 'markdown', 
+  enhance_markdown: true 
+});
+console.log(result.content);  // High-quality markdown
+
+await browser.close();
+```
+
+### Screenshot Example
+
+```typescript
+import { SentienceBrowser, screenshot } from './src';
+import { writeFileSync } from 'fs';
+
+const browser = new SentienceBrowser();
+await browser.start();
+
+await browser.getPage().goto('https://example.com');
+await browser.getPage().waitForLoadState('networkidle');
+
+// Capture PNG screenshot
+const dataUrl = await screenshot(browser, { format: 'png' });
+
+// Save to file
+const base64Data = dataUrl.split(',')[1];
+const imageData = Buffer.from(base64Data, 'base64');
+writeFileSync('screenshot.png', imageData);
+
+await browser.close();
+```
+
 ## Testing
 
 ```bash

package-lock.json

@@ -18,11 +18,13 @@
   },
   "dependencies": {
     "playwright": "^1.40.0",
+    "turndown": "^7.2.2",
     "zod": "^3.22.0"
   },
   "devDependencies": {
     "@types/jest": "^29.5.14",
     "@types/node": "^20.0.0",
+    "@types/turndown": "^5.0.3",
     "jest": "^29.0.0",
     "ts-jest": "^29.0.0",
     "ts-node": "^10.9.0",

package.json

@@ -12,5 +12,6 @@ export { Inspector, inspect } from './inspector';
 export { Recorder, Trace, TraceStep, record } from './recorder';
 export { ScriptGenerator, generate } from './generator';
 export { read, ReadOptions, ReadResult } from './read';
+export { screenshot, ScreenshotOptions } from './screenshot';
 export * from './types';
 

src/index.ts

@@ -7,6 +7,7 @@ import TurndownService from 'turndown';
 
 export interface ReadOptions {
   format?: 'raw' | 'text' | 'markdown';
+  enhanceMarkdown?: boolean;
 }
 
 export interface ReadResult {
@@ -46,69 +47,64 @@ export async function read(
 ): Promise<ReadResult> {
   const page = browser.getPage();
   const format = options.format || 'raw'; // Default to 'raw' for Turndown compatibility
+  const enhanceMarkdown = options.enhanceMarkdown !== false; // Default to true
 
-  // For markdown format, get raw HTML first, then convert with Turndown
-  if (format === 'markdown') {
-    // Get raw HTML from extension
-    const rawResult = (await page.evaluate(
+  if (format === 'markdown' && enhanceMarkdown) {
+    // Get raw HTML from the extension first
+    const rawHtmlResult = (await page.evaluate(
       (opts) => {
         return (window as any).sentience.read(opts);
       },
       { format: 'raw' }
     )) as ReadResult;
 
-    if (rawResult.status !== 'success') {
-      return rawResult;
-    }
-
-    // Convert to markdown using Turndown
-    try {
-      const turndownService = new TurndownService({
-        headingStyle: 'atx', // Use # for headings
-        bulletListMarker: '-', // Use - for lists
-        codeBlockStyle: 'fenced', // Use ``` for code blocks
-      });
-
-      // Add custom rules for better conversion
-      turndownService.addRule('strikethrough', {
-        filter: ['del', 's', 'strike'] as any,
-        replacement: (content: string) => `~~${content}~~`,
-      });
+    if (rawHtmlResult.status === 'success') {
+      const htmlContent = rawHtmlResult.content;
+      try {
+        const turndownService = new TurndownService({
+          headingStyle: 'atx',
+          hr: '---',
+          bulletListMarker: '-',
+          codeBlockStyle: 'fenced',
+          emDelimiter: '*',
+        });
 
-      // Strip unwanted tags
-      turndownService.remove(['script', 'style', 'nav', 'footer', 'header', 'noscript']);
+        // Add custom rules for better markdown
+        turndownService.addRule('strikethrough', {
+          filter: (node) => ['s', 'del', 'strike'].includes(node.nodeName.toLowerCase()),
+          replacement: function (content) {
+            return '~~' + content + '~~';
+          },
+        });
 
-      const htmlContent = rawResult.content;
-      const markdownContent = turndownService.turndown(htmlContent);
+        // Optionally strip certain tags entirely
+        turndownService.remove(['script', 'style', 'noscript', 'iframe'] as any);
 
-      // Return result with markdown content
-      return {
-        status: 'success',
-        url: rawResult.url,
-        format: 'markdown',
-        content: markdownContent,
-        length: markdownContent.length,
-      };
-    } catch (e) {
-      // If conversion fails, return error
-      return {
-        status: 'error',
-        url: rawResult.url,
-        format: 'markdown',
-        content: '',
-        length: 0,
-        error: `Markdown conversion failed: ${e}`,
-      };
+        const markdownContent = turndownService.turndown(htmlContent);
+        return {
+          status: 'success',
+          url: rawHtmlResult.url,
+          format: 'markdown',
+          content: markdownContent,
+          length: markdownContent.length,
+        };
+      } catch (e: any) {
+        console.warn(`Turndown conversion failed: ${e.message}, falling back to extension's markdown.`);
+        // Fallback to extension's markdown if Turndown fails
+      }
+    } else {
+      console.warn(`Failed to get raw HTML from extension: ${rawHtmlResult.error}, falling back to extension's markdown.`);
+      // Fallback to extension's markdown if getting raw HTML fails
     }
-  } else {
-    // For "raw" or "text", call extension directly
-    const result = (await page.evaluate(
-      (opts) => {
-        return (window as any).sentience.read(opts);
-      },
-      { format }
-    )) as ReadResult;
-
-    return result;
   }
+
+  // If not enhanced markdown, or fallback, call extension with requested format
+  const result = (await page.evaluate(
+    (opts) => {
+      return (window as any).sentience.read(opts);
+    },
+    { format }
+  )) as ReadResult;
+
+  return result;
 }

src/read.ts

@@ -0,0 +1,50 @@
+/**
+ * Screenshot functionality - standalone screenshot capture
+ */
+
+import { SentienceBrowser } from './browser';
+
+export interface ScreenshotOptions {
+  format?: 'png' | 'jpeg';
+  quality?: number; // 1-100, only used for JPEG
+}
+
+/**
+ * Capture screenshot of current page
+ *
+ * @param browser - SentienceBrowser instance
+ * @param options - Screenshot options
+ * @returns Base64-encoded screenshot data URL (e.g., "data:image/png;base64,...")
+ */
+export async function screenshot(
+  browser: SentienceBrowser,
+  options: ScreenshotOptions = {}
+): Promise<string> {
+  const page = browser.getPage();
+  const format = options.format || 'png';
+  const quality = options.quality;
+
+  if (format === 'jpeg' && quality !== undefined) {
+    if (quality < 1 || quality > 100) {
+      throw new Error('Quality must be between 1 and 100 for JPEG format');
+    }
+  }
+
+  // Use Playwright's screenshot with base64 encoding
+  const screenshotOptions: any = {
+    type: format,
+    encoding: 'base64',
+  };
+
+  if (format === 'jpeg' && quality !== undefined) {
+    screenshotOptions.quality = quality;
+  }
+
+  // Capture screenshot
+  const base64Data = await page.screenshot(screenshotOptions);
+
+  // Return as data URL
+  const mimeType = format === 'png' ? 'image/png' : 'image/jpeg';
+  return `data:${mimeType};base64,${base64Data}`;
+}
+