handle markdown read

Author: rcholic

README.md

@@ -97,13 +97,36 @@ npm run build
   - `.toHaveText(text)`
   - `.toHaveCount(n)`
 
+### Content Reading
+- `read(browser, options)` - Read page content
+  - **Default format: `"raw"`** - Returns HTML suitable for Turndown
+  - `format: "raw"` - Get cleaned HTML
+  - `format: "markdown"` - Get high-quality markdown (uses Turndown internally)
+  - `format: "text"` - Get plain text
+  
+  **Examples:**
+  ```typescript
+  import { read } from './src';
+  
+  // Get raw HTML (default)
+  const result = await read(browser);
+  const html = result.content;
+  
+  // Get high-quality markdown (uses Turndown automatically)
+  const result = await read(browser, { format: 'markdown' });
+  const markdown = result.content;
+  ```
+  
+  See `examples/read-markdown.ts` for complete examples.
+
 ## Examples
 
 See `examples/` directory:
 - `hello.ts` - Extension bridge verification
 - `basic-agent.ts` - Basic snapshot
 - `query-demo.ts` - Query engine
 - `wait-and-click.ts` - Wait and actions
+- `read-markdown.ts` - Reading page content and converting to markdown
 
 ## Testing
 

examples/read-markdown.ts

@@ -0,0 +1,67 @@
+/**
+ * Example: Reading page content and converting to markdown
+ *
+ * This example shows how to use the read() function to get page content
+ * and convert it to high-quality markdown using Turndown.
+ */
+
+import { SentienceBrowser, read } from '../src';
+import TurndownService from 'turndown';
+
+async function main() {
+  // Initialize browser
+  const browser = new SentienceBrowser();
+  await browser.start();
+
+  try {
+    // Navigate to a page
+    await browser.getPage().goto('https://example.com');
+    await browser.getPage().waitForLoadState('networkidle');
+
+    // Method 1: Get raw HTML (default) and convert with Turndown
+    console.log('=== Method 1: Raw HTML + Turndown (Recommended) ===');
+    const result = await read(browser); // format="raw" is default
+    const htmlContent = result.content;
+
+    // Convert to markdown using Turndown (better quality)
+    const turndownService = new TurndownService({
+      headingStyle: 'atx', // Use # for headings
+      bulletListMarker: '-', // Use - for lists
+      codeBlockStyle: 'fenced', // Use ``` for code blocks
+    });
+
+    // Add custom rules for better conversion
+    turndownService.addRule('strikethrough', {
+      filter: ['del', 's', 'strike'] as any,
+      replacement: (content: string) => `~~${content}~~`,
+    });
+
+    // Strip unwanted tags
+    turndownService.remove(['script', 'style', 'nav', 'footer', 'header', 'noscript']);
+
+    const markdown = turndownService.turndown(htmlContent);
+    console.log(`Markdown length: ${markdown.length} characters`);
+    console.log(markdown.substring(0, 500)); // Print first 500 chars
+    console.log('\n');
+
+    // Method 2: Get high-quality markdown directly (uses Turndown internally)
+    console.log('=== Method 2: Direct markdown (High-quality via Turndown) ===');
+    const result2 = await read(browser, { format: 'markdown' });
+    const highQualityMarkdown = result2.content;
+    console.log(`Markdown length: ${highQualityMarkdown.length} characters`);
+    console.log(highQualityMarkdown.substring(0, 500)); // Print first 500 chars
+    console.log('\n');
+
+    // Method 3: Get plain text
+    console.log('=== Method 3: Plain text ===');
+    const result3 = await read(browser, { format: 'text' });
+    const textContent = result3.content;
+    console.log(`Text length: ${textContent.length} characters`);
+    console.log(textContent.substring(0, 500)); // Print first 500 chars
+  } finally {
+    await browser.close();
+  }
+}
+
+main().catch(console.error);
+

src/index.ts

@@ -11,5 +11,6 @@ export { expect, Expectation } from './expect';
 export { Inspector, inspect } from './inspector';
 export { Recorder, Trace, TraceStep, record } from './recorder';
 export { ScriptGenerator, generate } from './generator';
+export { read, ReadOptions, ReadResult } from './read';
 export * from './types';
 

src/read.ts

@@ -0,0 +1,114 @@
+/**
+ * Read page content - supports raw HTML, text, and markdown formats
+ */
+
+import { SentienceBrowser } from './browser';
+import TurndownService from 'turndown';
+
+export interface ReadOptions {
+  format?: 'raw' | 'text' | 'markdown';
+}
+
+export interface ReadResult {
+  status: 'success' | 'error';
+  url: string;
+  format: 'raw' | 'text' | 'markdown';
+  content: string;
+  length: number;
+  error?: string;
+}
+
+/**
+ * Read page content as raw HTML, text, or markdown
+ *
+ * @param browser - SentienceBrowser instance
+ * @param options - Read options
+ * @returns ReadResult with page content
+ *
+ * @example
+ * // Get raw HTML (default)
+ * const result = await read(browser);
+ * const htmlContent = result.content;
+ *
+ * @example
+ * // Get high-quality markdown (uses Turndown internally)
+ * const result = await read(browser, { format: 'markdown' });
+ * const markdown = result.content;
+ *
+ * @example
+ * // Get plain text
+ * const result = await read(browser, { format: 'text' });
+ * const text = result.content;
+ */
+export async function read(
+  browser: SentienceBrowser,
+  options: ReadOptions = {}
+): Promise<ReadResult> {
+  const page = browser.getPage();
+  const format = options.format || 'raw'; // Default to 'raw' for Turndown compatibility
+
+  // For markdown format, get raw HTML first, then convert with Turndown
+  if (format === 'markdown') {
+    // Get raw HTML from extension
+    const rawResult = (await page.evaluate(
+      (opts) => {
+        return (window as any).sentience.read(opts);
+      },
+      { format: 'raw' }
+    )) as ReadResult;
+
+    if (rawResult.status !== 'success') {
+      return rawResult;
+    }
+
+    // Convert to markdown using Turndown
+    try {
+      const turndownService = new TurndownService({
+        headingStyle: 'atx', // Use # for headings
+        bulletListMarker: '-', // Use - for lists
+        codeBlockStyle: 'fenced', // Use ``` for code blocks
+      });
+
+      // Add custom rules for better conversion
+      turndownService.addRule('strikethrough', {
+        filter: ['del', 's', 'strike'] as any,
+        replacement: (content: string) => `~~${content}~~`,
+      });
+
+      // Strip unwanted tags
+      turndownService.remove(['script', 'style', 'nav', 'footer', 'header', 'noscript']);
+
+      const htmlContent = rawResult.content;
+      const markdownContent = turndownService.turndown(htmlContent);
+
+      // Return result with markdown content
+      return {
+        status: 'success',
+        url: rawResult.url,
+        format: 'markdown',
+        content: markdownContent,
+        length: markdownContent.length,
+      };
+    } catch (e) {
+      // If conversion fails, return error
+      return {
+        status: 'error',
+        url: rawResult.url,
+        format: 'markdown',
+        content: '',
+        length: 0,
+        error: `Markdown conversion failed: ${e}`,
+      };
+    }
+  } else {
+    // For "raw" or "text", call extension directly
+    const result = (await page.evaluate(
+      (opts) => {
+        return (window as any).sentience.read(opts);
+      },
+      { format }
+    )) as ReadResult;
+
+    return result;
+  }
+}