SMOODEV-2620: th crawl crawl/map, scrape extract/screenshot/render, search --scrape, knowledge add-url

.changeset/th-crawl-map-scrape-flags.md

@@ -0,0 +1,11 @@
+---
+"@smooai/smooth": minor
+---
+
+Extend the `th` agentic-tooling commands against the api-prime crawl/search/knowledge routes:
+
+- **`th crawl crawl <seed>`** — new authed whole-site crawl. Supports `--limit`, `--max-depth` (sent as `maxDiscoveryDepth`), `--extract` (JSON spec verbatim or wrapped as `{"prompt": …}`), `--json`, and `--org`. Default output is a compact `completed/total` summary plus one crawled URL per line.
+- **`th crawl map <url>`** — new authed URL-discovery command. Supports `--search`, `--limit`, `--include-subdomains`, `--json`, and `--org`; prints one discovered link per line by default.
+- **`th crawl scrape`** — new `--extract <SPEC>` (JSON verbatim or `{"prompt": …}`), `--screenshot` (appends `screenshot` to the formats), and `--render <MODE>` flags. `--extract`/`--render` are authed-only; the free tier surfaces the backend's rejection.
+- **`th search --scrape`** — forces `searchDepth: "advanced"` so each result is crawl-enriched with full page content (authed tier; clamped on the free tier).
+- **`th api knowledge add-url <url>`** — new authed command that kicks off a crawl→ingest job into the org's knowledge base via `POST /organizations/{org}/knowledge/websites` (`--name` defaults to the URL).

crates/smooth-cli/src/smooai/crawl.rs

@@ -38,25 +38,96 @@ pub enum Cmd {
         /// (`summary` needs a login — it's a billed LLM step.)
         #[arg(long = "format", value_name = "FORMAT")]
         formats: Vec<String>,
+        /// LLM structured extraction (authed tier only). A JSON schema/spec is
+        /// sent verbatim; any other string is wrapped as `{"prompt": <SPEC>}`.
+        #[arg(long, value_name = "SPEC")]
+        extract: Option<String>,
+        /// Also capture a screenshot (adds `screenshot` to the formats).
+        #[arg(long)]
+        screenshot: bool,
+        /// JS-render mode, e.g. `auto` or `always` (authed tier only).
+        #[arg(long, value_name = "MODE")]
+        render: Option<String>,
         /// Override the active org (authed tier only). Falls back to
         /// `SMOOAI_ORG_ID` then the credentials file's `active_org_id`.
         #[arg(long = "org-id", visible_alias = "org")]
         org: Option<String>,
     },
+    /// Crawl a whole site from a seed URL → markdown for every page (authed).
+    Crawl {
+        /// The seed URL to crawl from.
+        url: String,
+        /// Cap the number of pages crawled.
+        #[arg(long, value_name = "N")]
+        limit: Option<u64>,
+        /// Max link-following depth from the seed (sent as `maxDiscoveryDepth`).
+        #[arg(long = "max-depth", value_name = "N")]
+        max_depth: Option<u64>,
+        /// LLM structured extraction per page: a JSON spec is sent verbatim, any
+        /// other string is wrapped as `{"prompt": <SPEC>}`.
+        #[arg(long, value_name = "SPEC")]
+        extract: Option<String>,
+        /// Print the full JSON response instead of the compact summary.
+        #[arg(long)]
+        json: bool,
+        /// Override the active org. Falls back to `SMOOAI_ORG_ID` then the
+        /// credentials file's `active_org_id`.
+        #[arg(long = "org-id", visible_alias = "org")]
+        org: Option<String>,
+    },
+    /// Map a site → the list of discoverable URLs (no content; authed).
+    Map {
+        /// The URL to map.
+        url: String,
+        /// Only return links matching this search term.
+        #[arg(long, value_name = "TERM")]
+        search: Option<String>,
+        /// Cap the number of links returned.
+        #[arg(long, value_name = "N")]
+        limit: Option<u64>,
+        /// Include links on subdomains of the target.
+        #[arg(long = "include-subdomains")]
+        include_subdomains: bool,
+        /// Print the full JSON response instead of one link per line.
+        #[arg(long)]
+        json: bool,
+        /// Override the active org. Falls back to `SMOOAI_ORG_ID` then the
+        /// credentials file's `active_org_id`.
+        #[arg(long = "org-id", visible_alias = "org")]
+        org: Option<String>,
+    },
+}
+
+/// Parse an `--extract` spec: valid JSON is used verbatim, anything else is
+/// wrapped as `{"prompt": <SPEC>}`.
+fn parse_extract(spec: &str) -> Value {
+    serde_json::from_str::<Value>(spec).unwrap_or_else(|_| json!({ "prompt": spec }))
 }
 
 pub async fn cmd(cmd: Cmd) -> Result<()> {
     match cmd {
         Cmd::Scrape {
             url,
             json: as_json,
-            formats,
+            mut formats,
+            extract,
+            screenshot,
+            render,
             org,
         } => {
+            if screenshot {
+                formats.push("screenshot".to_string());
+            }
             let mut body = json!({ "url": url });
             if !formats.is_empty() {
                 body["formats"] = json!(formats);
             }
+            if let Some(spec) = &extract {
+                body["extract"] = parse_extract(spec);
+            }
+            if let Some(mode) = &render {
+                body["render"] = json!(mode);
+            }
 
             // Authed tier when logged in; otherwise the anonymous free tier.
             let resp = match require_authed().await {
@@ -69,6 +140,9 @@ pub async fn cmd(cmd: Cmd) -> Result<()> {
                 }
                 Err(_) => {
                     eprintln!("• not logged in — using the free tier (static only). Run `th auth login` for JS render + higher limits.");
+                    if extract.is_some() || render.is_some() {
+                        eprintln!("• note: --extract/--render are authed-only; the free tier will reject them.");
+                    }
                     public_scrape(&body).await?
                 }
             };
@@ -83,6 +157,85 @@ pub async fn cmd(cmd: Cmd) -> Result<()> {
                 }
             }
         }
+        Cmd::Crawl {
+            url,
+            limit,
+            max_depth,
+            extract,
+            json: as_json,
+            org,
+        } => {
+            let client = require_authed().await.context("`th crawl crawl` requires login — run `th auth login`")?;
+            let o = require_active_org(&client, org)?;
+            let mut body = json!({ "url": url });
+            if let Some(n) = limit {
+                body["limit"] = json!(n);
+            }
+            if let Some(n) = max_depth {
+                body["maxDiscoveryDepth"] = json!(n);
+            }
+            if let Some(spec) = &extract {
+                body["extract"] = parse_extract(spec);
+            }
+            let resp = client
+                .post(&format!("/organizations/{o}/crawl/crawl"), Some(&body))
+                .await
+                .context("POST crawl crawl")?;
+            if as_json {
+                print_json(&resp);
+            } else {
+                let completed = resp.get("completed").and_then(Value::as_u64).unwrap_or(0);
+                let total = resp.get("total").and_then(Value::as_u64).unwrap_or(0);
+                println!("{completed}/{total} pages crawled");
+                if let Some(data) = resp.get("data").and_then(|d| d.as_array()) {
+                    for page in data {
+                        if let Some(u) = page.get("url").and_then(|v| v.as_str()) {
+                            println!("{u}");
+                        }
+                    }
+                }
+            }
+        }
+        Cmd::Map {
+            url,
+            search,
+            limit,
+            include_subdomains,
+            json: as_json,
+            org,
+        } => {
+            let client = require_authed().await.context("`th crawl map` requires login — run `th auth login`")?;
+            let o = require_active_org(&client, org)?;
+            let mut body = json!({ "url": url });
+            if let Some(s) = &search {
+                body["search"] = json!(s);
+            }
+            if let Some(n) = limit {
+                body["limit"] = json!(n);
+            }
+            if include_subdomains {
+                body["includeSubdomains"] = json!(true);
+            }
+            let resp = client
+                .post(&format!("/organizations/{o}/crawl/map"), Some(&body))
+                .await
+                .context("POST crawl map")?;
+            if as_json {
+                print_json(&resp);
+            } else {
+                match resp.get("links").and_then(|l| l.as_array()) {
+                    Some(links) => {
+                        for link in links {
+                            match link.as_str() {
+                                Some(s) => println!("{s}"),
+                                None => println!("{link}"),
+                            }
+                        }
+                    }
+                    None => print_json(&resp),
+                }
+            }
+        }
     }
     Ok(())
 }

crates/smooth-cli/src/smooai/knowledge.rs

@@ -38,6 +38,17 @@ pub enum Cmd {
         #[arg(long = "org-id", visible_alias = "org")]
         org: Option<String>,
     },
+    /// Crawl a website into the org's knowledge base (async ingestion job).
+    AddUrl {
+        /// The website URL to crawl and ingest.
+        url: String,
+        /// Display name for the knowledge source (defaults to the URL).
+        #[arg(long, value_name = "NAME")]
+        name: Option<String>,
+        /// Override the active org. Falls back to `SMOOAI_ORG_ID` then the credentials file's `active_org_id`.
+        #[arg(long = "org-id", visible_alias = "org")]
+        org: Option<String>,
+    },
     /// Register a website as a knowledge source.
     Website {
         /// JSON body describing the website to crawl, or `-` for stdin.
@@ -122,6 +133,17 @@ pub async fn cmd(cmd: Cmd) -> Result<()> {
                     .context("POST knowledge upload")?,
             );
         }
+        Cmd::AddUrl { url, name, org } => {
+            let o = require_active_org(&client, org)?;
+            let name = name.unwrap_or_else(|| url.clone());
+            let body = serde_json::json!({ "urls": [url], "name": name });
+            print_json(
+                &client
+                    .post(&format!("/organizations/{o}/knowledge/websites"), Some(&body))
+                    .await
+                    .context("POST knowledge add-url")?,
+            );
+        }
         Cmd::Website { body, org } => {
             let o = require_active_org(&client, org)?;
             let b = read_body(&body)?;

crates/smooth-cli/src/smooai/websearch.rs

@@ -35,6 +35,10 @@ pub struct SearchArgs {
     /// Search depth: `basic` (default) or `advanced` (authed tier only).
     #[arg(long)]
     pub depth: Option<String>,
+    /// Fetch full page content for each result (crawl-enriched; forces
+    /// `advanced` depth). Authed tier only — ignored on the free tier.
+    #[arg(long)]
+    pub scrape: bool,
     /// Include a synthesized answer (authed tier only — a billed LLM step).
     #[arg(long)]
     pub answer: bool,
@@ -67,6 +71,7 @@ pub async fn run(args: SearchArgs) -> Result<()> {
         json: as_json,
         max,
         depth,
+        scrape,
         answer,
         org,
     } = args;
@@ -75,8 +80,12 @@ pub async fn run(args: SearchArgs) -> Result<()> {
     if let Some(m) = max {
         body["maxResults"] = json!(m);
     }
+    // `--scrape` forces advanced depth (crawl-enriches each result with full page
+    // content); an explicit `--depth` still wins if the user passed one.
     if let Some(d) = depth {
         body["searchDepth"] = json!(d);
+    } else if scrape {
+        body["searchDepth"] = json!("advanced");
     }
     if answer {
         body["includeAnswer"] = json!(true);