diff --git a/.changeset/th-crawl-map-scrape-flags.md b/.changeset/th-crawl-map-scrape-flags.md new file mode 100644 index 000000000..9e1db0e46 --- /dev/null +++ b/.changeset/th-crawl-map-scrape-flags.md @@ -0,0 +1,11 @@ +--- +"@smooai/smooth": minor +--- + +Extend the `th` agentic-tooling commands against the api-prime crawl/search/knowledge routes: + +- **`th crawl crawl `** — new authed whole-site crawl. Supports `--limit`, `--max-depth` (sent as `maxDiscoveryDepth`), `--extract` (JSON spec verbatim or wrapped as `{"prompt": …}`), `--json`, and `--org`. Default output is a compact `completed/total` summary plus one crawled URL per line. +- **`th crawl map `** — new authed URL-discovery command. Supports `--search`, `--limit`, `--include-subdomains`, `--json`, and `--org`; prints one discovered link per line by default. +- **`th crawl scrape`** — new `--extract ` (JSON verbatim or `{"prompt": …}`), `--screenshot` (appends `screenshot` to the formats), and `--render ` flags. `--extract`/`--render` are authed-only; the free tier surfaces the backend's rejection. +- **`th search --scrape`** — forces `searchDepth: "advanced"` so each result is crawl-enriched with full page content (authed tier; clamped on the free tier). +- **`th api knowledge add-url `** — new authed command that kicks off a crawl→ingest job into the org's knowledge base via `POST /organizations/{org}/knowledge/websites` (`--name` defaults to the URL). diff --git a/crates/smooth-cli/src/smooai/crawl.rs b/crates/smooth-cli/src/smooai/crawl.rs index 4d5abf823..38e9025f7 100644 --- a/crates/smooth-cli/src/smooai/crawl.rs +++ b/crates/smooth-cli/src/smooai/crawl.rs @@ -38,11 +38,70 @@ pub enum Cmd { /// (`summary` needs a login — it's a billed LLM step.) #[arg(long = "format", value_name = "FORMAT")] formats: Vec, + /// LLM structured extraction (authed tier only). A JSON schema/spec is + /// sent verbatim; any other string is wrapped as `{"prompt": }`. + #[arg(long, value_name = "SPEC")] + extract: Option, + /// Also capture a screenshot (adds `screenshot` to the formats). + #[arg(long)] + screenshot: bool, + /// JS-render mode, e.g. `auto` or `always` (authed tier only). + #[arg(long, value_name = "MODE")] + render: Option, /// Override the active org (authed tier only). Falls back to /// `SMOOAI_ORG_ID` then the credentials file's `active_org_id`. #[arg(long = "org-id", visible_alias = "org")] org: Option, }, + /// Crawl a whole site from a seed URL → markdown for every page (authed). + Crawl { + /// The seed URL to crawl from. + url: String, + /// Cap the number of pages crawled. + #[arg(long, value_name = "N")] + limit: Option, + /// Max link-following depth from the seed (sent as `maxDiscoveryDepth`). + #[arg(long = "max-depth", value_name = "N")] + max_depth: Option, + /// LLM structured extraction per page: a JSON spec is sent verbatim, any + /// other string is wrapped as `{"prompt": }`. + #[arg(long, value_name = "SPEC")] + extract: Option, + /// Print the full JSON response instead of the compact summary. + #[arg(long)] + json: bool, + /// Override the active org. Falls back to `SMOOAI_ORG_ID` then the + /// credentials file's `active_org_id`. + #[arg(long = "org-id", visible_alias = "org")] + org: Option, + }, + /// Map a site → the list of discoverable URLs (no content; authed). + Map { + /// The URL to map. + url: String, + /// Only return links matching this search term. + #[arg(long, value_name = "TERM")] + search: Option, + /// Cap the number of links returned. + #[arg(long, value_name = "N")] + limit: Option, + /// Include links on subdomains of the target. + #[arg(long = "include-subdomains")] + include_subdomains: bool, + /// Print the full JSON response instead of one link per line. + #[arg(long)] + json: bool, + /// Override the active org. Falls back to `SMOOAI_ORG_ID` then the + /// credentials file's `active_org_id`. + #[arg(long = "org-id", visible_alias = "org")] + org: Option, + }, +} + +/// Parse an `--extract` spec: valid JSON is used verbatim, anything else is +/// wrapped as `{"prompt": }`. +fn parse_extract(spec: &str) -> Value { + serde_json::from_str::(spec).unwrap_or_else(|_| json!({ "prompt": spec })) } pub async fn cmd(cmd: Cmd) -> Result<()> { @@ -50,13 +109,25 @@ pub async fn cmd(cmd: Cmd) -> Result<()> { Cmd::Scrape { url, json: as_json, - formats, + mut formats, + extract, + screenshot, + render, org, } => { + if screenshot { + formats.push("screenshot".to_string()); + } let mut body = json!({ "url": url }); if !formats.is_empty() { body["formats"] = json!(formats); } + if let Some(spec) = &extract { + body["extract"] = parse_extract(spec); + } + if let Some(mode) = &render { + body["render"] = json!(mode); + } // Authed tier when logged in; otherwise the anonymous free tier. let resp = match require_authed().await { @@ -69,6 +140,9 @@ pub async fn cmd(cmd: Cmd) -> Result<()> { } Err(_) => { eprintln!("• not logged in — using the free tier (static only). Run `th auth login` for JS render + higher limits."); + if extract.is_some() || render.is_some() { + eprintln!("• note: --extract/--render are authed-only; the free tier will reject them."); + } public_scrape(&body).await? } }; @@ -83,6 +157,85 @@ pub async fn cmd(cmd: Cmd) -> Result<()> { } } } + Cmd::Crawl { + url, + limit, + max_depth, + extract, + json: as_json, + org, + } => { + let client = require_authed().await.context("`th crawl crawl` requires login — run `th auth login`")?; + let o = require_active_org(&client, org)?; + let mut body = json!({ "url": url }); + if let Some(n) = limit { + body["limit"] = json!(n); + } + if let Some(n) = max_depth { + body["maxDiscoveryDepth"] = json!(n); + } + if let Some(spec) = &extract { + body["extract"] = parse_extract(spec); + } + let resp = client + .post(&format!("/organizations/{o}/crawl/crawl"), Some(&body)) + .await + .context("POST crawl crawl")?; + if as_json { + print_json(&resp); + } else { + let completed = resp.get("completed").and_then(Value::as_u64).unwrap_or(0); + let total = resp.get("total").and_then(Value::as_u64).unwrap_or(0); + println!("{completed}/{total} pages crawled"); + if let Some(data) = resp.get("data").and_then(|d| d.as_array()) { + for page in data { + if let Some(u) = page.get("url").and_then(|v| v.as_str()) { + println!("{u}"); + } + } + } + } + } + Cmd::Map { + url, + search, + limit, + include_subdomains, + json: as_json, + org, + } => { + let client = require_authed().await.context("`th crawl map` requires login — run `th auth login`")?; + let o = require_active_org(&client, org)?; + let mut body = json!({ "url": url }); + if let Some(s) = &search { + body["search"] = json!(s); + } + if let Some(n) = limit { + body["limit"] = json!(n); + } + if include_subdomains { + body["includeSubdomains"] = json!(true); + } + let resp = client + .post(&format!("/organizations/{o}/crawl/map"), Some(&body)) + .await + .context("POST crawl map")?; + if as_json { + print_json(&resp); + } else { + match resp.get("links").and_then(|l| l.as_array()) { + Some(links) => { + for link in links { + match link.as_str() { + Some(s) => println!("{s}"), + None => println!("{link}"), + } + } + } + None => print_json(&resp), + } + } + } } Ok(()) } diff --git a/crates/smooth-cli/src/smooai/knowledge.rs b/crates/smooth-cli/src/smooai/knowledge.rs index 6958abc99..22cb2c84d 100644 --- a/crates/smooth-cli/src/smooai/knowledge.rs +++ b/crates/smooth-cli/src/smooai/knowledge.rs @@ -38,6 +38,17 @@ pub enum Cmd { #[arg(long = "org-id", visible_alias = "org")] org: Option, }, + /// Crawl a website into the org's knowledge base (async ingestion job). + AddUrl { + /// The website URL to crawl and ingest. + url: String, + /// Display name for the knowledge source (defaults to the URL). + #[arg(long, value_name = "NAME")] + name: Option, + /// Override the active org. Falls back to `SMOOAI_ORG_ID` then the credentials file's `active_org_id`. + #[arg(long = "org-id", visible_alias = "org")] + org: Option, + }, /// Register a website as a knowledge source. Website { /// JSON body describing the website to crawl, or `-` for stdin. @@ -122,6 +133,17 @@ pub async fn cmd(cmd: Cmd) -> Result<()> { .context("POST knowledge upload")?, ); } + Cmd::AddUrl { url, name, org } => { + let o = require_active_org(&client, org)?; + let name = name.unwrap_or_else(|| url.clone()); + let body = serde_json::json!({ "urls": [url], "name": name }); + print_json( + &client + .post(&format!("/organizations/{o}/knowledge/websites"), Some(&body)) + .await + .context("POST knowledge add-url")?, + ); + } Cmd::Website { body, org } => { let o = require_active_org(&client, org)?; let b = read_body(&body)?; diff --git a/crates/smooth-cli/src/smooai/websearch.rs b/crates/smooth-cli/src/smooai/websearch.rs index a46761333..acae8c80a 100644 --- a/crates/smooth-cli/src/smooai/websearch.rs +++ b/crates/smooth-cli/src/smooai/websearch.rs @@ -35,6 +35,10 @@ pub struct SearchArgs { /// Search depth: `basic` (default) or `advanced` (authed tier only). #[arg(long)] pub depth: Option, + /// Fetch full page content for each result (crawl-enriched; forces + /// `advanced` depth). Authed tier only — ignored on the free tier. + #[arg(long)] + pub scrape: bool, /// Include a synthesized answer (authed tier only — a billed LLM step). #[arg(long)] pub answer: bool, @@ -67,6 +71,7 @@ pub async fn run(args: SearchArgs) -> Result<()> { json: as_json, max, depth, + scrape, answer, org, } = args; @@ -75,8 +80,12 @@ pub async fn run(args: SearchArgs) -> Result<()> { if let Some(m) = max { body["maxResults"] = json!(m); } + // `--scrape` forces advanced depth (crawl-enriches each result with full page + // content); an explicit `--depth` still wins if the user passed one. if let Some(d) = depth { body["searchDepth"] = json!(d); + } else if scrape { + body["searchDepth"] = json!("advanced"); } if answer { body["includeAnswer"] = json!(true);