Skip to content

th-1b445c: paginate th api crm {contacts,companies,deals} list #188

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
brentrager wants to merge 1 commit into
base: main
Choose a base branch
from
Open

th-1b445c: paginate th api crm {contacts,companies,deals} list #188

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .changeset/th-1b445c-crm-list-pagination.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
---
'@smooai/smooth': minor
---

Fix silent truncation in `th api crm contacts|companies|deals list`.

These commands only sent `?limit=` (default 50; the API clamps to a 200-row max page) with no offset and no auto-paginate, so they silently returned just the first page — a `contacts list --limit 1000` came back with 200 rows while the endpoint's `x-total-count` said 267, dropping 67 contacts with no warning. The endpoint was always fine; the CLI was the bug.

Each `list` command now takes `--offset <N>` (page past the first page) and `--all` (fetch every page, ignoring `--limit`/`--offset`). `--all` reuses the same offset-paginating loop the import scan already used — generalized into `fetch_all_resource` and paced to stay under the 100-req/60s rate limit. The deals `--stage` filter is not applied in `--all` mode (the endpoint filters by `stage`, not the paginator's `search`).
124 changes: 96 additions & 28 deletions crates/smooth-cli/src/smooai/crm.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,12 @@ pub enum CompaniesCmd {
/// Maximum number of companies to return.
#[arg(long, default_value = "50")]
limit: u32,
/// Skip the first N companies (page offset).
#[arg(long, default_value = "0")]
offset: u32,
/// Fetch every page (ignores --limit/--offset).
#[arg(long)]
all: bool,
/// Print raw JSON instead of the table.
#[arg(long)]
json: bool,
Expand Down Expand Up @@ -136,6 +142,12 @@ pub enum DealsCmd {
/// Maximum number of deals to return.
#[arg(long, default_value = "50")]
limit: u32,
/// Skip the first N deals (page offset).
#[arg(long, default_value = "0")]
offset: u32,
/// Fetch every page (ignores --limit/--offset; the stage filter is not applied in --all mode).
#[arg(long)]
all: bool,
/// Print raw JSON instead of the pipeline view.
#[arg(long)]
json: bool,
Expand Down Expand Up @@ -484,6 +496,12 @@ pub enum ContactsCmd {
/// Maximum number of contacts to return.
#[arg(long, default_value = "50")]
limit: u32,
/// Skip the first N contacts (page offset).
#[arg(long, default_value = "0")]
offset: u32,
/// Fetch every page (ignores --limit/--offset).
#[arg(long)]
all: bool,
},
/// Get a single contact by id.
Get {
Expand Down Expand Up @@ -563,13 +581,25 @@ fn resolve_org(override_org: Option<String>) -> Result<String> {
async fn contacts(cmd: ContactsCmd) -> Result<()> {
let client = UserClient::from_user_session().await?;
match cmd {
ContactsCmd::List { org, search, limit } => {
ContactsCmd::List {
org,
search,
limit,
offset,
all,
} => {
let org = resolve_org(org)?;
let mut path = format!("/organizations/{org}/crm/contacts?limit={limit}");
if let Some(s) = search.filter(|s| !s.trim().is_empty()) {
path.push_str(&format!("&search={}", urlencoding::encode(&s)));
let search = search.filter(|s| !s.trim().is_empty());
if all {
let rows = fetch_all_resource(&client, &org, "contacts", search.as_deref(), std::time::Duration::from_millis(350)).await?;
print_json(&Value::Array(rows));
} else {
let mut path = format!("/organizations/{org}/crm/contacts?limit={limit}&offset={offset}");
if let Some(s) = &search {
path.push_str(&format!("&search={}", urlencoding::encode(s)));
}
print_json(&client.get(&path).await.context("GET contacts")?);
}
print_json(&client.get(&path).await.context("GET contacts")?);
}
ContactsCmd::Get { contact_id, org } => {
let org = resolve_org(org)?;
Expand Down Expand Up @@ -615,17 +645,30 @@ async fn contacts(cmd: ContactsCmd) -> Result<()> {
async fn companies(cmd: CompaniesCmd) -> Result<()> {
let client = UserClient::from_user_session().await?;
match cmd {
CompaniesCmd::List { org, search, limit, json } => {
CompaniesCmd::List {
org,
search,
limit,
offset,
all,
json,
} => {
let org = resolve_org(org)?;
let mut path = format!("/organizations/{org}/crm/companies?limit={limit}");
if let Some(s) = search.filter(|s| !s.trim().is_empty()) {
path.push_str(&format!("&search={}", urlencoding::encode(&s)));
}
let body = client.get(&path).await.context("GET companies")?;
if json {
print_json(&body);
let search = search.filter(|s| !s.trim().is_empty());
if all {
let rows = fetch_all_resource(&client, &org, "companies", search.as_deref(), std::time::Duration::from_millis(350)).await?;
print_json(&Value::Array(rows));
} else {
render_companies(&body);
let mut path = format!("/organizations/{org}/crm/companies?limit={limit}&offset={offset}");
if let Some(s) = &search {
path.push_str(&format!("&search={}", urlencoding::encode(s)));
}
let body = client.get(&path).await.context("GET companies")?;
if json {
print_json(&body);
} else {
render_companies(&body);
}
}
}
CompaniesCmd::Show { company_id, org } => {
Expand Down Expand Up @@ -731,17 +774,32 @@ fn render_companies(body: &Value) {
async fn deals(cmd: DealsCmd) -> Result<()> {
let client = UserClient::from_user_session().await?;
match cmd {
DealsCmd::List { org, stage, limit, json } => {
DealsCmd::List {
org,
stage,
limit,
offset,
all,
json,
} => {
let org = resolve_org(org)?;
let mut path = format!("/organizations/{org}/crm/deals?limit={limit}");
if let Some(s) = stage.filter(|s| !s.trim().is_empty()) {
path.push_str(&format!("&stage={}", urlencoding::encode(&s)));
}
let body = client.get(&path).await.context("GET deals")?;
if json {
print_json(&body);
if all {
// The deals endpoint filters by `stage`, not `search`, so the
// shared paginator (which only knows `search`) can't apply it —
// --all pulls every deal regardless of the stage flag.
let rows = fetch_all_resource(&client, &org, "deals", None, std::time::Duration::from_millis(350)).await?;
print_json(&Value::Array(rows));
} else {
render_deals(&body);
let mut path = format!("/organizations/{org}/crm/deals?limit={limit}&offset={offset}");
if let Some(s) = stage.filter(|s| !s.trim().is_empty()) {
path.push_str(&format!("&stage={}", urlencoding::encode(&s)));
}
let body = client.get(&path).await.context("GET deals")?;
if json {
print_json(&body);
} else {
render_deals(&body);
}
}
}
DealsCmd::Show { deal_id, org } => {
Expand Down Expand Up @@ -1867,14 +1925,19 @@ fn norm_phone(v: &Value) -> Option<String> {
}
}

/// Fetch every contact in the org, paging in blocks of 200. Paces between
/// pages so the existing-contacts scan doesn't itself trip the rate limit.
async fn fetch_all(client: &UserClient, org: &str, rate: std::time::Duration) -> Result<Vec<Value>> {
/// Fetch every row of a CRM resource in the org, paging in blocks of 200
/// (the API's max page size) until a short page. Paces between pages so a
/// full scan doesn't itself trip the 100-req/60s rate limit. Adds
/// `&search=` (url-encoded) when a search term is given.
async fn fetch_all_resource(client: &UserClient, org: &str, resource: &str, search: Option<&str>, rate: std::time::Duration) -> Result<Vec<Value>> {
let mut all = Vec::new();
let mut offset = 0u32;
loop {
let path = format!("/organizations/{org}/crm/contacts?limit=200&offset={offset}");
let body = client.get(&path).await.with_context(|| format!("GET contacts (offset {offset})"))?;
let mut path = format!("/organizations/{org}/crm/{resource}?limit=200&offset={offset}");
if let Some(s) = search {
path.push_str(&format!("&search={}", urlencoding::encode(s)));
}
let body = client.get(&path).await.with_context(|| format!("GET {resource} (offset {offset})"))?;
let page = body.as_array().cloned().unwrap_or_default();
let n = page.len();
all.extend(page);
Expand All @@ -1887,6 +1950,11 @@ async fn fetch_all(client: &UserClient, org: &str, rate: std::time::Duration) ->
Ok(all)
}

/// Fetch every contact in the org — the import scan's existing-contacts pull.
async fn fetch_all(client: &UserClient, org: &str, rate: std::time::Duration) -> Result<Vec<Value>> {
fetch_all_resource(client, org, "contacts", None, rate).await
}

/// A single write to perform.
#[derive(Clone, Copy)]
enum Op<'a> {
Expand Down
Loading