test(cli-integration): make setup commands fail fast instead of skip

Replace all swallowed stderr/status in setup commands with proper
assertions. Tests now fail immediately with clear error messages when
prerequisite commands (webset-create, enrichment-create, etc.) fail,
instead of silently skipping or continuing with nil IDs.

Changes:
- Convert _stderr, _status to captured variables
- Replace skip with assert for command status checks
- Add refute_nil for ID validation
- Updated 21 test methods across 3 files

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

Author: benjaminjackson

test/integration/enrichments_cli_integration_test.rb

@@ -67,8 +67,11 @@ def test_enrichment_create_text_format
       create_ws_cmd = "bundle exec exe/exa-ai webset-create " \
                       "--search '{\"query\":\"Tech companies\",\"count\":1}' " \
                       "--output-format json"
-      ws_stdout, _stderr, _status = run_command(create_ws_cmd)
+      ws_stdout, ws_stderr, ws_status = run_command(create_ws_cmd)
+      assert ws_status.success?, "webset-create failed: #{ws_stderr}"
+
       webset = parse_json_output(ws_stdout)
+      refute_nil webset["id"], "webset-create returned no ID"
       webset_id = track_webset(webset["id"])
 
       # Create enrichment
@@ -97,8 +100,11 @@ def test_enrichment_create_options_format
       create_ws_cmd = "bundle exec exe/exa-ai webset-create " \
                       "--search '{\"query\":\"Startups\",\"count\":1}' " \
                       "--output-format json"
-      ws_stdout, _stderr, _status = run_command(create_ws_cmd)
+      ws_stdout, ws_stderr, ws_status = run_command(create_ws_cmd)
+      assert ws_status.success?, "webset-create failed: #{ws_stderr}"
+
       webset = parse_json_output(ws_stdout)
+      refute_nil webset["id"], "webset-create returned no ID"
       webset_id = track_webset(webset["id"])
 
       # Create enrichment with options format
@@ -133,8 +139,11 @@ def test_enrichment_create_options_from_file
       create_ws_cmd = "bundle exec exe/exa-ai webset-create " \
                       "--search '{\"query\":\"E-commerce companies\",\"count\":1}' " \
                       "--output-format json"
-      ws_stdout, _stderr, _status = run_command(create_ws_cmd)
+      ws_stdout, ws_stderr, ws_status = run_command(create_ws_cmd)
+      assert ws_status.success?, "webset-create failed: #{ws_stderr}"
+
       webset = parse_json_output(ws_stdout)
+      refute_nil webset["id"], "webset-create returned no ID"
       webset_id = track_webset(webset["id"])
 
       # Create temporary options file
@@ -175,16 +184,22 @@ def test_enrichment_get
       create_ws_cmd = "bundle exec exe/exa-ai webset-create " \
                       "--search '{\"query\":\"Healthcare companies\",\"count\":1}' " \
                       "--output-format json"
-      ws_stdout, _stderr, _status = run_command(create_ws_cmd)
+      ws_stdout, ws_stderr, ws_status = run_command(create_ws_cmd)
+      assert ws_status.success?, "webset-create failed: #{ws_stderr}"
+
       webset = parse_json_output(ws_stdout)
+      refute_nil webset["id"], "webset-create returned no ID"
       webset_id = track_webset(webset["id"])
 
       create_enrich_cmd = "bundle exec exe/exa-ai enrichment-create #{webset_id} " \
                           "--description 'Find phone number' " \
                           "--format text " \
                           "--output-format json"
-      enrich_stdout, _stderr, _status = run_command(create_enrich_cmd)
+      enrich_stdout, enrich_stderr, enrich_status = run_command(create_enrich_cmd)
+      assert enrich_status.success?, "enrichment-create failed: #{enrich_stderr}"
+
       enrichment = parse_json_output(enrich_stdout)
+      refute_nil enrichment["id"], "enrichment-create returned no ID"
       enrichment_id = track_enrichment(webset_id, enrichment["id"])
 
       # Get the enrichment
@@ -208,17 +223,23 @@ def test_enrichment_list
       create_ws_cmd = "bundle exec exe/exa-ai webset-create " \
                       "--search '{\"query\":\"Fintech companies\",\"count\":1}' " \
                       "--output-format json"
-      ws_stdout, _stderr, _status = run_command(create_ws_cmd)
+      ws_stdout, ws_stderr, ws_status = run_command(create_ws_cmd)
+      assert ws_status.success?, "webset-create failed: #{ws_stderr}"
+
       webset = parse_json_output(ws_stdout)
+      refute_nil webset["id"], "webset-create returned no ID"
       webset_id = track_webset(webset["id"])
 
       # Create an enrichment
       create_enrich_cmd = "bundle exec exe/exa-ai enrichment-create #{webset_id} " \
                           "--description 'Find headquarters location' " \
                           "--format text " \
                           "--output-format json"
-      enrich_stdout, _stderr, _status = run_command(create_enrich_cmd)
+      enrich_stdout, enrich_stderr, enrich_status = run_command(create_enrich_cmd)
+      assert enrich_status.success?, "enrichment-create failed: #{enrich_stderr}"
+
       enrichment = parse_json_output(enrich_stdout)
+      refute_nil enrichment["id"], "enrichment-create returned no ID"
       track_enrichment(webset_id, enrichment["id"])
 
       # List enrichments
@@ -243,16 +264,22 @@ def test_enrichment_update
       create_ws_cmd = "bundle exec exe/exa-ai webset-create " \
                       "--search '{\"query\":\"Retail companies\",\"count\":1}' " \
                       "--output-format json"
-      ws_stdout, _stderr, _status = run_command(create_ws_cmd)
+      ws_stdout, ws_stderr, ws_status = run_command(create_ws_cmd)
+      assert ws_status.success?, "webset-create failed: #{ws_stderr}"
+
       webset = parse_json_output(ws_stdout)
+      refute_nil webset["id"], "webset-create returned no ID"
       webset_id = track_webset(webset["id"])
 
       create_enrich_cmd = "bundle exec exe/exa-ai enrichment-create #{webset_id} " \
                           "--description 'Initial description' " \
                           "--format text " \
                           "--output-format json"
-      enrich_stdout, _stderr, _status = run_command(create_enrich_cmd)
+      enrich_stdout, enrich_stderr, enrich_status = run_command(create_enrich_cmd)
+      assert enrich_status.success?, "enrichment-create failed: #{enrich_stderr}"
+
       enrichment = parse_json_output(enrich_stdout)
+      refute_nil enrichment["id"], "enrichment-create returned no ID"
       enrichment_id = track_enrichment(webset_id, enrichment["id"])
 
       # Update the enrichment
@@ -278,16 +305,22 @@ def test_enrichment_delete
       create_ws_cmd = "bundle exec exe/exa-ai webset-create " \
                       "--search '{\"query\":\"Manufacturing companies\",\"count\":1}' " \
                       "--output-format json"
-      ws_stdout, _stderr, _status = run_command(create_ws_cmd)
+      ws_stdout, ws_stderr, ws_status = run_command(create_ws_cmd)
+      assert ws_status.success?, "webset-create failed: #{ws_stderr}"
+
       webset = parse_json_output(ws_stdout)
+      refute_nil webset["id"], "webset-create returned no ID"
       webset_id = track_webset(webset["id"])
 
       create_enrich_cmd = "bundle exec exe/exa-ai enrichment-create #{webset_id} " \
                           "--description 'To be deleted' " \
                           "--format text " \
                           "--output-format json"
-      enrich_stdout, _stderr, _status = run_command(create_enrich_cmd)
+      enrich_stdout, enrich_stderr, enrich_status = run_command(create_enrich_cmd)
+      assert enrich_status.success?, "enrichment-create failed: #{enrich_stderr}"
+
       enrichment = parse_json_output(enrich_stdout)
+      refute_nil enrichment["id"], "enrichment-create returned no ID"
       enrichment_id = track_enrichment(webset_id, enrichment["id"])
 
       # Delete the enrichment (with --force to skip confirmation)
@@ -313,16 +346,22 @@ def test_enrichment_cancel
       create_ws_cmd = "bundle exec exe/exa-ai webset-create " \
                       "--search '{\"query\":\"Education companies\",\"count\":1}' " \
                       "--output-format json"
-      ws_stdout, _stderr, _status = run_command(create_ws_cmd)
+      ws_stdout, ws_stderr, ws_status = run_command(create_ws_cmd)
+      assert ws_status.success?, "webset-create failed: #{ws_stderr}"
+
       webset = parse_json_output(ws_stdout)
+      refute_nil webset["id"], "webset-create returned no ID"
       webset_id = track_webset(webset["id"])
 
       create_enrich_cmd = "bundle exec exe/exa-ai enrichment-create #{webset_id} " \
                           "--description 'To be cancelled' " \
                           "--format text " \
                           "--output-format json"
-      enrich_stdout, _stderr, _status = run_command(create_enrich_cmd)
+      enrich_stdout, enrich_stderr, enrich_status = run_command(create_enrich_cmd)
+      assert enrich_status.success?, "enrichment-create failed: #{enrich_stderr}"
+
       enrichment = parse_json_output(enrich_stdout)
+      refute_nil enrichment["id"], "enrichment-create returned no ID"
       enrichment_id = track_enrichment(webset_id, enrichment["id"])
 
       # Cancel the enrichment
@@ -346,8 +385,11 @@ def test_enrichment_create_pretty_format
       create_ws_cmd = "bundle exec exe/exa-ai webset-create " \
                       "--search '{\"query\":\"AI companies\",\"count\":1}' " \
                       "--output-format json"
-      ws_stdout, _stderr, _status = run_command(create_ws_cmd)
+      ws_stdout, ws_stderr, ws_status = run_command(create_ws_cmd)
+      assert ws_status.success?, "webset-create failed: #{ws_stderr}"
+
       webset = parse_json_output(ws_stdout)
+      refute_nil webset["id"], "webset-create returned no ID"
       webset_id = track_webset(webset["id"])
 
       # Create enrichment with pretty format

test/integration/websets_cli_integration_test.rb

@@ -69,10 +69,10 @@ def test_webset_create_basic
               "--output-format json"
 
     stdout, stderr, status = run_command(command)
-    skip "webset-create failed: #{stderr}" unless status.success?
+    assert status.success?, "webset-create failed: #{stderr}"
 
     result = parse_json_output(stdout)
-    skip "webset-create returned no ID" if result["id"].nil?
+    refute_nil result["id"], "webset-create returned no ID"
 
     track_webset(result["id"])
 
@@ -101,10 +101,10 @@ def test_webset_create_with_search_file
               "--output-format json"
 
     stdout, stderr, status = run_command(command)
-    skip "webset-create failed: #{stderr}" unless status.success?
+    assert status.success?, "webset-create failed: #{stderr}"
 
     result = parse_json_output(stdout)
-    skip "webset-create returned no ID" if result["id"].nil?
+    refute_nil result["id"], "webset-create returned no ID"
 
     track_webset(result["id"])
 
@@ -127,10 +127,10 @@ def test_webset_create_with_metadata
                 "--output-format json"
 
       stdout, stderr, status = run_command(command)
-      skip "webset-create failed: #{stderr}" unless status.success?
+      assert status.success?, "webset-create failed: #{stderr}"
 
       result = parse_json_output(stdout)
-      skip "webset-create returned no ID" if result["id"].nil?
+      refute_nil result["id"], "webset-create returned no ID"
 
       track_webset(result["id"])
 
@@ -152,10 +152,10 @@ def test_webset_create_with_external_id
                 "--output-format json"
 
       stdout, stderr, status = run_command(command)
-      skip "webset-create failed: #{stderr}" unless status.success?
+      assert status.success?, "webset-create failed: #{stderr}"
 
       result = parse_json_output(stdout)
-      skip "webset-create returned no ID" if result["id"].nil?
+      refute_nil result["id"], "webset-create returned no ID"
 
       track_webset(result["id"])
 
@@ -189,10 +189,10 @@ def test_webset_create_with_enrichments
               "--output-format json"
 
     stdout, stderr, status = run_command(command)
-    skip "webset-create failed: #{stderr}" unless status.success?
+    assert status.success?, "webset-create failed: #{stderr}"
 
     result = parse_json_output(stdout)
-    skip "webset-create returned no ID" if result["id"].nil?
+    refute_nil result["id"], "webset-create returned no ID"
 
     track_webset(result["id"])
 
@@ -216,10 +216,10 @@ def test_webset_get
                        "--output-format json"
 
       create_stdout, create_stderr, create_status = run_command(create_command)
-      skip "webset-create failed: #{create_stderr}" unless create_status.success?
+      assert create_status.success?, "webset-create failed: #{create_stderr}"
 
       created = parse_json_output(create_stdout)
-      skip "webset-create returned no ID" if created["id"].nil?
+      refute_nil created["id"], "webset-create returned no ID"
 
       webset_id = track_webset(created["id"])
 
@@ -246,10 +246,10 @@ def test_webset_get_pretty_format
                        "--output-format json"
 
       create_stdout, create_stderr, create_status = run_command(create_command)
-      skip "webset-create failed: #{create_stderr}" unless create_status.success?
+      assert create_status.success?, "webset-create failed: #{create_stderr}"
 
       created = parse_json_output(create_stdout)
-      skip "webset-create returned no ID" if created["id"].nil?
+      refute_nil created["id"], "webset-create returned no ID"
 
       webset_id = track_webset(created["id"])
 
@@ -342,10 +342,10 @@ def test_webset_update
                        "--output-format json"
 
       create_stdout, create_stderr, create_status = run_command(create_command)
-      skip "webset-create failed: #{create_stderr}" unless create_status.success?
+      assert create_status.success?, "webset-create failed: #{create_stderr}"
 
       created = parse_json_output(create_stdout)
-      skip "webset-create returned no ID" if created["id"].nil?
+      refute_nil created["id"], "webset-create returned no ID"
 
       webset_id = track_webset(created["id"])
 
@@ -378,10 +378,10 @@ def test_webset_delete
                        "--output-format json"
 
       create_stdout, create_stderr, create_status = run_command(create_command)
-      skip "webset-create failed: #{create_stderr}" unless create_status.success?
+      assert create_status.success?, "webset-create failed: #{create_stderr}"
 
       created = parse_json_output(create_stdout)
-      skip "webset-create returned no ID" if created["id"].nil?
+      refute_nil created["id"], "webset-create returned no ID"
 
       webset_id = track_webset(created["id"])
 
@@ -409,10 +409,10 @@ def test_webset_cancel
                        "--output-format json"
 
       create_stdout, create_stderr, create_status = run_command(create_command)
-      skip "webset-create failed: #{create_stderr}" unless create_status.success?
+      assert create_status.success?, "webset-create failed: #{create_stderr}"
 
       created = parse_json_output(create_stdout)
-      skip "webset-create returned no ID" if created["id"].nil?
+      refute_nil created["id"], "webset-create returned no ID"
 
       webset_id = track_webset(created["id"])
 
@@ -590,10 +590,10 @@ def test_webset_search_create
                      "--search '{\"query\":\"Tech companies\",\"count\":1}' " \
                      "--output-format json"
     create_stdout, create_stderr, create_status = run_command(create_command)
-    skip "webset-create failed: #{create_stderr}" unless create_status.success?
+    assert create_status.success?, "webset-create failed: #{create_stderr}"
 
     created = parse_json_output(create_stdout)
-    skip "webset-create returned no ID" if created["id"].nil?
+    refute_nil created["id"], "webset-create returned no ID"
 
     webset_id = track_webset(created["id"])
 
@@ -618,18 +618,21 @@ def test_webset_search_get
                      "--search '{\"query\":\"Software companies\",\"count\":1}' " \
                      "--output-format json"
     create_stdout, create_stderr, create_status = run_command(create_command)
-    skip "webset-create failed: #{create_stderr}" unless create_status.success?
+    assert create_status.success?, "webset-create failed: #{create_stderr}"
 
     created = parse_json_output(create_stdout)
-    skip "webset-create returned no ID" if created["id"].nil?
+    refute_nil created["id"], "webset-create returned no ID"
 
     webset_id = track_webset(created["id"])
 
     # Create a search within the webset
     search_command = "bundle exec exe/exa-ai-webset-search-create #{webset_id} " \
                      "--query 'Machine learning startups' --output-format json"
-    search_stdout, _stderr, _status = run_command(search_command)
+    search_stdout, search_stderr, search_status = run_command(search_command)
+    assert search_status.success?, "webset-search-create failed: #{search_stderr}"
+
     search = parse_json_output(search_stdout)
+    refute_nil search["id"], "webset-search-create returned no ID"
     search_id = track_search(webset_id, search["id"])
 
     # Get the search
@@ -651,18 +654,21 @@ def test_webset_search_cancel
                      "--search '{\"query\":\"Healthcare companies\",\"count\":1}' " \
                      "--output-format json"
     create_stdout, create_stderr, create_status = run_command(create_command)
-    skip "webset-create failed: #{create_stderr}" unless create_status.success?
+    assert create_status.success?, "webset-create failed: #{create_stderr}"
 
     created = parse_json_output(create_stdout)
-    skip "webset-create returned no ID" if created["id"].nil?
+    refute_nil created["id"], "webset-create returned no ID"
 
     webset_id = track_webset(created["id"])
 
     # Create a search within the webset
     search_command = "bundle exec exe/exa-ai-webset-search-create #{webset_id} " \
                      "--query 'Medical device companies' --output-format json"
-    search_stdout, _stderr, _status = run_command(search_command)
+    search_stdout, search_stderr, search_status = run_command(search_command)
+    assert search_status.success?, "webset-search-create failed: #{search_stderr}"
+
     search = parse_json_output(search_stdout)
+    refute_nil search["id"], "webset-search-create returned no ID"
     search_id = track_search(webset_id, search["id"])
 
     # Cancel the search

test/integration/websets_items_cli_integration_test.rb

@@ -95,7 +95,7 @@ def test_basic_item_listing
     command = "bundle exec exe/exa-ai-webset-item-list #{webset_id} --output-format json"
     stdout, stderr, status = run_command(command)
 
-    skip "webset-item-list failed: #{stderr}" unless status.success?
+    assert status.success?, "webset-item-list failed: #{stderr}"
 
     result = parse_json_output(stdout)
 
@@ -123,7 +123,7 @@ def test_cursor_based_pagination
     command1 = "bundle exec exe/exa-ai-webset-item-list #{webset_id} --limit 2 --output-format json"
     stdout1, stderr1, status1 = run_command(command1)
 
-    skip "First page request failed: #{stderr1}" unless status1.success?
+    assert status1.success?, "First page request failed: #{stderr1}"
 
     result1 = parse_json_output(stdout1)
 
@@ -145,7 +145,7 @@ def test_cursor_based_pagination
     command2 = "bundle exec exe/exa-ai-webset-item-list #{webset_id} --limit 2 --cursor '#{cursor}' --output-format json"
     stdout2, stderr2, status2 = run_command(command2)
 
-    skip "Second page request failed: #{stderr2}" unless status2.success?
+    assert status2.success?, "Second page request failed: #{stderr2}"
 
     result2 = parse_json_output(stdout2)