test: improve error messages in CLI integration test assertions

Capture stderr output and include it in assertion failure messages across all CLI integration tests for better debugging when tests fail. This ensures that diagnostic information from failed commands is visible when assertions fail.

Changes:
- Replace unused `_stderr` with captured `stderr` variable in test helper calls
- Append stderr and stdout to assertion messages for all CLI commands
- Applies to all integration test files: cli_executables, enrichments, websets, and webset_items tests

These changes maintain existing test logic while improving observability of failures.

Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>

Author: benjaminjackson

test/integration/cli_executables_test.rb

@@ -17,9 +17,9 @@ def run_executable(command)
 
   # Main dispatcher executable
   def test_exa_ai_help
-    stdout, _stderr, status = run_executable("bundle exec exe/exa-ai --help")
+    stdout, stderr, status = run_executable("bundle exec exe/exa-ai --help")
 
-    assert status.success?, "exa-ai --help should exit successfully"
+    assert status.success?, "exa-ai --help should exit successfully. stderr: #{stderr}, stdout: #{stdout}"
     assert_includes stdout, "Exa CLI"
     assert_includes stdout, "Core Search:"
     assert_includes stdout, "search"
@@ -28,27 +28,27 @@ def test_exa_ai_help
   end
 
   def test_exa_ai_version
-    stdout, _stderr, status = run_executable("bundle exec exe/exa-ai --version")
+    stdout, stderr, status = run_executable("bundle exec exe/exa-ai --version")
 
-    assert status.success?, "exa-ai --version should exit successfully"
+    assert status.success?, "exa-ai --version should exit successfully. stderr: #{stderr}, stdout: #{stdout}"
     assert_match(/\d+\.\d+\.\d+/, stdout)
   end
 
   # Search command
   def test_search_help
-    stdout, _stderr, status = run_executable("bundle exec exe/exa-ai-search --help")
+    stdout, stderr, status = run_executable("bundle exec exe/exa-ai-search --help")
 
-    assert status.success?, "exa-ai-search --help should exit successfully"
+    assert status.success?, "exa-ai-search --help should exit successfully. stderr: #{stderr}, stdout: #{stdout}"
     assert_includes stdout, "Usage:"
     assert_includes stdout, "search"
     assert_includes stdout, "QUERY"
   end
 
   # Answer command
   def test_answer_help
-    stdout, _stderr, status = run_executable("bundle exec exe/exa-ai-answer --help")
+    stdout, stderr, status = run_executable("bundle exec exe/exa-ai-answer --help")
 
-    assert status.success?, "exa-ai-answer --help should exit successfully"
+    assert status.success?, "exa-ai-answer --help should exit successfully. stderr: #{stderr}, stdout: #{stdout}"
     assert_includes stdout, "Usage:"
     assert_includes stdout, "answer"
     assert_includes stdout, "QUERY"
@@ -58,65 +58,65 @@ def test_answer_help
 
   # Context command
   def test_context_help
-    stdout, _stderr, status = run_executable("bundle exec exe/exa-ai-context --help")
+    stdout, stderr, status = run_executable("bundle exec exe/exa-ai-context --help")
 
-    assert status.success?, "exa-ai-context --help should exit successfully"
+    assert status.success?, "exa-ai-context --help should exit successfully. stderr: #{stderr}, stdout: #{stdout}"
     assert_includes stdout, "Usage:"
     assert_includes stdout, "context"
     assert_includes stdout, "query"
   end
 
   # Get contents command
   def test_get_contents_help
-    stdout, _stderr, status = run_executable("bundle exec exe/exa-ai-get-contents --help")
+    stdout, stderr, status = run_executable("bundle exec exe/exa-ai-get-contents --help")
 
-    assert status.success?, "exa-ai-get-contents --help should exit successfully"
+    assert status.success?, "exa-ai-get-contents --help should exit successfully. stderr: #{stderr}, stdout: #{stdout}"
     assert_includes stdout, "Usage:"
     assert_includes stdout, "get-contents"
   end
 
   # Research start command
   def test_research_start_help
-    stdout, _stderr, status = run_executable("bundle exec exe/exa-ai-research-start --help")
+    stdout, stderr, status = run_executable("bundle exec exe/exa-ai-research-start --help")
 
-    assert status.success?, "exa-ai-research-start --help should exit successfully"
+    assert status.success?, "exa-ai-research-start --help should exit successfully. stderr: #{stderr}, stdout: #{stdout}"
     assert_includes stdout, "Usage:"
     assert_includes stdout, "research-start"
     assert_includes stdout, "--instructions"
   end
 
   # Research get command
   def test_research_get_help
-    stdout, _stderr, status = run_executable("bundle exec exe/exa-ai-research-get --help")
+    stdout, stderr, status = run_executable("bundle exec exe/exa-ai-research-get --help")
 
-    assert status.success?, "exa-ai-research-get --help should exit successfully"
+    assert status.success?, "exa-ai-research-get --help should exit successfully. stderr: #{stderr}, stdout: #{stdout}"
     assert_includes stdout, "Usage:"
     assert_includes stdout, "research-get"
     assert_includes stdout, "research_id"
   end
 
   # Research list command
   def test_research_list_help
-    stdout, _stderr, status = run_executable("bundle exec exe/exa-ai-research-list --help")
+    stdout, stderr, status = run_executable("bundle exec exe/exa-ai-research-list --help")
 
-    assert status.success?, "exa-ai-research-list --help should exit successfully"
+    assert status.success?, "exa-ai-research-list --help should exit successfully. stderr: #{stderr}, stdout: #{stdout}"
     assert_includes stdout, "Usage:"
     assert_includes stdout, "research-list"
   end
 
   # Test that main dispatcher can route to subcommands
   def test_dispatcher_routes_to_search_help
-    stdout, _stderr, status = run_executable("bundle exec exe/exa-ai search --help")
+    stdout, stderr, status = run_executable("bundle exec exe/exa-ai search --help")
 
-    assert status.success?, "exa-ai search --help should route correctly"
+    assert status.success?, "exa-ai search --help should route correctly. stderr: #{stderr}, stdout: #{stdout}"
     assert_includes stdout, "Usage:"
     assert_includes stdout, "search"
   end
 
   def test_dispatcher_routes_to_answer_help
-    stdout, _stderr, status = run_executable("bundle exec exe/exa-ai answer --help")
+    stdout, stderr, status = run_executable("bundle exec exe/exa-ai answer --help")
 
-    assert status.success?, "exa-ai answer --help should route correctly"
+    assert status.success?, "exa-ai answer --help should route correctly. stderr: #{stderr}, stdout: #{stdout}"
     assert_includes stdout, "Usage:"
     assert_includes stdout, "answer"
   end
@@ -125,7 +125,7 @@ def test_dispatcher_routes_to_answer_help
   def test_invalid_command_shows_error
     stdout, stderr, status = run_executable("bundle exec exe/exa-ai invalid-command")
 
-    refute status.success?, "Invalid command should exit with error"
+    refute status.success?, "Invalid command should exit with error. stderr: #{stderr}, stdout: #{stdout}"
     # Error message could be in stdout or stderr
     combined_output = stdout + stderr
     assert_includes combined_output, "Unknown command"

test/integration/enrichments_cli_integration_test.rb

@@ -77,9 +77,9 @@ def test_enrichment_create_text_format
                 "--format text " \
                 "--output-format json"
 
-      stdout, _stderr, status = run_command(command)
+      stdout, stderr, status = run_command(command)
 
-      assert status.success?, "enrichment-create should succeed"
+      assert status.success?, "enrichment-create should succeed. stderr: #{stderr}, stdout: #{stdout}"
       result = parse_json_output(stdout)
       track_enrichment(webset_id, result["id"])
 
@@ -114,9 +114,9 @@ def test_enrichment_create_options_format
                 "--options '#{JSON.generate(options)}' " \
                 "--output-format json"
 
-      stdout, _stderr, status = run_command(command)
+      stdout, stderr, status = run_command(command)
 
-      assert status.success?, "enrichment-create with options should succeed"
+      assert status.success?, "enrichment-create with options should succeed. stderr: #{stderr}, stdout: #{stdout}"
       result = parse_json_output(stdout)
       track_enrichment(webset_id, result["id"])
 
@@ -154,9 +154,9 @@ def test_enrichment_create_options_from_file
                 "--options @#{options_file.path} " \
                 "--output-format json"
 
-      stdout, _stderr, status = run_command(command)
+      stdout, stderr, status = run_command(command)
 
-      assert status.success?, "enrichment-create with options file should succeed"
+      assert status.success?, "enrichment-create with options file should succeed. stderr: #{stderr}, stdout: #{stdout}"
       result = parse_json_output(stdout)
       track_enrichment(webset_id, result["id"])
 
@@ -190,9 +190,9 @@ def test_enrichment_get
       # Get the enrichment
       command = "bundle exec exe/exa-ai enrichment-get #{webset_id} #{enrichment_id} --output-format json"
 
-      stdout, _stderr, status = run_command(command)
+      stdout, stderr, status = run_command(command)
 
-      assert status.success?, "enrichment-get should succeed"
+      assert status.success?, "enrichment-get should succeed. stderr: #{stderr}, stdout: #{stdout}"
       result = parse_json_output(stdout)
 
       assert_equal enrichment_id, result["id"]
@@ -224,9 +224,9 @@ def test_enrichment_list
       # List enrichments
       command = "bundle exec exe/exa-ai enrichment-list #{webset_id} --output-format json"
 
-      stdout, _stderr, status = run_command(command)
+      stdout, stderr, status = run_command(command)
 
-      assert status.success?, "enrichment-list should succeed"
+      assert status.success?, "enrichment-list should succeed. stderr: #{stderr}, stdout: #{stdout}"
       result = parse_json_output(stdout)
 
       # List response has data array
@@ -260,9 +260,9 @@ def test_enrichment_update
                 "--description 'Updated description' " \
                 "--output-format json"
 
-      stdout, _stderr, status = run_command(command)
+      stdout, stderr, status = run_command(command)
 
-      assert status.success?, "enrichment-update should succeed"
+      assert status.success?, "enrichment-update should succeed. stderr: #{stderr}, stdout: #{stdout}"
       result = parse_json_output(stdout)
 
       assert_equal enrichment_id, result["id"]
@@ -294,9 +294,9 @@ def test_enrichment_delete
       command = "bundle exec exe/exa-ai enrichment-delete #{webset_id} #{enrichment_id} " \
                 "--force --output-format json"
 
-      stdout, _stderr, status = run_command(command)
+      stdout, stderr, status = run_command(command)
 
-      assert status.success?, "enrichment-delete should succeed"
+      assert status.success?, "enrichment-delete should succeed. stderr: #{stderr}, stdout: #{stdout}"
       result = parse_json_output(stdout)
 
       # API returns the enrichment object after deletion
@@ -328,9 +328,9 @@ def test_enrichment_cancel
       # Cancel the enrichment
       command = "bundle exec exe/exa-ai enrichment-cancel #{webset_id} #{enrichment_id} --output-format json"
 
-      stdout, _stderr, status = run_command(command)
+      stdout, stderr, status = run_command(command)
 
-      assert status.success?, "enrichment-cancel should succeed"
+      assert status.success?, "enrichment-cancel should succeed. stderr: #{stderr}, stdout: #{stdout}"
       result = parse_json_output(stdout)
 
       assert_equal enrichment_id, result["id"]
@@ -356,9 +356,9 @@ def test_enrichment_create_pretty_format
                 "--format url " \
                 "--output-format pretty"
 
-      stdout, _stderr, status = run_command(command)
+      stdout, stderr, status = run_command(command)
 
-      assert status.success?, "enrichment-create with pretty format should succeed"
+      assert status.success?, "enrichment-create with pretty format should succeed. stderr: #{stderr}, stdout: #{stdout}"
       # Pretty format outputs human-readable text, not JSON
       assert_includes stdout, "Enrichment ID:"
       assert_includes stdout, "Webset ID:"
@@ -381,7 +381,7 @@ def test_enrichment_create_invalid_format
 
     stdout, stderr, status = run_command(command)
 
-    refute status.success?, "enrichment-create with invalid format should fail"
+    refute status.success?, "enrichment-create with invalid format should fail. stderr: #{stderr}, stdout: #{stdout}"
     combined = stdout + stderr
     assert_includes combined.downcase, "error"
   end
@@ -394,7 +394,7 @@ def test_enrichment_create_missing_description
 
     stdout, stderr, status = run_command(command)
 
-    refute status.success?, "enrichment-create without description should fail"
+    refute status.success?, "enrichment-create without description should fail. stderr: #{stderr}, stdout: #{stdout}"
     combined = stdout + stderr
     assert_includes combined, "description"
   end
@@ -410,16 +410,16 @@ def test_enrichment_create_options_without_options
 
     stdout, stderr, status = run_command(command)
 
-    refute status.success?, "enrichment-create with options format but no options should fail"
+    refute status.success?, "enrichment-create with options format but no options should fail. stderr: #{stderr}, stdout: #{stdout}"
     combined = stdout + stderr
     assert_includes combined.downcase, "options"
   end
 
   # Test help output for each command
   def test_enrichment_create_help
-    stdout, _stderr, status = run_command("bundle exec exe/exa-ai enrichment-create --help")
+    stdout, stderr, status = run_command("bundle exec exe/exa-ai enrichment-create --help")
 
-    assert status.success?, "enrichment-create --help should succeed"
+    assert status.success?, "enrichment-create --help should succeed. stderr: #{stderr}, stdout: #{stdout}"
     assert_includes stdout, "Usage:"
     assert_includes stdout, "webset_id"
     assert_includes stdout, "--description"
@@ -428,46 +428,46 @@ def test_enrichment_create_help
   end
 
   def test_enrichment_get_help
-    stdout, _stderr, status = run_command("bundle exec exe/exa-ai enrichment-get --help")
+    stdout, stderr, status = run_command("bundle exec exe/exa-ai enrichment-get --help")
 
-    assert status.success?, "enrichment-get --help should succeed"
+    assert status.success?, "enrichment-get --help should succeed. stderr: #{stderr}, stdout: #{stdout}"
     assert_includes stdout, "Usage:"
     assert_includes stdout, "webset_id"
     assert_includes stdout, "enrichment_id"
   end
 
   def test_enrichment_list_help
-    stdout, _stderr, status = run_command("bundle exec exe/exa-ai enrichment-list --help")
+    stdout, stderr, status = run_command("bundle exec exe/exa-ai enrichment-list --help")
 
-    assert status.success?, "enrichment-list --help should succeed"
+    assert status.success?, "enrichment-list --help should succeed. stderr: #{stderr}, stdout: #{stdout}"
     assert_includes stdout, "Usage:"
     assert_includes stdout, "webset_id"
   end
 
   def test_enrichment_update_help
-    stdout, _stderr, status = run_command("bundle exec exe/exa-ai enrichment-update --help")
+    stdout, stderr, status = run_command("bundle exec exe/exa-ai enrichment-update --help")
 
-    assert status.success?, "enrichment-update --help should succeed"
+    assert status.success?, "enrichment-update --help should succeed. stderr: #{stderr}, stdout: #{stdout}"
     assert_includes stdout, "Usage:"
     assert_includes stdout, "webset_id"
     assert_includes stdout, "enrichment_id"
     assert_includes stdout, "--description"
   end
 
   def test_enrichment_delete_help
-    stdout, _stderr, status = run_command("bundle exec exe/exa-ai enrichment-delete --help")
+    stdout, stderr, status = run_command("bundle exec exe/exa-ai enrichment-delete --help")
 
-    assert status.success?, "enrichment-delete --help should succeed"
+    assert status.success?, "enrichment-delete --help should succeed. stderr: #{stderr}, stdout: #{stdout}"
     assert_includes stdout, "Usage:"
     assert_includes stdout, "webset_id"
     assert_includes stdout, "enrichment_id"
     assert_includes stdout, "--force"
   end
 
   def test_enrichment_cancel_help
-    stdout, _stderr, status = run_command("bundle exec exe/exa-ai enrichment-cancel --help")
+    stdout, stderr, status = run_command("bundle exec exe/exa-ai enrichment-cancel --help")
 
-    assert status.success?, "enrichment-cancel --help should succeed"
+    assert status.success?, "enrichment-cancel --help should succeed. stderr: #{stderr}, stdout: #{stdout}"
     assert_includes stdout, "Usage:"
     assert_includes stdout, "webset_id"
     assert_includes stdout, "enrichment_id"