diff --git a/src/agent.rs b/src/agent.rs index aaa211a..c2cba51 100644 --- a/src/agent.rs +++ b/src/agent.rs @@ -131,6 +131,7 @@ pub struct Agent { pub omit_tools_field_when_empty: bool, pub plan_tool_enforcement: PlanToolEnforcementMode, pub mcp_pin_enforcement: McpPinEnforcementMode, + pub raw_tool_result_dir: Option, pub plan_step_constraints: Vec, pub current_plan: Vec, pub tool_call_budget: ToolCallBudget, diff --git a/src/agent/gate_paths.rs b/src/agent/gate_paths.rs index 5319142..474d5ef 100644 --- a/src/agent/gate_paths.rs +++ b/src/agent/gate_paths.rs @@ -424,6 +424,7 @@ impl Agent

{ false, crate::tools::ToolResultMeta { side_effects: tool_side_effects(&tc.name), + duration_ms: None, bytes: None, exit_code: None, stderr_truncated: None, diff --git a/src/agent/timeouts.rs b/src/agent/timeouts.rs index a145039..80e2103 100644 --- a/src/agent/timeouts.rs +++ b/src/agent/timeouts.rs @@ -48,6 +48,7 @@ impl Agent

{ false, ToolResultMeta { side_effects: tool_side_effects(&tc.name), + duration_ms: Some(timeout_ms), bytes: None, exit_code: None, stderr_truncated: None, diff --git a/src/agent/tool_helpers.rs b/src/agent/tool_helpers.rs index e595209..4670f70 100644 --- a/src/agent/tool_helpers.rs +++ b/src/agent/tool_helpers.rs @@ -1649,6 +1649,29 @@ impl Agent

{ self.apply_update_plan_tool_success(&run_id, step, tc); } self.update_taint_for_tool_result(&run_id, step, tc, &content, messages.len(), taint_state); + let (model_tool_msg, artifact_warnings) = + crate::tools::to_model_observation_message_with_warnings( + &tool_msg, + tc, + &self.tool_rt.workdir, + self.raw_tool_result_dir.as_deref(), + ); + for warning in artifact_warnings { + self.emit_event( + &run_id, + step, + EventKind::Error, + serde_json::json!({ + "source": "tool_result_artifact", + "non_fatal": true, + "code": warning.code, + "warning": warning.message, + "path": warning.path, + "tool_call_id": tc.id, + "name": tc.name + }), + ); + } self.record_allowed_tool_result( &run_id, step, @@ -1682,7 +1705,7 @@ impl Agent

{ failed_repeat_counts, invalid_patch_format_attempts, successful_write_tool_ok_this_step, - tool_msg, + model_tool_msg, messages, observed_tool_decisions, ); diff --git a/src/agent_runtime.rs b/src/agent_runtime.rs index 56db6fb..6fabd01 100644 --- a/src/agent_runtime.rs +++ b/src/agent_runtime.rs @@ -336,6 +336,7 @@ pub(crate) async fn run_agent_with_ui( omit_tools_field_when_empty: false, plan_tool_enforcement: effective_plan_tool_enforcement, mcp_pin_enforcement: args.mcp_pin_enforcement, + raw_tool_result_dir: Some(paths.runs_dir.join(format!("{run_id}.tool-results"))), plan_step_constraints, current_plan: Vec::new(), tool_call_budget: ToolCallBudget { diff --git a/src/agent_tests.rs b/src/agent_tests.rs index 0646868..b70b8b4 100644 --- a/src/agent_tests.rs +++ b/src/agent_tests.rs @@ -193,6 +193,7 @@ fn edit_workflow_system_prompt_matches_write_file_contract() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -340,6 +341,7 @@ async fn compaction_failure_emits_run_end_provider_error() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -460,6 +462,7 @@ async fn non_stream_mode_uses_non_stream_generate() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -556,6 +559,7 @@ async fn task_memory_message_is_injected_into_transcript() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -666,6 +670,7 @@ async fn build_initial_messages_contains_tool_contract_version_marker() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -3377,6 +3382,7 @@ async fn emits_tool_exec_target_before_exec_start() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -3483,6 +3489,7 @@ async fn plan_tool_enforcement_hard_denies_disallowed_tool() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Hard, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: vec![PlanStepConstraint { step_id: "S1".to_string(), intended_tools: vec!["list_dir".to_string()], @@ -3595,6 +3602,7 @@ async fn operator_interrupt_delivers_post_tool_and_cancels_remaining_turn_work() omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -3712,6 +3720,7 @@ async fn operator_next_delivers_at_turn_idle_without_interrupt() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -3823,6 +3832,7 @@ async fn halting_is_blocked_when_plan_steps_are_pending() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Hard, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: vec![PlanStepConstraint { step_id: "S1".to_string(), intended_tools: vec!["read_file".to_string()], @@ -3924,6 +3934,7 @@ async fn emits_step_lifecycle_events_for_pending_plan_halt() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Hard, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: vec![PlanStepConstraint { step_id: "S1".to_string(), intended_tools: vec!["read_file".to_string()], @@ -4031,6 +4042,7 @@ async fn tool_budget_exceeded_returns_deterministic_exit() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget { @@ -4135,6 +4147,7 @@ async fn multiple_tool_calls_in_single_step_fail_with_protocol_violation() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -4236,6 +4249,7 @@ async fn planner_enforced_final_output_uses_user_output_field() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Hard, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: vec![PlanStepConstraint { step_id: "S1".to_string(), intended_tools: vec!["read_file".to_string()], @@ -4348,6 +4362,7 @@ async fn schema_repair_retry_happens_before_execution() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -4466,6 +4481,7 @@ async fn repeated_malformed_tool_calls_fail_fast_with_protocol_violation() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -4602,6 +4618,7 @@ async fn edit_aliases_do_not_trip_malformed_tool_guard() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -4711,6 +4728,7 @@ async fn repeated_failed_unknown_tool_calls_are_blocked_by_repeat_guard() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -4825,6 +4843,7 @@ async fn repeated_invalid_patch_format_fails_fast_with_protocol_violation() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -4967,6 +4986,7 @@ async fn runtime_post_write_verification_allows_finalize_without_model_read_back omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -5137,6 +5157,7 @@ async fn runtime_post_write_missing_closeout_gets_one_bounded_final_answer_turn( omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -5312,6 +5333,7 @@ async fn runtime_pre_tool_plan_text_does_not_count_as_post_tool_closeout() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -5468,6 +5490,7 @@ async fn post_write_known_validation_goes_directly_to_validation_only_phase() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -5600,6 +5623,7 @@ async fn echoed_tool_result_wrapper_is_blocked_before_finalization() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -5730,6 +5754,7 @@ async fn fabricated_tool_result_after_read_gets_path_aware_write_recovery() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -5860,6 +5885,7 @@ async fn echoed_box_wrapper_is_blocked_before_finalization() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -5987,6 +6013,7 @@ async fn runtime_noop_apply_patch_does_not_finalize_ok() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -6136,6 +6163,7 @@ async fn runtime_exact_final_answer_retry_allows_one_bounded_retry() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -6277,6 +6305,7 @@ async fn runtime_exact_final_answer_retry_classifies_noncompliant_output() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -6419,6 +6448,7 @@ async fn runtime_exact_final_answer_requires_successful_validation_command() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -6574,6 +6604,7 @@ async fn runtime_required_validation_guard_allows_one_bounded_shell_retry() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -6732,6 +6763,7 @@ async fn runtime_required_validation_phase_direct_handoff_repairs_empty_turn() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -6897,6 +6929,7 @@ async fn runtime_required_validation_phase_repairs_exact_final_answer_into_shell omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -7057,6 +7090,7 @@ async fn runtime_required_validation_phase_repairs_prose_only_into_shell() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -7215,6 +7249,7 @@ async fn runtime_required_validation_phase_repairs_wrong_tool_into_shell() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -7383,6 +7418,7 @@ async fn runtime_post_validation_phase_blocks_tool_drift_and_recovers_to_final_a omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -7551,6 +7587,7 @@ async fn runtime_failed_validation_blocks_shell_retry_until_code_changes() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -7705,6 +7742,7 @@ async fn runtime_exact_final_answer_allows_matching_successful_validation_comman omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -7853,6 +7891,7 @@ async fn runtime_exact_final_answer_recovers_wrapped_exact_block_after_validatio omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -7992,6 +8031,7 @@ async fn runtime_read_then_done_recovers_with_corrective_write_instruction() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -8150,6 +8190,7 @@ async fn runtime_post_write_guard_retry_is_machine_classified() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -8334,6 +8375,7 @@ async fn runtime_post_write_verification_timeout_fails_deterministically() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget { @@ -8493,6 +8535,7 @@ async fn runtime_tool_execution_timeout_is_bounded() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget { @@ -8611,6 +8654,7 @@ async fn invalid_patch_format_attempts_are_scoped_per_tool_key() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -8727,6 +8771,7 @@ async fn tool_only_prompt_repairs_once_then_allows_tool_call() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -8871,6 +8916,7 @@ async fn repeated_failed_str_replace_forces_pivot_before_repeat_block() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -9038,6 +9084,7 @@ async fn repeated_failed_apply_patch_forces_smaller_fix_pivot_before_repeat_bloc omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -9167,6 +9214,7 @@ async fn tool_only_prompt_repeated_prose_fails_fast() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), @@ -9273,6 +9321,7 @@ async fn invalid_done_transition_fails_with_planner_error() { omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Hard, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: vec![ PlanStepConstraint { step_id: "S1".to_string(), diff --git a/src/agent_tool_exec.rs b/src/agent_tool_exec.rs index 52b7ab3..c27dd0e 100644 --- a/src/agent_tool_exec.rs +++ b/src/agent_tool_exec.rs @@ -71,6 +71,7 @@ pub(crate) async fn run_tool_once( false, ToolResultMeta { side_effects: tool_side_effects(&tc.name), + duration_ms: None, bytes: None, exit_code: None, stderr_truncated: None, @@ -95,6 +96,7 @@ pub(crate) async fn run_tool_once( false, ToolResultMeta { side_effects: tool_side_effects(&tc.name), + duration_ms: None, bytes: None, exit_code: None, stderr_truncated: None, diff --git a/src/cli_args.rs b/src/cli_args.rs index a6d4ed1..3640b62 100644 --- a/src/cli_args.rs +++ b/src/cli_args.rs @@ -699,6 +699,25 @@ pub(crate) struct DenyArgs { #[derive(Debug, Subcommand)] pub(crate) enum ReplaySubcommand { + Artifacts { + run_id: String, + + #[arg(long, default_value_t = false)] + json: bool, + }, + + Artifact { + run_id: String, + + artifact_ref: String, + + #[arg(long, default_value_t = false)] + path_only: bool, + + #[arg(long, default_value_t = false)] + json: bool, + }, + Verify { run_id: String, diff --git a/src/cli_dispatch_eval_replay.rs b/src/cli_dispatch_eval_replay.rs index 5c0b0f4..cbdb823 100644 --- a/src/cli_dispatch_eval_replay.rs +++ b/src/cli_dispatch_eval_replay.rs @@ -34,19 +34,40 @@ pub(crate) async fn handle_replay_command( paths: &store::StatePaths, ) -> anyhow::Result<()> { match &args.command { + Some(ReplaySubcommand::Artifacts { run_id, json }) => { + let record = load_replay_run_record(paths, run_id)?; + let list = store::list_replay_artifacts(&paths.state_dir, &record)?; + if *json { + println!("{}", serde_json::to_string_pretty(&list)?); + } else { + print!("{}", store::render_replay_artifact_list(&list)); + } + Ok(()) + } + Some(ReplaySubcommand::Artifact { + run_id, + artifact_ref, + path_only, + json, + }) => { + let record = load_replay_run_record(paths, run_id)?; + let inspection = + store::inspect_replay_artifact(&paths.state_dir, &record, artifact_ref)?; + if *path_only { + println!("{}", inspection.summary.resolved_path); + } else if *json { + println!("{}", serde_json::to_string_pretty(&inspection)?); + } else { + print!("{}", store::render_replay_artifact_inspection(&inspection)); + } + Ok(()) + } Some(ReplaySubcommand::Verify { run_id, strict, json, }) => { - let record = store::load_run_record(&paths.state_dir, run_id).map_err(|e| { - anyhow!( - "failed to load run '{}': {}. runs dir: {}", - run_id, - e, - paths.runs_dir.display() - ) - })?; + let record = load_replay_run_record(paths, run_id)?; let report = verify_run_record(&record, *strict)?; @@ -95,6 +116,20 @@ pub(crate) async fn handle_replay_command( } } +fn load_replay_run_record( + paths: &store::StatePaths, + run_id: &str, +) -> anyhow::Result { + store::load_run_record(&paths.state_dir, run_id).map_err(|e| { + anyhow!( + "failed to load run '{}': {}. runs dir: {}", + run_id, + e, + paths.runs_dir.display() + ) + }) +} + async fn resume_from_runtime_checkpoint( checkpoint: &store::RuntimeRunCheckpointRecordV1, paths: &store::StatePaths, diff --git a/src/eval/runner_runtime.rs b/src/eval/runner_runtime.rs index 181fc64..7a2c866 100644 --- a/src/eval/runner_runtime.rs +++ b/src/eval/runner_runtime.rs @@ -513,6 +513,7 @@ pub(crate) async fn run_single( omit_tools_field_when_empty: false, plan_tool_enforcement: crate::agent::PlanToolEnforcementMode::Off, mcp_pin_enforcement: crate::agent::McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget { diff --git a/src/mcp/registry.rs b/src/mcp/registry.rs index ddaf031..9535fff 100644 --- a/src/mcp/registry.rs +++ b/src/mcp/registry.rs @@ -273,6 +273,7 @@ impl McpRegistry { false, ToolResultMeta { side_effects: tool_side_effects(&tc.name), + duration_ms: None, bytes: None, exit_code: None, stderr_truncated: None, @@ -326,6 +327,7 @@ impl McpRegistry { false, ToolResultMeta { side_effects: tool_side_effects(&tc.name), + duration_ms: Some(meta.elapsed_ms), bytes: None, exit_code: None, stderr_truncated: None, @@ -357,6 +359,7 @@ impl McpRegistry { was_truncated, ToolResultMeta { side_effects: tool_side_effects(&tc.name), + duration_ms: Some(meta.elapsed_ms), bytes: Some(result_str.len() as u64), exit_code: None, stderr_truncated: None, diff --git a/src/store.rs b/src/store.rs index 4a288af..1d29365 100644 --- a/src/store.rs +++ b/src/store.rs @@ -2,6 +2,7 @@ use std::path::{Path, PathBuf}; use crate::trust::policy::McpAllowSummary; +mod artifact_inspection; mod hash; mod io; mod render; @@ -12,6 +13,12 @@ pub use crate::agent_runtime::state::{ InterruptKindV1, PhaseSummaryEntryV1, RetryState, RunCheckpointV1 as RuntimeStateCheckpointV1, RunPhase, ValidationState, }; +#[allow(unused_imports)] +pub use artifact_inspection::{ + inspect_replay_artifact, list_replay_artifacts, render_replay_artifact_inspection, + render_replay_artifact_list, ReplayArtifactInspection, ReplayArtifactInspectionError, + ReplayArtifactList, ReplayArtifactRefSummary, +}; pub use hash::{ cli_trust_mode, config_hash_hex, hash_tool_schema, mcp_tool_snapshot_hash_hex, provider_to_string, sha256_hex, stable_path_string, tool_schema_hash_hex_map, diff --git a/src/store/artifact_inspection.rs b/src/store/artifact_inspection.rs new file mode 100644 index 0000000..ef5a3c7 --- /dev/null +++ b/src/store/artifact_inspection.rs @@ -0,0 +1,893 @@ +use std::fmt; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::tools::{ + load_tool_result_artifact_manifest, resolve_tool_result_artifact_ref, ToolResultContentRef, +}; +use crate::types::Role; + +use super::RunRecord; + +const OBSERVATION_SCHEMA_VERSION: &str = "openagent.observation.v1"; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ReplayArtifactRefSummary { + pub run_id: String, + pub tool_call_id: String, + pub tool_name: String, + pub label: String, + #[serde(rename = "ref")] + pub artifact_ref: ToolResultContentRef, + pub ref_value: String, + pub size_bytes: Option, + pub sha16: Option, + pub sha256: Option, + pub relative_path: Option, + pub resolved_path: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ReplayArtifactList { + pub run_id: String, + pub artifact_dir: String, + pub artifacts: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ReplayArtifactInspection { + pub summary: ReplayArtifactRefSummary, + pub content: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ReplayArtifactInspectionError { + MissingManifest { + path: String, + }, + UnknownRef { + artifact_ref: String, + }, + UnsafeRef { + artifact_ref: String, + reason: String, + }, + InvalidRefKind { + kind: String, + }, + MalformedRef { + artifact_ref: String, + error: String, + }, + MissingArtifactFile { + path: String, + }, + Io { + path: String, + error: String, + }, +} + +impl fmt::Display for ReplayArtifactInspectionError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::MissingManifest { path } => { + write!(f, "missing tool-result artifact manifest: {path}") + } + Self::UnknownRef { artifact_ref } => { + write!(f, "unknown tool-result artifact ref: {artifact_ref}") + } + Self::UnsafeRef { + artifact_ref, + reason, + } => write!( + f, + "unsafe tool-result artifact ref '{artifact_ref}': {reason}" + ), + Self::InvalidRefKind { kind } => { + write!(f, "invalid tool-result artifact ref kind: {kind}") + } + Self::MalformedRef { + artifact_ref, + error, + } => write!( + f, + "malformed tool-result artifact ref '{artifact_ref}': {error}" + ), + Self::MissingArtifactFile { path } => { + write!(f, "tool-result artifact file missing: {path}") + } + Self::Io { path, error } => write!(f, "failed to read artifact {path}: {error}"), + } + } +} + +impl std::error::Error for ReplayArtifactInspectionError {} + +fn run_tool_result_artifact_dir(state_dir: &Path, run_id: &str) -> PathBuf { + state_dir + .join("runs") + .join(format!("{run_id}.tool-results")) +} + +pub fn list_replay_artifacts( + state_dir: &Path, + record: &RunRecord, +) -> Result { + let artifact_dir = run_tool_result_artifact_dir(state_dir, &record.metadata.run_id); + let extracted = extract_observation_artifact_refs(record); + if extracted.is_empty() { + return Ok(ReplayArtifactList { + run_id: record.metadata.run_id.clone(), + artifact_dir: artifact_dir.display().to_string(), + artifacts: Vec::new(), + }); + } + + let manifest_path = artifact_dir.join("manifest.json"); + if !manifest_path.exists() { + return Err(ReplayArtifactInspectionError::MissingManifest { + path: manifest_path.display().to_string(), + }); + } + let manifest = load_tool_result_artifact_manifest(&artifact_dir).map_err(|err| { + ReplayArtifactInspectionError::Io { + path: manifest_path.display().to_string(), + error: err.to_string(), + } + })?; + + let mut artifacts = Vec::new(); + for extracted_ref in extracted { + let entry = manifest + .artifacts + .iter() + .find(|entry| entry.artifact_ref == extracted_ref.artifact_ref) + .ok_or_else(|| ReplayArtifactInspectionError::UnknownRef { + artifact_ref: extracted_ref.artifact_ref.path.clone(), + })?; + let resolved = resolve_ref_for_replay(&artifact_dir, &extracted_ref.artifact_ref) + .map_err(|err| map_resolve_error(&extracted_ref.artifact_ref.path, err))?; + ensure_artifact_file_exists(&resolved)?; + artifacts.push(ReplayArtifactRefSummary { + run_id: record.metadata.run_id.clone(), + tool_call_id: extracted_ref.tool_call_id, + tool_name: extracted_ref.tool_name, + label: extracted_ref.label, + ref_value: extracted_ref.artifact_ref.path.clone(), + artifact_ref: extracted_ref.artifact_ref, + size_bytes: Some(entry.size_bytes), + sha16: Some(entry.sha16.clone()), + sha256: Some(entry.sha256.clone()), + relative_path: Some(entry.path.clone()), + resolved_path: resolved.display().to_string(), + }); + } + + Ok(ReplayArtifactList { + run_id: record.metadata.run_id.clone(), + artifact_dir: artifact_dir.display().to_string(), + artifacts, + }) +} + +pub fn inspect_replay_artifact( + state_dir: &Path, + record: &RunRecord, + artifact_ref_arg: &str, +) -> Result { + let artifact_dir = run_tool_result_artifact_dir(state_dir, &record.metadata.run_id); + if let Some(parsed_ref) = parse_ref_object(artifact_ref_arg)? { + let summary = summary_for_explicit_ref(state_dir, record, &artifact_dir, &parsed_ref)?; + let content = read_artifact_content(Path::new(&summary.resolved_path))?; + return Ok(ReplayArtifactInspection { summary, content }); + } + + let list = list_replay_artifacts(state_dir, record)?; + if let Some(summary) = list + .artifacts + .into_iter() + .find(|artifact| artifact_matches_arg(artifact, artifact_ref_arg)) + { + let content = read_artifact_content(Path::new(&summary.resolved_path))?; + return Ok(ReplayArtifactInspection { summary, content }); + } + + let unresolved_probe = ToolResultContentRef { + kind: "run_artifact_path".to_string(), + path: artifact_ref_arg.to_string(), + sha256: String::new(), + bytes: 0, + }; + match resolve_ref_for_replay(&artifact_dir, &unresolved_probe) { + Ok(path) => { + if path.exists() { + Err(ReplayArtifactInspectionError::UnknownRef { + artifact_ref: artifact_ref_arg.to_string(), + }) + } else { + Err(ReplayArtifactInspectionError::MissingArtifactFile { + path: path.display().to_string(), + }) + } + } + Err(err) => Err(map_resolve_error(artifact_ref_arg, err)), + } +} + +pub fn render_replay_artifact_list(list: &ReplayArtifactList) -> String { + let mut out = String::new(); + out.push_str(&format!("run_id: {}\n", list.run_id)); + out.push_str(&format!("artifact_dir: {}\n", list.artifact_dir)); + out.push_str(&format!("artifacts: {}\n", list.artifacts.len())); + for artifact in &list.artifacts { + out.push_str(&format!( + "- tool_call_id={} tool={} label={} size_bytes={} sha16={} sha256={} ref={} path={}\n", + artifact.tool_call_id, + artifact.tool_name, + artifact.label, + artifact + .size_bytes + .map(|v| v.to_string()) + .unwrap_or_else(|| "-".to_string()), + artifact.sha16.as_deref().unwrap_or("-"), + artifact.sha256.as_deref().unwrap_or("-"), + artifact.ref_value, + artifact.resolved_path + )); + } + out +} + +pub fn render_replay_artifact_inspection(inspection: &ReplayArtifactInspection) -> String { + let artifact = &inspection.summary; + let mut out = String::new(); + out.push_str("artifact:\n"); + out.push_str(&format!(" tool_call_id: {}\n", artifact.tool_call_id)); + out.push_str(&format!(" tool: {}\n", artifact.tool_name)); + out.push_str(&format!(" label: {}\n", artifact.label)); + out.push_str(&format!(" ref: {}\n", artifact.ref_value)); + out.push_str(&format!( + " size_bytes: {}\n", + artifact + .size_bytes + .map(|v| v.to_string()) + .unwrap_or_else(|| "-".to_string()) + )); + out.push_str(&format!( + " sha16: {}\n", + artifact.sha16.as_deref().unwrap_or("-") + )); + out.push_str(&format!( + " sha256: {}\n", + artifact.sha256.as_deref().unwrap_or("-") + )); + out.push_str(&format!(" path: {}\n", artifact.resolved_path)); + out.push_str("content:\n"); + out.push_str(&inspection.content); + if !inspection.content.ends_with('\n') { + out.push('\n'); + } + out +} + +#[derive(Debug, Clone)] +struct ExtractedArtifactRef { + tool_call_id: String, + tool_name: String, + label: String, + artifact_ref: ToolResultContentRef, +} + +fn extract_observation_artifact_refs(record: &RunRecord) -> Vec { + let mut out = Vec::new(); + for message in &record.transcript { + if !matches!(message.role, Role::Tool) { + continue; + } + let Some(content) = message.content.as_deref() else { + continue; + }; + let Ok(envelope) = serde_json::from_str::(content) else { + continue; + }; + let Some(observation) = envelope.get("observation").and_then(Value::as_object) else { + continue; + }; + if observation.get("schema_version").and_then(Value::as_str) + != Some(OBSERVATION_SCHEMA_VERSION) + { + continue; + } + let tool_call_id = message + .tool_call_id + .clone() + .or_else(|| { + envelope + .get("tool_call_id") + .and_then(Value::as_str) + .map(ToOwned::to_owned) + }) + .unwrap_or_else(|| "unknown".to_string()); + let tool_name = message + .tool_name + .clone() + .or_else(|| { + observation + .get("tool_name") + .and_then(Value::as_str) + .map(ToOwned::to_owned) + }) + .or_else(|| { + envelope + .get("tool_name") + .and_then(Value::as_str) + .map(ToOwned::to_owned) + }) + .unwrap_or_else(|| "unknown".to_string()); + for (label, key) in [ + ("raw_result", "raw_result_ref"), + ("stdout", "stdout_ref"), + ("stderr", "stderr_ref"), + ] { + let Some(value) = observation.get(key) else { + continue; + }; + if let Ok(artifact_ref) = serde_json::from_value::(value.clone()) + { + out.push(ExtractedArtifactRef { + tool_call_id: tool_call_id.clone(), + tool_name: tool_name.clone(), + label: label.to_string(), + artifact_ref, + }); + } + } + } + out +} + +fn parse_ref_object( + raw: &str, +) -> Result, ReplayArtifactInspectionError> { + if !raw.trim_start().starts_with('{') { + return Ok(None); + } + serde_json::from_str::(raw) + .map(Some) + .map_err(|err| ReplayArtifactInspectionError::MalformedRef { + artifact_ref: raw.to_string(), + error: err.to_string(), + }) +} + +fn summary_for_explicit_ref( + state_dir: &Path, + record: &RunRecord, + artifact_dir: &Path, + artifact_ref: &ToolResultContentRef, +) -> Result { + let resolved = resolve_ref_for_replay(artifact_dir, artifact_ref) + .map_err(|err| map_resolve_error(&artifact_ref.path, err))?; + ensure_artifact_file_exists(&resolved)?; + let list = list_replay_artifacts(state_dir, record)?; + list.artifacts + .into_iter() + .find(|summary| summary.artifact_ref == *artifact_ref) + .ok_or_else(|| ReplayArtifactInspectionError::UnknownRef { + artifact_ref: artifact_ref.path.clone(), + }) +} + +fn artifact_matches_arg(artifact: &ReplayArtifactRefSummary, raw: &str) -> bool { + artifact.ref_value == raw + || artifact.relative_path.as_deref() == Some(raw) + || artifact.sha16.as_deref() == Some(raw) + || artifact.sha256.as_deref() == Some(raw) +} + +fn resolve_ref_for_replay( + artifact_dir: &Path, + artifact_ref: &ToolResultContentRef, +) -> Result { + resolve_tool_result_artifact_ref(artifact_dir, artifact_ref) +} + +fn ensure_artifact_file_exists(path: &Path) -> Result<(), ReplayArtifactInspectionError> { + if path.is_file() { + Ok(()) + } else { + Err(ReplayArtifactInspectionError::MissingArtifactFile { + path: path.display().to_string(), + }) + } +} + +fn read_artifact_content(path: &Path) -> Result { + ensure_artifact_file_exists(path)?; + std::fs::read_to_string(path).map_err(|err| ReplayArtifactInspectionError::Io { + path: path.display().to_string(), + error: err.to_string(), + }) +} + +fn map_resolve_error( + artifact_ref: &str, + err: crate::tools::ToolResultArtifactResolveError, +) -> ReplayArtifactInspectionError { + match err { + crate::tools::ToolResultArtifactResolveError::InvalidRefKind(kind) => { + ReplayArtifactInspectionError::InvalidRefKind { kind } + } + crate::tools::ToolResultArtifactResolveError::PathTraversal(path) => { + ReplayArtifactInspectionError::UnsafeRef { + artifact_ref: path, + reason: "path traversal is not allowed".to_string(), + } + } + crate::tools::ToolResultArtifactResolveError::UnknownRef => { + ReplayArtifactInspectionError::UnknownRef { + artifact_ref: artifact_ref.to_string(), + } + } + crate::tools::ToolResultArtifactResolveError::InvalidManifestPath(path) => { + ReplayArtifactInspectionError::UnsafeRef { + artifact_ref: path, + reason: "manifest path is outside the artifact directory".to_string(), + } + } + crate::tools::ToolResultArtifactResolveError::Io(error) + | crate::tools::ToolResultArtifactResolveError::Parse(error) => { + ReplayArtifactInspectionError::Io { + path: artifact_ref.to_string(), + error, + } + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use serde_json::json; + use tempfile::tempdir; + + use super::{ + inspect_replay_artifact, list_replay_artifacts, render_replay_artifact_inspection, + render_replay_artifact_list, ReplayArtifactInspectionError, + }; + use crate::agent::{AgentExitReason, AgentOutcome}; + use crate::compaction::{CompactionMode, CompactionSettings, ToolResultPersist}; + use crate::store::{ + load_run_record, resolve_state_paths, write_run_record, PolicyRecordInfo, RunCliConfig, + }; + use crate::tools::{ + envelope_to_message, to_model_observation_message, to_tool_result_envelope_with_error, + ToolErrorCode, ToolErrorDetail, ToolResultMeta, + }; + use crate::types::{Message, Role, SideEffects, ToolCall}; + + fn test_cli_config() -> RunCliConfig { + RunCliConfig { + mode: "single".to_string(), + agent_mode: "build".to_string(), + output_mode: "human".to_string(), + provider: "mock".to_string(), + base_url: "mock://local".to_string(), + model: "mock-model".to_string(), + temperature: None, + top_p: None, + max_tokens: None, + seed: None, + planner_model: None, + worker_model: None, + planner_max_steps: None, + planner_output: None, + planner_strict: None, + enforce_plan_tools: "off".to_string(), + mcp_pin_enforcement: "hard".to_string(), + trust_mode: "off".to_string(), + allow_shell: false, + allow_write: false, + enable_write_tools: false, + exec_target: "host".to_string(), + docker_image: None, + docker_workdir: None, + docker_network: None, + docker_user: None, + docker_config_summary: None, + max_tool_output_bytes: 0, + max_read_bytes: 0, + max_wall_time_ms: 0, + max_total_tool_calls: 0, + max_mcp_calls: 0, + max_filesystem_read_calls: 0, + max_filesystem_write_calls: 0, + max_shell_calls: 0, + max_network_calls: 0, + max_browser_calls: 0, + tool_exec_timeout_ms: 0, + post_write_verify_timeout_ms: 0, + approval_mode: "interrupt".to_string(), + auto_approve_scope: "run".to_string(), + approval_key: "v1".to_string(), + unsafe_mode: false, + no_limits: false, + unsafe_bypass_allow_flags: false, + stream: false, + events_path: None, + max_context_chars: 0, + compaction_mode: "off".to_string(), + compaction_keep_last: 0, + tool_result_persist: "digest".to_string(), + hooks_mode: "off".to_string(), + caps_mode: "off".to_string(), + hooks_config_path: String::new(), + hooks_strict: false, + hooks_timeout_ms: 0, + hooks_max_stdout_bytes: 0, + tool_args_strict: "on".to_string(), + taint: "off".to_string(), + taint_mode: "propagate".to_string(), + taint_digest_bytes: 4096, + repro: "off".to_string(), + repro_env: "safe".to_string(), + repro_out: None, + use_session_settings: false, + resolved_settings_source: BTreeMap::new(), + tui_enabled: false, + tui_refresh_ms: 0, + tui_max_log_lines: 0, + http_max_retries: 0, + http_timeout_ms: 0, + http_connect_timeout_ms: 0, + http_stream_idle_timeout_ms: 0, + http_max_response_bytes: 0, + http_max_line_bytes: 0, + tool_catalog: Vec::new(), + mcp_tool_snapshot: Vec::new(), + mcp_tool_catalog_hash_hex: None, + mcp_servers: Vec::new(), + mcp_config_path: None, + policy_version: None, + includes_resolved: Vec::new(), + mcp_allowlist: None, + instructions_config_path: None, + instructions_config_hash_hex: None, + instruction_model_profile: None, + instruction_task_profile: None, + instruction_task_profile_task_kind: None, + instruction_message_count: 0, + project_guidance_hash_hex: None, + project_guidance_sources: Vec::new(), + project_guidance_truncated: false, + project_guidance_bytes_loaded: 0, + project_guidance_bytes_kept: 0, + repo_map_hash_hex: None, + repo_map_format: None, + repo_map_truncated: false, + repo_map_truncated_reason: None, + repo_map_bytes_scanned: 0, + repo_map_bytes_kept: 0, + repo_map_file_count_included: 0, + repo_map_injected: false, + repo_map_likely_target_files_count: 0, + lsp_context_provider: None, + lsp_context_schema_version: None, + lsp_context_truncated: false, + lsp_context_truncation_reason: None, + lsp_context_bytes_kept: 0, + lsp_context_diagnostics_included: 0, + lsp_context_symbol_query: None, + lsp_context_symbols_included: 0, + lsp_context_definitions_included: 0, + lsp_context_references_included: 0, + lsp_context_injected: false, + lsp_context_likely_target_files_count: 0, + active_profile: None, + profile_source: None, + profile_hash_hex: None, + activated_packs: Vec::new(), + } + } + + fn write_record_with_messages( + run_id: &str, + messages: Vec, + ) -> (tempfile::TempDir, crate::store::StatePaths) { + let tmp = tempdir().expect("tempdir"); + let paths = resolve_state_paths(tmp.path(), None, None, None, None); + let outcome = AgentOutcome { + run_id: run_id.to_string(), + started_at: "2026-01-01T00:00:00Z".to_string(), + finished_at: "2026-01-01T00:00:01Z".to_string(), + exit_reason: AgentExitReason::Ok, + final_output: "done".to_string(), + error: None, + messages, + tool_calls: Vec::new(), + tool_decisions: Vec::new(), + compaction_settings: CompactionSettings { + max_context_chars: 0, + mode: CompactionMode::Off, + keep_last: 20, + tool_result_persist: ToolResultPersist::Digest, + }, + final_prompt_size_chars: 0, + compaction_report: None, + hook_invocations: Vec::new(), + provider_retry_count: 0, + provider_error_count: 0, + token_usage: None, + taint: None, + }; + write_run_record( + &paths, + test_cli_config(), + PolicyRecordInfo { + source: "none".to_string(), + hash_hex: None, + version: None, + includes_resolved: Vec::new(), + mcp_allowlist: None, + }, + "config".to_string(), + &outcome, + crate::planner::RunMode::Single, + None, + None, + BTreeMap::new(), + None, + None, + None, + Vec::new(), + Vec::new(), + None, + None, + None, + Vec::new(), + Vec::new(), + Vec::new(), + None, + None, + Vec::new(), + None, + ) + .expect("write run record"); + (tmp, paths) + } + + fn persisted_record_with_artifacts() -> (tempfile::TempDir, crate::store::StatePaths, String) { + let tmp = tempdir().expect("tempdir"); + let paths = resolve_state_paths(tmp.path(), None, None, None, None); + let run_id = "run_artifacts"; + let tc = ToolCall { + id: "tc_shell".to_string(), + name: "shell".to_string(), + arguments: json!({"command":"cargo test"}), + }; + let raw_inner = json!({ + "status": 1, + "stdout": format!("{}\nstdout final", "stdout noise\n".repeat(500)), + "stderr": format!("{}\nstderr final", "stderr noise\n".repeat(500)), + "stdout_truncated": false, + "stderr_truncated": false + }) + .to_string(); + let raw_msg = envelope_to_message(to_tool_result_envelope_with_error( + &tc, + "builtin", + false, + raw_inner, + false, + Some(ToolErrorDetail { + code: ToolErrorCode::ShellExecNonZeroExit, + message: "Shell command exited with non-zero status: 1.".to_string(), + expected_schema: None, + received_args: None, + minimal_example: None, + available_tools: None, + }), + ToolResultMeta { + side_effects: SideEffects::ShellExec, + duration_ms: Some(10), + bytes: None, + exit_code: Some(1), + stderr_truncated: Some(false), + stdout_truncated: Some(false), + source: "builtin".to_string(), + execution_target: "host".to_string(), + warnings: None, + warnings_max: None, + warnings_truncated: None, + docker: None, + }, + )); + let artifact_dir = paths.runs_dir.join(format!("{run_id}.tool-results")); + let compact = + to_model_observation_message(&raw_msg, &tc, tmp.path(), Some(artifact_dir.as_path())); + let outcome = AgentOutcome { + run_id: run_id.to_string(), + started_at: "2026-01-01T00:00:00Z".to_string(), + finished_at: "2026-01-01T00:00:01Z".to_string(), + exit_reason: AgentExitReason::Ok, + final_output: "done".to_string(), + error: None, + messages: vec![compact], + tool_calls: vec![tc], + tool_decisions: Vec::new(), + compaction_settings: CompactionSettings { + max_context_chars: 0, + mode: CompactionMode::Off, + keep_last: 20, + tool_result_persist: ToolResultPersist::Digest, + }, + final_prompt_size_chars: 0, + compaction_report: None, + hook_invocations: Vec::new(), + provider_retry_count: 0, + provider_error_count: 0, + token_usage: None, + taint: None, + }; + write_run_record( + &paths, + test_cli_config(), + PolicyRecordInfo { + source: "none".to_string(), + hash_hex: None, + version: None, + includes_resolved: Vec::new(), + mcp_allowlist: None, + }, + "config".to_string(), + &outcome, + crate::planner::RunMode::Single, + None, + None, + BTreeMap::new(), + None, + None, + None, + Vec::new(), + Vec::new(), + None, + None, + None, + Vec::new(), + Vec::new(), + Vec::new(), + None, + None, + Vec::new(), + None, + ) + .expect("write run record"); + (tmp, paths, run_id.to_string()) + } + + #[test] + fn replay_artifacts_discovers_refs_from_persisted_run_record() { + let (_tmp, paths, run_id) = persisted_record_with_artifacts(); + let record = load_run_record(&paths.state_dir, &run_id).expect("load record"); + let list = list_replay_artifacts(&paths.state_dir, &record).expect("list"); + assert_eq!(list.artifacts.len(), 3); + let labels = list + .artifacts + .iter() + .map(|a| a.label.as_str()) + .collect::>(); + assert_eq!(labels, vec!["raw_result", "stdout", "stderr"]); + let text = render_replay_artifact_list(&list); + assert!(text.contains("tool_call_id=tc_shell")); + assert!(text.contains("label=raw_result")); + assert!(text.contains("size_bytes=")); + assert!(text.contains("sha16=")); + assert!(text.contains("sha256=")); + assert!(text.contains("ref=")); + assert!(serde_json::to_string_pretty(&list) + .expect("json") + .contains("\"label\": \"stdout\"")); + } + + #[test] + fn replay_artifact_resolves_valid_ref_and_reads_content() { + let (_tmp, paths, run_id) = persisted_record_with_artifacts(); + let record = load_run_record(&paths.state_dir, &run_id).expect("load record"); + let list = list_replay_artifacts(&paths.state_dir, &record).expect("list"); + let raw = list + .artifacts + .iter() + .find(|a| a.label == "raw_result") + .expect("raw"); + let inspection = + inspect_replay_artifact(&paths.state_dir, &record, &raw.ref_value).expect("inspect"); + assert_eq!(inspection.summary.label, "raw_result"); + assert!(inspection.content.contains("openagent.tool_result.v1")); + assert_eq!( + inspection.summary.resolved_path, + std::fs::canonicalize(&inspection.summary.resolved_path) + .expect("canonical") + .display() + .to_string() + ); + let rendered = render_replay_artifact_inspection(&inspection); + assert!(rendered.contains("content:\n")); + assert!(rendered.contains("label: raw_result")); + } + + #[test] + fn replay_artifact_path_only_value_is_resolved_safe_path() { + let (_tmp, paths, run_id) = persisted_record_with_artifacts(); + let record = load_run_record(&paths.state_dir, &run_id).expect("load record"); + let list = list_replay_artifacts(&paths.state_dir, &record).expect("list"); + let stdout = list + .artifacts + .iter() + .find(|a| a.label == "stdout") + .expect("stdout"); + let inspection = + inspect_replay_artifact(&paths.state_dir, &record, &stdout.ref_value).expect("inspect"); + assert_eq!(inspection.summary.resolved_path, stdout.resolved_path); + assert!(inspection.summary.resolved_path.ends_with(".txt")); + } + + #[test] + fn replay_artifact_unknown_and_traversal_refs_fail_clearly() { + let (_tmp, paths, run_id) = persisted_record_with_artifacts(); + let record = load_run_record(&paths.state_dir, &run_id).expect("load record"); + let unknown = + inspect_replay_artifact(&paths.state_dir, &record, "not-a-known-ref").unwrap_err(); + assert!(matches!( + unknown, + ReplayArtifactInspectionError::UnknownRef { .. } + )); + + let traversal = inspect_replay_artifact( + &paths.state_dir, + &record, + r#"{"kind":"run_artifact_path","path":"../outside.json","sha256":"","bytes":0}"#, + ) + .unwrap_err(); + assert!(matches!( + traversal, + ReplayArtifactInspectionError::UnsafeRef { .. } + )); + } + + #[test] + fn replay_artifacts_missing_manifest_fails_clearly() { + let (_tmp, paths, run_id) = persisted_record_with_artifacts(); + let manifest = paths + .runs_dir + .join(format!("{run_id}.tool-results")) + .join("manifest.json"); + std::fs::remove_file(&manifest).expect("remove manifest"); + let record = load_run_record(&paths.state_dir, &run_id).expect("load record"); + let err = list_replay_artifacts(&paths.state_dir, &record).unwrap_err(); + assert!(matches!( + err, + ReplayArtifactInspectionError::MissingManifest { .. } + )); + } + + #[test] + fn replay_artifacts_old_run_without_observations_lists_zero() { + let (_tmp, paths) = write_record_with_messages( + "old_run", + vec![Message { + role: Role::Tool, + content: Some("plain old tool result".to_string()), + tool_call_id: Some("tc_old".to_string()), + tool_name: Some("shell".to_string()), + tool_calls: None, + }], + ); + let record = load_run_record(&paths.state_dir, "old_run").expect("load record"); + let list = list_replay_artifacts(&paths.state_dir, &record).expect("list"); + assert!(list.artifacts.is_empty()); + } +} diff --git a/src/tools.rs b/src/tools.rs index d04c5e2..6709a0f 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -15,6 +15,7 @@ mod exec_plan; mod exec_shell; mod exec_support; mod exec_write; +mod observation; mod schema; pub(crate) use catalog::normalize_builtin_tool_args; @@ -26,6 +27,15 @@ pub use envelope::{ pub(crate) use exec_plan::parse_update_plan_args; pub use exec_plan::{PlanItem, PlanStatus}; use exec_support::ToolExecution; +#[cfg(test)] +pub(crate) use observation::to_model_observation_message; +pub(crate) use observation::to_model_observation_message_with_warnings; +#[allow(unused_imports)] +pub use observation::{ + load_tool_result_artifact_manifest, resolve_tool_result_artifact_ref, + ToolResultArtifactManifestEntryV1, ToolResultArtifactManifestV1, + ToolResultArtifactResolveError, +}; pub use schema::{ compact_builtin_schema, invalid_args_detail, minimal_builtin_example, sorted_builtin_tool_names, validate_builtin_tool_args, validate_schema_args, @@ -61,6 +71,8 @@ pub struct ToolRuntime { pub struct ToolResultMeta { pub side_effects: SideEffects, #[serde(skip_serializing_if = "Option::is_none")] + pub duration_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub bytes: Option, #[serde(skip_serializing_if = "Option::is_none")] pub exit_code: Option, @@ -88,7 +100,7 @@ pub struct ToolWarningDetail { pub reason: String, } -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct ToolResultContentRef { pub kind: String, pub path: String, @@ -183,10 +195,11 @@ pub async fn execute_tool_streaming( tc: &ToolCall, shell_stream: Option, ) -> Message { + let started = std::time::Instant::now(); let normalized_args = catalog::normalize_builtin_tool_args(&tc.name, &tc.arguments); let side_effects = tool_side_effects(&tc.name); if let Err(e) = validate_builtin_tool_args(&tc.name, &normalized_args, rt.tool_args_strict) { - return invalid_args_tool_message( + let mut message = invalid_args_tool_message( tc, "builtin", &e, @@ -195,8 +208,10 @@ pub async fn execute_tool_streaming( ExecTargetKind::Docker => "docker".to_string(), }, ); + observation::set_tool_result_duration(&mut message, started.elapsed().as_millis() as u64); + return message; } - let exec = match tc.name.as_str() { + let mut exec = match tc.name.as_str() { "list_dir" => exec_fs::run_list_dir(rt, &normalized_args).await, "read_file" => exec_fs::run_read_file(rt, &normalized_args).await, "glob" => exec_fs::run_glob(rt, &normalized_args).await, @@ -225,6 +240,7 @@ pub async fn execute_tool_streaming( }), meta: ToolResultMeta { side_effects, + duration_ms: None, bytes: None, exit_code: None, stderr_truncated: None, @@ -241,6 +257,7 @@ pub async fn execute_tool_streaming( }, }, }; + exec.meta.duration_ms = Some(started.elapsed().as_millis() as u64); envelope_to_message(to_tool_result_envelope_with_error( tc, "builtin", diff --git a/src/tools/envelope.rs b/src/tools/envelope.rs index aa1a81b..ceb53f4 100644 --- a/src/tools/envelope.rs +++ b/src/tools/envelope.rs @@ -66,6 +66,7 @@ pub fn invalid_args_tool_message( Some(invalid_args_detail(&tc.name, &tc.arguments, err)), ToolResultMeta { side_effects: tool_side_effects(&tc.name), + duration_ms: None, bytes: None, exit_code: None, stderr_truncated: None, diff --git a/src/tools/exec_support.rs b/src/tools/exec_support.rs index 46e4e20..fe2a0a8 100644 --- a/src/tools/exec_support.rs +++ b/src/tools/exec_support.rs @@ -52,6 +52,7 @@ pub(super) fn target_to_exec(side_effects: SideEffects, out: TargetResult) -> To error: shell_error, meta: ToolResultMeta { side_effects, + duration_ms: None, bytes: out.bytes, exit_code: out.exit_code, stderr_truncated: out.stderr_truncated, @@ -72,6 +73,7 @@ pub(super) fn target_to_exec(side_effects: SideEffects, out: TargetResult) -> To pub(super) fn base_meta(rt: &ToolRuntime, side_effects: SideEffects) -> ToolResultMeta { ToolResultMeta { side_effects, + duration_ms: None, bytes: None, exit_code: None, stderr_truncated: None, diff --git a/src/tools/exec_write.rs b/src/tools/exec_write.rs index 17f92bd..13c59be 100644 --- a/src/tools/exec_write.rs +++ b/src/tools/exec_write.rs @@ -268,6 +268,7 @@ async fn run_exact_replace(rt: &ToolRuntime, args: &Value, tool_name: &str) -> T error: None, meta: ToolResultMeta { side_effects: SideEffects::FilesystemWrite, + duration_ms: None, bytes: Some(replaced.len() as u64), exit_code: None, stderr_truncated: None, diff --git a/src/tools/observation.rs b/src/tools/observation.rs new file mode 100644 index 0000000..c25931f --- /dev/null +++ b/src/tools/observation.rs @@ -0,0 +1,748 @@ +use std::fmt; +use std::path::{Component, Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::store::{ensure_dir, sha256_hex, write_json_atomic}; +use crate::tools::ToolResultContentRef; +use crate::types::{Message, ToolCall}; + +const OBSERVATION_SCHEMA_VERSION: &str = "openagent.observation.v1"; +const TOOL_RESULT_ARTIFACT_MANIFEST_SCHEMA_VERSION: &str = + "openagent.tool_result_artifacts_manifest.v1"; +const MANIFEST_FILE_NAME: &str = "manifest.json"; +const HEAD_BYTES: usize = 1200; +const TAIL_BYTES: usize = 4000; +const MAX_DIAGNOSTICS: usize = 8; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ToolResultArtifactManifestV1 { + pub schema_version: String, + pub artifacts: Vec, +} + +impl Default for ToolResultArtifactManifestV1 { + fn default() -> Self { + Self { + schema_version: TOOL_RESULT_ARTIFACT_MANIFEST_SCHEMA_VERSION.to_string(), + artifacts: Vec::new(), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ToolResultArtifactManifestEntryV1 { + pub sequence: u64, + #[serde(rename = "ref")] + pub artifact_ref: ToolResultContentRef, + pub path: String, + pub tool_name: String, + pub tool_call_id: String, + pub label: String, + pub content_kind: String, + pub size_bytes: u64, + pub sha16: String, + pub sha256: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ToolResultArtifactWarning { + pub code: &'static str, + pub message: String, + pub path: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[allow(dead_code)] +pub enum ToolResultArtifactResolveError { + InvalidRefKind(String), + PathTraversal(String), + UnknownRef, + InvalidManifestPath(String), + Io(String), + Parse(String), +} + +impl fmt::Display for ToolResultArtifactResolveError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidRefKind(kind) => write!(f, "unsupported artifact ref kind: {kind}"), + Self::PathTraversal(path) => write!(f, "artifact ref path is unsafe: {path}"), + Self::UnknownRef => write!(f, "unknown tool-result artifact ref"), + Self::InvalidManifestPath(path) => { + write!(f, "manifest artifact path is unsafe: {path}") + } + Self::Io(err) => write!(f, "failed to read tool-result artifact manifest: {err}"), + Self::Parse(err) => write!(f, "failed to parse tool-result artifact manifest: {err}"), + } + } +} + +impl std::error::Error for ToolResultArtifactResolveError {} + +#[derive(Debug, Clone, Serialize)] +struct ObservationV1 { + schema_version: &'static str, + tool_name: String, + status: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + exit_code: Option, + #[serde(skip_serializing_if = "Option::is_none")] + duration_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + cwd: Option, + #[serde(skip_serializing_if = "Option::is_none")] + target_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + command: Option, + #[serde(skip_serializing_if = "Option::is_none")] + stdout_head: Option, + #[serde(skip_serializing_if = "Option::is_none")] + stdout_tail: Option, + #[serde(skip_serializing_if = "Option::is_none")] + stderr_tail: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + changed_files: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + diagnostics: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + failure_class: Option, + #[serde(skip_serializing_if = "Option::is_none")] + suggested_next_action: Option, + #[serde(skip_serializing_if = "Option::is_none")] + raw_result_ref: Option, + #[serde(skip_serializing_if = "Option::is_none")] + stdout_ref: Option, + #[serde(skip_serializing_if = "Option::is_none")] + stderr_ref: Option, + truncated: bool, + truncation_policy: ObservationTruncationPolicyV1, + raw_content_sha256: String, + raw_content_bytes: usize, + raw_content_truncated: bool, +} + +#[derive(Debug, Clone, Copy, Serialize)] +enum ObservationTruncationPolicyV1 { + #[serde(rename = "head_1200_tail_4000")] + Head1200Tail4000, +} + +#[derive(Debug, Clone, Serialize)] +struct ObservationDiagnosticV1 { + stream: &'static str, + line: String, +} + +#[cfg(test)] +pub(crate) fn to_model_observation_message( + raw_tool_msg: &Message, + tc: &ToolCall, + workdir: &Path, + raw_tool_result_dir: Option<&Path>, +) -> Message { + to_model_observation_message_with_warnings(raw_tool_msg, tc, workdir, raw_tool_result_dir).0 +} + +pub(crate) fn to_model_observation_message_with_warnings( + raw_tool_msg: &Message, + tc: &ToolCall, + workdir: &Path, + raw_tool_result_dir: Option<&Path>, +) -> (Message, Vec) { + if tc.name.starts_with("mcp.") { + return (raw_tool_msg.clone(), Vec::new()); + } + let Some(raw_content) = raw_tool_msg.content.as_deref() else { + return (raw_tool_msg.clone(), Vec::new()); + }; + let Ok(mut envelope) = serde_json::from_str::(raw_content) else { + return (raw_tool_msg.clone(), Vec::new()); + }; + let mut warnings = Vec::new(); + let observation = build_observation( + &envelope, + tc, + workdir, + raw_content, + raw_tool_result_dir, + &mut warnings, + ); + let Some(obj) = envelope.as_object_mut() else { + return (raw_tool_msg.clone(), warnings); + }; + let Ok(observation_value) = serde_json::to_value(&observation) else { + return (raw_tool_msg.clone(), warnings); + }; + let Ok(observation_text) = serde_json::to_string(&observation_value) else { + return (raw_tool_msg.clone(), warnings); + }; + obj.insert("content".to_string(), Value::String(observation_text)); + obj.insert("observation".to_string(), observation_value); + let compact_content = match serde_json::to_string(&envelope) { + Ok(content) => content, + Err(_) => return (raw_tool_msg.clone(), warnings), + }; + let mut out = raw_tool_msg.clone(); + out.content = Some(compact_content); + (out, warnings) +} + +pub(crate) fn set_tool_result_duration(message: &mut Message, duration_ms: u64) { + let Some(content) = message.content.as_deref() else { + return; + }; + let Ok(mut envelope) = serde_json::from_str::(content) else { + return; + }; + if let Some(meta) = envelope.get_mut("meta").and_then(Value::as_object_mut) { + meta.insert("duration_ms".to_string(), Value::from(duration_ms)); + } + if let Ok(updated) = serde_json::to_string(&envelope) { + message.content = Some(updated); + } +} + +fn build_observation( + envelope: &Value, + tc: &ToolCall, + workdir: &Path, + raw_content: &str, + raw_tool_result_dir: Option<&Path>, + warnings: &mut Vec, +) -> ObservationV1 { + let ok = envelope.get("ok").and_then(Value::as_bool).unwrap_or(false); + let meta = envelope.get("meta").unwrap_or(&Value::Null); + let raw_tool_content = envelope + .get("content") + .and_then(Value::as_str) + .unwrap_or_default(); + let inner = serde_json::from_str::(raw_tool_content).ok(); + let stdout = inner + .as_ref() + .and_then(|v| v.get("stdout")) + .and_then(Value::as_str); + let stderr = inner + .as_ref() + .and_then(|v| v.get("stderr")) + .and_then(Value::as_str); + let target_path = target_path(envelope, tc, inner.as_ref()); + let changed_files = changed_files(tc, target_path.as_deref(), inner.as_ref(), ok); + let diagnostics = diagnostics(stdout, stderr); + let failure_class = (!ok).then(|| failure_class(envelope, inner.as_ref())); + let raw_content_truncated = envelope + .get("truncated") + .and_then(Value::as_bool) + .unwrap_or(false); + let stdout_truncated = stream_truncated(meta, inner.as_ref(), "stdout", stdout, HEAD_BYTES); + let stderr_truncated = stream_truncated(meta, inner.as_ref(), "stderr", stderr, TAIL_BYTES); + ObservationV1 { + schema_version: OBSERVATION_SCHEMA_VERSION, + tool_name: tc.name.clone(), + status: if ok { "ok" } else { "failed" }, + exit_code: meta + .get("exit_code") + .and_then(Value::as_i64) + .or_else(|| inner.as_ref()?.get("status")?.as_i64()), + duration_ms: meta.get("duration_ms").and_then(Value::as_u64), + cwd: shell_cwd(tc, workdir), + target_path, + command: shell_command(tc), + stdout_head: stdout.and_then(|s| non_empty(head_utf8_bytes(s, HEAD_BYTES))), + stdout_tail: stdout.and_then(|s| tail_if_useful(s, TAIL_BYTES)), + stderr_tail: stderr.and_then(|s| non_empty(tail_utf8_bytes(s, TAIL_BYTES))), + changed_files, + diagnostics, + suggested_next_action: suggested_next_action(envelope, inner.as_ref(), tc, ok), + failure_class, + raw_result_ref: spool_text_artifact( + raw_tool_result_dir, + tc, + "raw_result", + "json", + "application/json", + raw_content, + warnings, + ), + stdout_ref: stdout.and_then(|s| { + stdout_truncated.then(|| { + spool_text_artifact( + raw_tool_result_dir, + tc, + "stdout", + "txt", + "text/plain", + s, + warnings, + ) + })? + }), + stderr_ref: stderr.and_then(|s| { + stderr_truncated.then(|| { + spool_text_artifact( + raw_tool_result_dir, + tc, + "stderr", + "txt", + "text/plain", + s, + warnings, + ) + })? + }), + truncated: raw_content_truncated || stdout_truncated || stderr_truncated, + truncation_policy: ObservationTruncationPolicyV1::Head1200Tail4000, + raw_content_sha256: sha256_hex(raw_content.as_bytes()), + raw_content_bytes: raw_content.len(), + raw_content_truncated, + } +} + +fn stream_truncated( + meta: &Value, + inner: Option<&Value>, + stream: &str, + value: Option<&str>, + model_limit_bytes: usize, +) -> bool { + let meta_key = format!("{stream}_truncated"); + meta.get(&meta_key) + .and_then(Value::as_bool) + .or_else(|| inner?.get(&meta_key).and_then(Value::as_bool)) + .unwrap_or(false) + || value + .map(|s| model_limit_bytes > 0 && s.len() > model_limit_bytes) + .unwrap_or(false) +} + +fn spool_text_artifact( + artifact_dir: Option<&Path>, + tc: &ToolCall, + label: &str, + extension: &str, + content_kind: &str, + content: &str, + warnings: &mut Vec, +) -> Option { + let artifact_dir = artifact_dir?; + if ensure_dir(artifact_dir).is_err() { + warnings.push(ToolResultArtifactWarning { + code: "tool_result_artifact_dir_create_failed", + message: format!( + "failed to create tool-result artifact directory {}", + artifact_dir.display() + ), + path: Some(artifact_dir.display().to_string()), + }); + return None; + } + let bytes = content.as_bytes(); + let sha256 = sha256_hex(bytes); + let tool = sanitize_filename_component(&tc.name); + let id = sanitize_filename_component(&tc.id); + let label = sanitize_filename_component(label); + let extension = sanitize_filename_component(extension); + let file_name = format!("{tool}__{id}__{label}__{}.{}", &sha256[..16], extension); + let path = artifact_dir.join(file_name); + if let Err(err) = std::fs::write(&path, bytes) { + warnings.push(ToolResultArtifactWarning { + code: "tool_result_artifact_write_failed", + message: format!( + "failed to write tool-result artifact {}: {err}", + path.display() + ), + path: Some(path.display().to_string()), + }); + return None; + } + let artifact_ref = ToolResultContentRef { + kind: "run_artifact_path".to_string(), + path: path.to_string_lossy().to_string(), + sha256, + bytes: bytes.len() as u64, + }; + if let Err(err) = + record_tool_result_artifact(artifact_dir, tc, &label, content_kind, &path, &artifact_ref) + { + warnings.push(ToolResultArtifactWarning { + code: "tool_result_artifact_manifest_write_failed", + message: err, + path: Some(manifest_path(artifact_dir).display().to_string()), + }); + } + Some(artifact_ref) +} + +fn record_tool_result_artifact( + artifact_dir: &Path, + tc: &ToolCall, + label: &str, + content_kind: &str, + artifact_path: &Path, + artifact_ref: &ToolResultContentRef, +) -> Result<(), String> { + let manifest_path = manifest_path(artifact_dir); + let mut manifest = if manifest_path.exists() { + let content = std::fs::read_to_string(&manifest_path) + .map_err(|e| format!("failed to read {}: {e}", manifest_path.display()))?; + serde_json::from_str::(&content) + .map_err(|e| format!("failed to parse {}: {e}", manifest_path.display()))? + } else { + ToolResultArtifactManifestV1::default() + }; + if manifest + .artifacts + .iter() + .any(|entry| entry.artifact_ref == *artifact_ref) + { + return Ok(()); + } + let relative_path = artifact_path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| { + format!( + "artifact path has no valid file name: {}", + artifact_path.display() + ) + })? + .to_string(); + let sequence = manifest.artifacts.len() as u64 + 1; + manifest.artifacts.push(ToolResultArtifactManifestEntryV1 { + sequence, + artifact_ref: artifact_ref.clone(), + path: relative_path, + tool_name: tc.name.clone(), + tool_call_id: tc.id.clone(), + label: label.to_string(), + content_kind: content_kind.to_string(), + size_bytes: artifact_ref.bytes, + sha16: artifact_ref.sha256[..16].to_string(), + sha256: artifact_ref.sha256.clone(), + }); + write_json_atomic(&manifest_path, &manifest) + .map_err(|e| format!("failed to write {}: {e}", manifest_path.display())) +} + +fn manifest_path(artifact_dir: &Path) -> PathBuf { + artifact_dir.join(MANIFEST_FILE_NAME) +} + +#[allow(dead_code)] +pub fn load_tool_result_artifact_manifest( + artifact_dir: &Path, +) -> Result { + let path = manifest_path(artifact_dir); + let content = std::fs::read_to_string(&path) + .map_err(|e| ToolResultArtifactResolveError::Io(e.to_string()))?; + serde_json::from_str::(&content) + .map_err(|e| ToolResultArtifactResolveError::Parse(e.to_string())) +} + +#[allow(dead_code)] +pub fn resolve_tool_result_artifact_ref( + artifact_dir: &Path, + artifact_ref: &ToolResultContentRef, +) -> Result { + if artifact_ref.kind != "run_artifact_path" { + return Err(ToolResultArtifactResolveError::InvalidRefKind( + artifact_ref.kind.clone(), + )); + } + reject_traversal_path(&artifact_ref.path)?; + let manifest = load_tool_result_artifact_manifest(artifact_dir)?; + let entry = manifest + .artifacts + .iter() + .find(|entry| entry.artifact_ref == *artifact_ref) + .ok_or(ToolResultArtifactResolveError::UnknownRef)?; + let relative = safe_manifest_relative_path(&entry.path)?; + let resolved = artifact_dir.join(relative); + if resolved.exists() { + let root = std::fs::canonicalize(artifact_dir) + .map_err(|e| ToolResultArtifactResolveError::Io(e.to_string()))?; + let candidate = std::fs::canonicalize(&resolved) + .map_err(|e| ToolResultArtifactResolveError::Io(e.to_string()))?; + if !candidate.starts_with(&root) { + return Err(ToolResultArtifactResolveError::InvalidManifestPath( + entry.path.clone(), + )); + } + return Ok(candidate); + } + Ok(resolved) +} + +#[allow(dead_code)] +fn reject_traversal_path(path: &str) -> Result<(), ToolResultArtifactResolveError> { + if Path::new(path) + .components() + .any(|component| matches!(component, Component::ParentDir)) + { + return Err(ToolResultArtifactResolveError::PathTraversal( + path.to_string(), + )); + } + Ok(()) +} + +#[allow(dead_code)] +fn safe_manifest_relative_path(path: &str) -> Result { + let path_buf = PathBuf::from(path); + if path_buf.components().any(|component| { + matches!( + component, + Component::Prefix(_) | Component::RootDir | Component::ParentDir + ) + }) { + return Err(ToolResultArtifactResolveError::InvalidManifestPath( + path.to_string(), + )); + } + Ok(path_buf) +} + +fn sanitize_filename_component(value: &str) -> String { + let mut out = String::with_capacity(value.len()); + for ch in value.chars() { + let ok = ch.is_ascii_alphanumeric() || matches!(ch, '.' | '-' | '_'); + out.push(if ok { ch } else { '_' }); + } + if out.is_empty() { + "unknown".to_string() + } else { + out + } +} + +fn shell_cwd(tc: &ToolCall, workdir: &Path) -> Option { + if tc.name != "shell" { + return None; + } + tc.arguments + .get("cwd") + .and_then(Value::as_str) + .map(ToOwned::to_owned) + .or_else(|| Some(workdir.display().to_string())) +} + +fn shell_command(tc: &ToolCall) -> Option { + if tc.name != "shell" { + return None; + } + if let Some(command) = tc.arguments.get("command").and_then(Value::as_str) { + return Some(command.to_string()); + } + let cmd = tc.arguments.get("cmd").and_then(Value::as_str)?; + let mut parts = vec![cmd.to_string()]; + if let Some(args) = tc.arguments.get("args").and_then(Value::as_array) { + for arg in args { + if let Some(arg) = arg.as_str() { + parts.push(arg.to_string()); + } + } + } + Some(parts.join(" ")) +} + +fn target_path(envelope: &Value, tc: &ToolCall, inner: Option<&Value>) -> Option { + tc.arguments + .get("path") + .and_then(Value::as_str) + .map(ToOwned::to_owned) + .or_else(|| { + inner? + .get("path") + .and_then(Value::as_str) + .map(ToOwned::to_owned) + }) + .or_else(|| { + envelope + .get("error") + .and_then(|e| e.get("received_args")) + .and_then(|a| a.get("path")) + .and_then(Value::as_str) + .map(ToOwned::to_owned) + }) +} + +fn changed_files( + tc: &ToolCall, + path: Option<&str>, + inner: Option<&Value>, + ok: bool, +) -> Vec { + if !ok + || !matches!( + tc.name.as_str(), + "apply_patch" | "edit" | "write_file" | "str_replace" + ) + { + return Vec::new(); + } + if inner + .and_then(|v| v.get("changed")) + .and_then(Value::as_bool) + .is_some_and(|changed| !changed) + { + return Vec::new(); + } + path.map(|p| vec![p.to_string()]).unwrap_or_default() +} + +fn diagnostics(stdout: Option<&str>, stderr: Option<&str>) -> Vec { + let mut out = Vec::new(); + collect_diagnostics("stdout", stdout.unwrap_or_default(), &mut out); + collect_diagnostics("stderr", stderr.unwrap_or_default(), &mut out); + out +} + +fn collect_diagnostics(stream: &'static str, text: &str, out: &mut Vec) { + for line in text.lines() { + if out.len() >= MAX_DIAGNOSTICS { + return; + } + let trimmed = line.trim(); + let lower = trimmed.to_ascii_lowercase(); + let is_diagnostic = lower.starts_with("error") + || lower.starts_with("failed") + || lower.starts_with("not ok ") + || lower.contains(" assertion") + || lower.contains(" panicked") + || lower.contains("panic") + || lower.contains("exception") + || lower.contains("traceback") + || lower.contains("expected") + || lower.contains("mismatched types") + || lower.contains("cannot find"); + if is_diagnostic && !trimmed.is_empty() { + out.push(ObservationDiagnosticV1 { + stream, + line: truncate_chars(trimmed, 500), + }); + } + } +} + +fn failure_class(envelope: &Value, inner: Option<&Value>) -> String { + let code = envelope + .get("error") + .and_then(|e| e.get("code")) + .and_then(Value::as_str) + .unwrap_or_default(); + match code { + "tool_args_invalid" | "tool_args_malformed_json" => "E_SCHEMA".to_string(), + "tool_path_denied" | "path_out_of_scope" | "tool_disabled" | "shell_gate_deny" => { + "E_POLICY".to_string() + } + "shell_exec_timeout" => "E_TIMEOUT".to_string(), + "shell_exec_not_found" => "E_EXEC_NOT_FOUND".to_string(), + "shell_exec_non_zero_exit" => "E_NONZERO_EXIT".to_string(), + _ => { + if inner + .and_then(|v| v.get("timed_out")) + .and_then(Value::as_bool) + .unwrap_or(false) + { + "E_TIMEOUT".to_string() + } else { + "E_TOOL_FAILED".to_string() + } + } + } +} + +fn suggested_next_action( + envelope: &Value, + inner: Option<&Value>, + tc: &ToolCall, + ok: bool, +) -> Option { + if ok { + return None; + } + let code = envelope + .get("error") + .and_then(|e| e.get("code")) + .and_then(Value::as_str) + .unwrap_or_default(); + if code == "tool_args_invalid" || code == "tool_args_malformed_json" { + return Some("Emit a corrected tool call with valid arguments.".to_string()); + } + if code == "tool_path_denied" || code == "path_out_of_scope" { + return Some( + "Use a workdir-relative path and avoid absolute paths or '..' traversal.".to_string(), + ); + } + if inner + .and_then(|v| v.get("timed_out")) + .and_then(Value::as_bool) + .unwrap_or(false) + || code == "shell_exec_timeout" + { + return Some( + "Narrow the command, make it non-interactive, or rerun with a larger timeout_ms." + .to_string(), + ); + } + match tc.name.as_str() { + "shell" => Some( + "Inspect the failing diagnostics, edit the relevant source, then rerun the command." + .to_string(), + ), + "apply_patch" | "edit" | "str_replace" => Some( + "Re-read the current file, adjust the edit to match the actual content, then retry." + .to_string(), + ), + "read_file" | "grep" | "glob" | "list_dir" => Some( + "Check the path or query shape, then retry with a scoped workdir-relative target." + .to_string(), + ), + _ => None, + } +} + +fn head_utf8_bytes(input: &str, max_bytes: usize) -> String { + truncate_utf8_prefix(input, max_bytes).0 +} + +fn tail_if_useful(input: &str, max_bytes: usize) -> Option { + if input.len() <= HEAD_BYTES { + return None; + } + non_empty(tail_utf8_bytes(input, max_bytes)) +} + +fn tail_utf8_bytes(input: &str, max_bytes: usize) -> String { + if max_bytes == 0 || input.len() <= max_bytes { + return input.to_string(); + } + let mut start = input.len().saturating_sub(max_bytes); + while start < input.len() && !input.is_char_boundary(start) { + start += 1; + } + input[start..].to_string() +} + +fn truncate_utf8_prefix(input: &str, max_bytes: usize) -> (String, bool) { + if max_bytes == 0 || input.len() <= max_bytes { + return (input.to_string(), false); + } + let mut end = max_bytes; + while end > 0 && !input.is_char_boundary(end) { + end -= 1; + } + (input[..end].to_string(), true) +} + +fn non_empty(s: String) -> Option { + (!s.is_empty()).then_some(s) +} + +fn truncate_chars(input: &str, max_chars: usize) -> String { + if input.chars().count() <= max_chars { + return input.to_string(); + } + input.chars().take(max_chars).collect() +} diff --git a/src/tools/tests.rs b/src/tools/tests.rs index 0f22990..83bb8b7 100644 --- a/src/tools/tests.rs +++ b/src/tools/tests.rs @@ -4,8 +4,11 @@ use serde_json::{json, Value}; use tempfile::tempdir; use super::{ - builtin_tools_enabled, execute_tool, tool_side_effects, validate_builtin_tool_args, - validate_schema_args, ToolArgsStrict, ToolRuntime, + builtin_tools_enabled, execute_tool, load_tool_result_artifact_manifest, + resolve_tool_result_artifact_ref, to_model_observation_message, to_tool_result_envelope, + to_tool_result_envelope_with_error, tool_side_effects, validate_builtin_tool_args, + validate_schema_args, ToolArgsStrict, ToolErrorCode, ToolErrorDetail, ToolResultContentRef, + ToolResultMeta, ToolRuntime, }; use crate::target::{ExecTargetKind, HostTarget}; use crate::types::{SideEffects, ToolCall}; @@ -156,6 +159,247 @@ async fn update_plan_returns_compact_side_effect_free_result() { assert_eq!(inner["in_progress"], json!("Implement update_plan")); } +#[test] +fn model_observation_for_shell_failure_preserves_tail_without_raw_dump() { + let tc = ToolCall { + id: "tc_shell_obs".to_string(), + name: "shell".to_string(), + arguments: json!({"cmd":"cargo","args":["test"],"cwd":"crates/demo"}), + }; + let stdout = format!( + "compile start\n{}\nnot ok 7 - parser trims whitespace\nExpected 7 but got NaN\n", + "middle noise\n".repeat(800) + ); + let stderr = format!( + "{}error: test failed, to rerun pass --lib\n", + "stderr noise\n".repeat(500) + ); + let raw_inner = json!({ + "status": 1, + "stdout": stdout, + "stderr": stderr, + "stdout_truncated": false, + "stderr_truncated": false + }) + .to_string(); + let raw_msg = super::envelope_to_message(to_tool_result_envelope_with_error( + &tc, + "builtin", + false, + raw_inner, + false, + Some(ToolErrorDetail { + code: ToolErrorCode::ShellExecNonZeroExit, + message: "Shell command exited with non-zero status: 1.".to_string(), + expected_schema: None, + received_args: None, + minimal_example: None, + available_tools: None, + }), + ToolResultMeta { + side_effects: SideEffects::ShellExec, + duration_ms: Some(123), + bytes: Some(12_000), + exit_code: Some(1), + stderr_truncated: Some(false), + stdout_truncated: Some(false), + source: "builtin".to_string(), + execution_target: "host".to_string(), + warnings: None, + warnings_max: None, + warnings_truncated: None, + docker: None, + }, + )); + + let artifact_dir = tempdir().expect("artifact dir"); + let raw_content = raw_msg.content.as_deref().expect("raw content").to_string(); + let compact = to_model_observation_message( + &raw_msg, + &tc, + Path::new("C:/repo"), + Some(artifact_dir.path()), + ); + let envelope: Value = serde_json::from_str(&compact.content.unwrap()).expect("envelope"); + let observation = envelope.get("observation").expect("observation"); + assert_eq!( + observation.get("schema_version").and_then(Value::as_str), + Some("openagent.observation.v1") + ); + assert_eq!( + observation.get("exit_code").and_then(Value::as_i64), + Some(1) + ); + assert_eq!( + observation.get("duration_ms").and_then(Value::as_u64), + Some(123) + ); + assert!(observation + .get("stdout_tail") + .and_then(Value::as_str) + .unwrap_or_default() + .contains("Expected 7 but got NaN")); + let raw_result_ref = observation.get("raw_result_ref").expect("raw_result_ref"); + assert_eq!( + raw_result_ref.get("kind").and_then(Value::as_str), + Some("run_artifact_path") + ); + let raw_result_path = raw_result_ref + .get("path") + .and_then(Value::as_str) + .expect("raw result path"); + assert_eq!( + std::fs::read_to_string(raw_result_path).expect("raw artifact"), + raw_content + ); + let manifest = + load_tool_result_artifact_manifest(artifact_dir.path()).expect("artifact manifest"); + assert_eq!( + manifest.schema_version, + "openagent.tool_result_artifacts_manifest.v1" + ); + let raw_entry = manifest + .artifacts + .iter() + .find(|entry| entry.label == "raw_result") + .expect("raw manifest entry"); + assert_eq!(raw_entry.sequence, 1); + assert_eq!(raw_entry.tool_name, "shell"); + assert_eq!(raw_entry.tool_call_id, "tc_shell_obs"); + assert_eq!(raw_entry.content_kind, "application/json"); + assert_eq!(raw_entry.size_bytes, raw_content.len() as u64); + assert_eq!(raw_entry.artifact_ref.path, raw_result_path); + assert_eq!(raw_entry.artifact_ref.bytes, raw_content.len() as u64); + assert_eq!(raw_entry.sha16.len(), 16); + assert_eq!(raw_entry.sha256.len(), 64); + assert!(observation.get("stdout_ref").is_some()); + let stderr_ref = observation.get("stderr_ref").expect("stderr_ref"); + let stderr_path = stderr_ref + .get("path") + .and_then(Value::as_str) + .expect("stderr path"); + assert!(std::fs::read_to_string(stderr_path) + .expect("stderr artifact") + .contains("error: test failed")); + assert_eq!( + observation.get("truncated").and_then(Value::as_bool), + Some(true) + ); + assert_eq!( + observation.get("truncation_policy").and_then(Value::as_str), + Some("head_1200_tail_4000") + ); + assert!(!envelope + .get("content") + .and_then(Value::as_str) + .unwrap_or_default() + .contains("middle noise\nmiddle noise\nmiddle noise")); + assert_eq!( + observation.get("failure_class").and_then(Value::as_str), + Some("E_NONZERO_EXIT") + ); + assert!(observation + .get("suggested_next_action") + .and_then(Value::as_str) + .unwrap_or_default() + .contains("rerun the command")); +} + +#[test] +fn tool_result_artifact_resolver_accepts_manifested_ref_and_rejects_bad_refs() { + let tc = ToolCall { + id: "tc_resolver".to_string(), + name: "read_file".to_string(), + arguments: json!({"path":"src/main.rs"}), + }; + let raw_msg = super::envelope_to_message(to_tool_result_envelope( + &tc, + "builtin", + true, + json!({"path":"src/main.rs","content":"hello"}).to_string(), + false, + ToolResultMeta { + side_effects: SideEffects::FilesystemRead, + duration_ms: Some(3), + bytes: Some(5), + exit_code: None, + stderr_truncated: None, + stdout_truncated: None, + source: "builtin".to_string(), + execution_target: "host".to_string(), + warnings: None, + warnings_max: None, + warnings_truncated: None, + docker: None, + }, + )); + let artifact_dir = tempdir().expect("artifact dir"); + let compact = + to_model_observation_message(&raw_msg, &tc, Path::new("."), Some(artifact_dir.path())); + let envelope: Value = serde_json::from_str(&compact.content.unwrap()).expect("envelope"); + let raw_ref: ToolResultContentRef = + serde_json::from_value(envelope["observation"]["raw_result_ref"].clone()).expect("raw ref"); + + let resolved = + resolve_tool_result_artifact_ref(artifact_dir.path(), &raw_ref).expect("resolve ref"); + assert_eq!( + std::fs::read_to_string(resolved).expect("artifact"), + raw_msg.content.unwrap() + ); + + let mut unknown_ref = raw_ref.clone(); + unknown_ref.path = artifact_dir + .path() + .join("missing-artifact.json") + .to_string_lossy() + .to_string(); + assert!(resolve_tool_result_artifact_ref(artifact_dir.path(), &unknown_ref).is_err()); + + let mut malicious_ref = raw_ref; + malicious_ref.path = "../outside.json".to_string(); + assert!(resolve_tool_result_artifact_ref(artifact_dir.path(), &malicious_ref).is_err()); +} + +#[test] +fn model_observation_for_write_reports_changed_file() { + let tc = ToolCall { + id: "tc_edit_obs".to_string(), + name: "edit".to_string(), + arguments: json!({"path":"src/main.rs","old_string":"old","new_string":"new"}), + }; + let raw_msg = super::envelope_to_message(to_tool_result_envelope( + &tc, + "builtin", + true, + json!({"path":"src/main.rs","changed":true,"bytes_written":42}).to_string(), + false, + ToolResultMeta { + side_effects: SideEffects::FilesystemWrite, + duration_ms: Some(7), + bytes: Some(42), + exit_code: None, + stderr_truncated: None, + stdout_truncated: None, + source: "builtin".to_string(), + execution_target: "host".to_string(), + warnings: None, + warnings_max: None, + warnings_truncated: None, + docker: None, + }, + )); + + let compact = to_model_observation_message(&raw_msg, &tc, Path::new("."), None); + let envelope: Value = serde_json::from_str(&compact.content.unwrap()).expect("envelope"); + let changed = envelope + .get("observation") + .and_then(|o| o.get("changed_files")) + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + assert_eq!(changed, vec![json!("src/main.rs")]); +} + #[test] fn update_plan_rejects_multiple_in_progress_items() { let err = validate_builtin_tool_args( diff --git a/tests/mcp_impl_regression.rs b/tests/mcp_impl_regression.rs index 496fd0f..11d8321 100644 --- a/tests/mcp_impl_regression.rs +++ b/tests/mcp_impl_regression.rs @@ -256,6 +256,7 @@ fn make_agent_with_mcp( omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(), diff --git a/tests/tool_call_accuracy_ci.rs b/tests/tool_call_accuracy_ci.rs index 7c4ca90..24f2179 100644 --- a/tests/tool_call_accuracy_ci.rs +++ b/tests/tool_call_accuracy_ci.rs @@ -194,6 +194,7 @@ fn build_agent( omit_tools_field_when_empty: false, plan_tool_enforcement: PlanToolEnforcementMode::Off, mcp_pin_enforcement: McpPinEnforcementMode::Hard, + raw_tool_result_dir: None, plan_step_constraints: Vec::::new(), current_plan: Vec::new(), tool_call_budget: ToolCallBudget::default(),