Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 114 additions & 10 deletions src/agent.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ pub mod task_contract;
mod timeouts;
pub mod tool_facts;
mod tool_helpers;
pub(crate) mod validators;
pub use agent_types::{
AgentExitReason, AgentOutcome, AgentTaintRecord, McpPinEnforcementMode, McpRuntimeTraceEntry,
PlanStepConstraint, PlanToolEnforcementMode, PolicyLoadedInfo, ToolCallBudget,
Expand Down Expand Up @@ -295,6 +296,7 @@ impl<P: ModelProvider> Agent<P> {
required_command: self
.required_validation_command(user_prompt)
.map(ToOwned::to_owned),
inferred_validator: None,
satisfied: false,
repair_mode: false,
exact_final_answer_required: self.exact_final_answer_required(user_prompt),
Expand Down Expand Up @@ -343,6 +345,17 @@ impl<P: ModelProvider> Agent<P> {
tool_calls: &mut Vec<ToolCall>,
) -> bool {
if tool_calls.len() == 1 && tool_calls[0].name == "shell" {
if normalized_shell_command(&tool_calls[0].arguments).as_deref()
!= Some(required_command)
{
tool_calls[0] = ToolCall {
id: format!("tc_validation_shell_{}", Uuid::new_v4().simple()),
name: "shell".to_string(),
arguments: json!({ "command": required_command }),
};
assistant.content = Some(String::new());
return true;
}
if !assistant
.content
.as_deref()
Expand Down Expand Up @@ -447,7 +460,12 @@ impl<P: ModelProvider> Agent<P> {
if runtime_checkpoint.phase != crate::agent_runtime::state::RunPhase::Validating {
return Ok(PhaseLoopControl::Proceed);
}
if !self.validation_shell_available() {
if !self.validation_shell_available()
&& runtime_checkpoint
.validation_state
.inferred_validator
.is_none()
{
runtime_checkpoint.phase = crate::agent_runtime::state::RunPhase::Executing;
runtime_checkpoint
.retry_state
Expand Down Expand Up @@ -477,13 +495,21 @@ impl<P: ModelProvider> Agent<P> {
);
}
}
let validation_phase_message = runtime_checkpoint
.validation_state
.required_command
.as_deref()
.map(|command| {
crate::agent::completion_policy::required_validation_phase_message(command)
})
.unwrap_or_else(|| self.required_validation_phase_message(user_prompt));
let decision = decide_required_validation_phase_response(
user_prompt,
self.validation_shell_available(),
runtime_checkpoint,
&resp.assistant,
&resp.tool_calls,
self.required_validation_phase_message(user_prompt),
validation_phase_message,
);
match apply_required_validation_guard_decision(decision, &resp.assistant, messages) {
GuardEffect::Proceed => Ok(PhaseLoopControl::Proceed),
Expand Down Expand Up @@ -2831,15 +2857,47 @@ impl<P: ModelProvider> Agent<P> {
Err(outcome) => return Err(outcome),
}

if let Some(effect) =
completion_blocked_effect_from_post_tool_refresh(refresh_phase_state_from_tool_facts(
user_prompt,
runtime_checkpoint,
observed_tool_calls,
observed_tool_executions,
successful_write_tool_ok_this_step,
))
let refresh = refresh_phase_state_from_tool_facts(
user_prompt,
runtime_checkpoint,
observed_tool_calls,
observed_tool_executions,
successful_write_tool_ok_this_step,
);
if let crate::agent::phase_transitions::PostToolPhaseRefreshEffect::StopAfterRepairBudgetExhausted {
reason,
} = &refresh
{
self.emit_event(
run_id,
step,
EventKind::Error,
serde_json::json!({
"error": reason,
"source": "runtime_required_validation_guard",
"failure_class": "E_RUNTIME_VALIDATION_REPAIR_EXHAUSTED",
"repair_budget_max": 1
}),
);
return Err(self.finalize_planner_error_with_end(
step,
run_id.to_string(),
started_at.to_string(),
reason.clone(),
messages.clone(),
observed_tool_calls.to_vec(),
observed_tool_decisions.to_vec(),
context_size_chars(messages),
last_compaction_report.clone(),
hook_invocations.clone(),
*provider_retry_count,
*provider_error_count,
*saw_token_usage,
total_token_usage,
taint_state,
));
}
if let Some(effect) = completion_blocked_effect_from_post_tool_refresh(refresh) {
self.emit_phase_transition(run_id, step, &effect.transition);
self.emit_event(
run_id,
Expand Down Expand Up @@ -2878,6 +2936,52 @@ impl<P: ModelProvider> Agent<P> {
successful_write_tool_ok_this_step: bool,
request_context_chars: usize,
) -> Result<PhaseStepDispatch, AgentOutcome> {
if let Some(validator) = runtime_checkpoint
.validation_state
.inferred_validator
.clone()
{
if runtime_checkpoint.validation_state.satisfied
&& !runtime_checkpoint
.validation_state
.exact_final_answer_required
&& observed_tool_executions
.last()
.is_some_and(|execution| execution.name == "shell" && execution.ok)
{
let outcome = crate::agent::validators::ValidatorOutcome::Passed;
self.emit_event(
run_id,
step,
EventKind::CompletionSatisfied,
serde_json::json!({
"reason": "inferred_validator_passed",
"validator_kind": validator.kind,
"validator_label": validator.label,
"command": validator.command,
"outcome": outcome
}),
);
return Err(self.finalize_ok_with_end(
step,
run_id.to_string(),
started_at.to_string(),
format!("Validation passed: `{}`.", validator.command),
messages.clone(),
observed_tool_calls.to_vec(),
observed_tool_decisions.to_vec(),
context_size_chars(messages),
last_compaction_report.clone(),
hook_invocations.to_vec(),
*provider_retry_count,
*provider_error_count,
*saw_token_usage,
total_token_usage,
taint_state,
));
}
}

if let Some(control) = self
.handle_verified_write_follow_on_phase(
user_prompt,
Expand Down
55 changes: 41 additions & 14 deletions src/agent/completion_policy.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,17 @@
pub(crate) enum VerifiedWriteCompletionDecision {
FinalizeNow,
StartFinalAnswerPhase(String),
StartRequiredValidationPhase(String),
StartRequiredValidationPhase {
developer_message: String,
command: String,
inferred_validator: Option<crate::agent::validators::ValidatorCommand>,
},
}

pub(crate) struct RequiredValidationFollowOn {
pub(crate) developer_message: String,
pub(crate) command: String,
pub(crate) inferred_validator: Option<crate::agent::validators::ValidatorCommand>,
}

#[derive(Debug, Clone, PartialEq, Eq)]
Expand Down Expand Up @@ -55,7 +65,7 @@ pub(crate) struct RuntimePhaseTransitionDecision {
pub(crate) completion_reason: &'static str,
}

fn required_validation_phase_message(required_command: &str) -> String {
pub(crate) fn required_validation_phase_message(required_command: &str) -> String {
format!(
"Validation required now. Return exactly one shell tool call and no prose. Run `{required_command}`. Example arguments: {{\"command\":\"{required_command}\"}}. After it succeeds, reply with the final answer only."
)
Expand Down Expand Up @@ -302,18 +312,18 @@ pub(crate) fn decide_validation_phase_transition(
}

pub(crate) fn decide_verified_write_completion(
required_validation_command: Option<&str>,
required_validation: Option<RequiredValidationFollowOn>,
exact_final_answer_required: bool,
_verified_paths: &[String],
has_post_tool_assistant_closeout: bool,
_post_write_follow_on_turn_count: u32,
) -> VerifiedWriteCompletionDecision {
if required_validation_command.is_some() && !has_post_tool_assistant_closeout {
return VerifiedWriteCompletionDecision::StartRequiredValidationPhase(
required_validation_phase_message(
required_validation_command.unwrap_or("the required validation command"),
),
);
if let Some(required_validation) = required_validation {
return VerifiedWriteCompletionDecision::StartRequiredValidationPhase {
developer_message: required_validation.developer_message,
command: required_validation.command,
inferred_validator: required_validation.inferred_validator,
};
}
if !has_post_tool_assistant_closeout {
return VerifiedWriteCompletionDecision::StartFinalAnswerPhase(
Expand All @@ -338,17 +348,32 @@ mod tests {
post_validation_final_answer_transition_decision,
required_validation_boundary_transition_decision, resume_phase_from_checkpoint_state,
validation_resume_execution_transition_decision, RequiredValidationCompletionDecision,
RuntimeCheckpointResumeKind, ValidationFacts, ValidationPhaseTransitionDecision,
VerifiedWriteCompletionDecision,
RequiredValidationFollowOn, RuntimeCheckpointResumeKind, ValidationFacts,
ValidationPhaseTransitionDecision, VerifiedWriteCompletionDecision,
};
use crate::agent::ToolFactV1;

#[test]
fn verified_write_policy_starts_validation_when_prompt_requires_it() {
let decision = decide_verified_write_completion(Some("cargo test"), false, &[], false, 0);
let decision = decide_verified_write_completion(
Some(RequiredValidationFollowOn {
developer_message: "run cargo test".to_string(),
command: "cargo test".to_string(),
inferred_validator: None,
}),
false,
&[],
false,
0,
);
match decision {
VerifiedWriteCompletionDecision::StartRequiredValidationPhase(message) => {
assert!(message.contains("cargo test"));
VerifiedWriteCompletionDecision::StartRequiredValidationPhase {
developer_message,
command,
..
} => {
assert!(developer_message.contains("cargo test"));
assert_eq!(command, "cargo test");
}
_ => panic!("expected validation phase"),
}
Expand Down Expand Up @@ -589,6 +614,7 @@ mod tests {
tool_protocol_state: crate::agent_runtime::state::ToolProtocolState::default(),
validation_state: crate::agent_runtime::state::ValidationState {
required_command: Some("cargo test".to_string()),
inferred_validator: None,
satisfied: false,
repair_mode: false,
exact_final_answer_required: false,
Expand Down Expand Up @@ -643,6 +669,7 @@ mod tests {
tool_protocol_state: crate::agent_runtime::state::ToolProtocolState::default(),
validation_state: crate::agent_runtime::state::ValidationState {
required_command: Some("cargo test".to_string()),
inferred_validator: None,
satisfied: true,
repair_mode: false,
exact_final_answer_required: true,
Expand Down
33 changes: 21 additions & 12 deletions src/agent/phase_transitions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ pub(crate) enum PostToolPhaseRefreshEffect {
transition: RuntimePhaseTransitionDecision,
reason: String,
},
StopAfterRepairBudgetExhausted {
reason: String,
},
}

pub(crate) struct VerifiedWriteFollowOnUpdate {
Expand Down Expand Up @@ -136,6 +139,18 @@ pub(crate) fn refresh_phase_state_from_tool_facts(
match decide_validation_phase_transition(&validation_facts, successful_write_tool_ok_this_step)
{
ValidationPhaseTransitionDecision::EnterRepair => {
let budget = crate::agent::validators::RepairBudget::one_repair(
runtime_checkpoint.retry_state.validation_repair_turn_count,
);
if !budget.can_authorize_repair() {
return PostToolPhaseRefreshEffect::StopAfterRepairBudgetExhausted {
reason: "validation failed after the single allowed repair turn".to_string(),
};
}
runtime_checkpoint.retry_state.validation_repair_turn_count = runtime_checkpoint
.retry_state
.validation_repair_turn_count
.saturating_add(1);
runtime_checkpoint.phase = crate::agent_runtime::state::RunPhase::Executing;
runtime_checkpoint.validation_state.repair_mode = true;
runtime_checkpoint.validation_state.collecting_final_answer = false;
Expand Down Expand Up @@ -171,7 +186,7 @@ pub(crate) fn refresh_phase_state_from_tool_facts(
}

pub(crate) fn apply_verified_write_follow_on(
user_prompt: &str,
_user_prompt: &str,
runtime_checkpoint: &mut crate::agent_runtime::state::RunCheckpointV1,
result: &VerifiedWriteResult,
) -> Option<VerifiedWriteFollowOnUpdate> {
Expand Down Expand Up @@ -199,24 +214,18 @@ pub(crate) fn apply_verified_write_follow_on(
developer_message: message.clone(),
})
}
VerifiedWriteResult::StartRequiredValidationPhase(message) => {
VerifiedWriteResult::StartRequiredValidationPhase(follow_on) => {
runtime_checkpoint.retry_state.post_write_guard_retry_count = 0;
runtime_checkpoint
.retry_state
.blocked_runtime_completion_count = 0;
runtime_checkpoint.phase = crate::agent_runtime::state::RunPhase::Validating;
if runtime_checkpoint
.validation_state
.required_command
.is_none()
{
runtime_checkpoint.validation_state.required_command =
crate::agent_impl_guard::prompt_required_validation_command(user_prompt)
.map(str::to_string);
}
runtime_checkpoint.validation_state.required_command = Some(follow_on.command.clone());
runtime_checkpoint.validation_state.inferred_validator =
follow_on.inferred_validator.clone();
Some(VerifiedWriteFollowOnUpdate {
control: PhaseLoopControl::ContinueAgentStep,
developer_message: message.clone(),
developer_message: follow_on.developer_message.clone(),
})
}
VerifiedWriteResult::Done(_) => None,
Expand Down
13 changes: 13 additions & 0 deletions src/agent/response_guards.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,19 @@ pub(crate) fn decide_required_validation_phase_response(
return RequiredValidationPhaseDecision::Proceed;
}
if !validation_shell_available {
if runtime_checkpoint
.validation_state
.inferred_validator
.is_some()
{
runtime_checkpoint
.retry_state
.blocked_required_validation_phase_count = 0;
return RequiredValidationPhaseDecision::PlannerError {
reason: "inferred validator could not run because the shell tool is unavailable or disabled by policy".to_string(),
blocked_count: 0,
};
}
runtime_checkpoint.phase = RunPhase::Executing;
runtime_checkpoint
.retry_state
Expand Down
Loading
Loading