eval_process.py

import json
import argparse
import os
import time
from openai import OpenAI # <-- Changed import
## TODO: REMINDER: the log file contains the task info, the process info

# --- API Configuration ---
# We no longer need the Google-specific apiUrl.
# The user will provide the base_url and api_key via arguments.

def get_system_prompt():
    """
    Defines the system prompt, now including the JSON schema
    to guide the model in "json_object" mode.
    """
    
    # Define the JSON schema as a string to be included in the prompt
    json_schema_description = """
{
    "analysis_summary": "A high-level summary of the agent's problem-solving process, noting its strategy, use of feedback, and overall success.",
    "continuous_problem_solving_rating": "A 1-10 rating (number) of the agent's continuous problem-solving ability. 1 = Stuck in loops, ignores feedback. 10 = Highly efficient, logical, and successful.",
    "key_observations": [
        "A bulleted list (array of strings) of 3-5 specific observations from the log (e.g., 'Correctly identified and fixed a NameError at round 3', 'Wasted 4 rounds trying the same failed approach')."
    ],
    "final_verdict": "A final verdict string (e.g., 'SUCCESS', 'FAILURE', 'PARTIAL')."
}
"""

    return f"""You are an expert code reviewer and AI process analyst, specializing in computational science and iterative development. Your task is to analyze the provided log file from an AI coding agent.

Context:
- The agent's task is a complex computational imaging problem.
- The log file shows the agent's full iterative process:
  1. It receives a problem.
  2. It writes Python code.
  3. The code is executed.
  4. It receives feedback (e.g., 'SUCCESS' or an error message).
  5. It revises the code in a loop based on the feedback.

Your Analysis Must Focus On:
1.  **Progress:** Is the agent making tangible progress, or is it getting stuck in loops (e.g., repeating the same error)?
2.  **Feedback Use:** Does the agent *understand* and *use* the error messages to make intelligent revisions? Or does it flail, trying random changes?
3.  **Efficiency:** How many rounds does it take? Does it solve the problem logically, or by brute force?
4.  **Final Outcome:** Did the agent successfully solve the problem as described in its task?

Output:
You MUST provide your analysis ONLY in a structured JSON format. Do not write any other text. The JSON object must conform to the following schema:
{json_schema_description}
"""

# We no longer need get_generation_schema() as it's now in the prompt.
# We no longer need retry_post_request() as the OpenAI client handles retries.

def analyze_log(log_content, base_url, api_key, model_name):
    """
    Makes the API call to a Gemini-OpenAI compatible endpoint.
    """
    
    print(f"Initializing OpenAI client for base_url: {base_url}")
    try:
        client = OpenAI(
            base_url=base_url,
            api_key=api_key,
            timeout=120.0 # Add a longer timeout for large log analysis
        )
    except Exception as e:
        print(f"Error initializing OpenAI client: {e}")
        return None
    
    system_prompt = get_system_prompt()
    
    print(f"Sending log to model '{model_name}' for analysis... this may take a moment.")
    
    try:
        # Call the chat completions endpoint using JSON object mode
        completion = client.chat.completions.create(
            model=model_name,
            messages=[
                {"role": "system", "content": system_prompt},
                {"role": "user", "content": log_content}
            ],
            response_format={"type": "json_object"}
        )
        
        # Get the JSON string from the response
        json_text = completion.choices[0].message.content
        
        # Parse the JSON string into a Python dictionary
        data = json.loads(json_text)
        return data

    except Exception as e:
        print(f"An error occurred during the API call: {e}")
        return None

def main():
    parser = argparse.ArgumentParser(description="Analyze a coding agent's log file using a Gemini-OpenAI compatible API.")
    
    # --- New required arguments ---
    parser.add_argument(
        "--base_url",
        type=str,
        required=True,
        help="The base URL of your OpenAI-compatible API provider (e.g., 'https://api.example.com/v1')"
    )
    parser.add_argument(
        "--api_key",
        type=str,
        required=True,
        help="Your API key for the provider. (Can also be set via OPENAI_API_KEY env var)"
    )
    # --- ---
    
    parser.add_argument(
        "--log_file",
        type=str,
        required=True,
        help="Path to the agent's log file (.log or .txt)"
    )
    parser.add_argument(
        "--model_name",
        type=str,
        default="gemini-2.5-flash-preview-09-2025", # Change this to your provider's model ID
        help="The name/ID of the Gemini model as your provider calls it."
    )
    
    args = parser.parse_args()
    
    # You can also set the API key via environment variable
    api_key = args.api_key or os.environ.get("OPENAI_API_KEY")
    if not api_key:
        print("Error: API key must be provided via --api_key or OPENAI_API_KEY environment variable.")
        return

    if not os.path.exists(args.log_file):
        print(f"Error: Log file not found at {args.log_file}")
        return
        
    print(f"Reading log file: {args.log_file}")
    with open(args.log_file, 'r', encoding='utf-8') as f:
        log_content = f.read()
        
    if not log_content.strip():
        print("Error: Log file is empty.")
        return
        
    analysis_data = analyze_log(log_content, args.base_url, api_key, args.model_name)
    
    if analysis_data:
        print("\n--- Gemini Analysis Complete ---")
        # Pretty-print the JSON response
        print(json.dumps(analysis_data, indent=2))
        print(analysis_data["continuous_problem_solving_rating"])
        print("----------------------------------")

if __name__ == "__main__":
    main()