agentv/.agentv/targets.yaml at 707761b0e1d5464df6af851270d77d105853c7dc · EntityProcess/agentv · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
# AgentV Evaluation Targets
# Consolidated from root, examples/features, and examples/showcase.
# Per-eval .agentv folders override these for specific eval cases.
#
# "grader" is the LLM used for scoring; agent targets reference it via
# grader_target so eval execution and grading use separate models.

targets:
  # ── Default target (use) ───────────────────────────────────────────
  # Evals without an explicit target resolve to "default". The use
  # redirects to a named target, controlled via AGENT_TARGET env var.
  # One env var switches the entire provider config (auth, model, etc.).
  # Example: AGENT_TARGET=copilot-cli  or  AGENT_TARGET=claude
  - name: default
    use_target: ${{ AGENT_TARGET }}

  # ── LLM target (text generation, no agent binary needed) ────────────
  # Delegates to GRADER_TARGET — same provider used for grading and LLM evals.
  - name: llm
    use_target: ${{ GRADER_TARGET }}

  # ── Grader (LLM-as-judge) ──────────────────────────────────────────
  # Used by agent targets via grader_target. Switch provider via GRADER_TARGET.
  - name: grader
    use_target: ${{ GRADER_TARGET }}

  # ── Named agent targets ───────────────────────────────────────────
  - name: copilot-cli
    provider: copilot-cli
    model: ${{ COPILOT_MODEL }}
    grader_target: grader
    log_format: json

  - name: copilot-sdk
    provider: copilot-sdk
    model: ${{ COPILOT_MODEL }}
    grader_target: grader
    log_format: json

  - name: claude
    provider: claude
    grader_target: grader
    log_format: json

  - name: claude-sdk
    provider: claude-sdk
    grader_target: grader
    log_format: json

  - name: pi
    provider: pi-coding-agent
    subprovider: openrouter
    model: ${{ OPENROUTER_MODEL }}
    api_key: ${{ OPENROUTER_API_KEY }}
    grader_target: grader
    tools: read,bash,edit,write
    log_format: json

  - name: pi-cli
    provider: pi-cli
    subprovider: openrouter
    model: ${{ OPENROUTER_MODEL }}
    api_key: ${{ OPENROUTER_API_KEY }}
    grader_target: grader

  - name: codex
    provider: codex
    grader_target: grader
    cwd: ${{ CODEX_WORKSPACE_DIR }}
    log_dir: ${{ CODEX_LOG_DIR }}
    log_format: json

  # ── LLM targets (direct model access) ─────────────────────────────
  - name: azure-llm
    provider: azure
    endpoint: ${{ AZURE_OPENAI_ENDPOINT }}
    api_key: ${{ AZURE_OPENAI_API_KEY }}
    model: ${{ AZURE_DEPLOYMENT_NAME }}
    version: ${{ AZURE_OPENAI_API_VERSION }}

  - name: gemini-llm
    provider: gemini
    api_key: ${{ GOOGLE_GENERATIVE_AI_API_KEY }}
    model: ${{ GEMINI_MODEL_NAME }}

  - name: gemini-flash
    provider: gemini
    model: gemini-3-flash-preview
    api_key: ${{ GOOGLE_GENERATIVE_AI_API_KEY }}

  - name: openai
    provider: openai
    endpoint: ${{ OPENAI_ENDPOINT }}
    api_key: ${{ OPENAI_API_KEY }}
    model: ${{ OPENAI_MODEL }}

  - name: openrouter
    provider: openrouter
    api_key: ${{ OPENROUTER_API_KEY }}
    model: ${{ OPENROUTER_MODEL }}