Update notes on SBV

Author: klieret

js/leaderboardFilters.js

@@ -409,7 +409,7 @@ function updateLeaderboardDescription(leaderboardName) {
     if (!textContainer) return;
 
     const descriptions = {
-        'bash-only': '<em>Bash Only</em> evaluates all LMs with a <a href="https://github.com/SWE-agent/mini-swe-agent">minimal agent</a> on SWE-bench Verified (<a href="bash-only.html">details</a>)',
+        'bash-only': '<em>Bash Only</em> evaluates all LMs with a <a href="https://github.com/SWE-agent/mini-swe-agent">minimal agent</a> on SWE-bench Verified.<br/>Grayed out results were obtained with a different agent version (<a href="bash-only.html">details</a>).',
         'multilingual': '<em>Multilingual</em> features 300 tasks across 9 programming languages (<a href="multilingual-leaderboard.html">details</a>)',
         'lite': '<em>Lite</em> is a subset of 300 instances for less costly evaluation (<a href="lite.html">details</a>)',
         'verified': '<em>Verified</em> is a human-filtered subset of 500 instances (<a href="https://openai.com/index/introducing-swe-bench-verified/">details</a>)',

templates/pages/bash-only.html

@@ -34,14 +34,18 @@ <h2>Overview</h2>
 
             <ul>
                 <li>We use <a href="https://github.com/SWE-agent/mini-swe-agent/blob/main/src/minisweagent/config/extra/swebench.yaml">this configuration</a> for all models.</li>
-                <li>The LM temperature is set to 0.0 if the temperature parameter is supported.</li>
-                <li><a href="https://mini-swe-agent.com/latest/usage/swebench/">This guide</a> shows how to run the evaluation yourself.</li>
+                <li>The release number in the leaderboard corresponds to the version of the mini-SWE-agent used to run the evaluation.</li>
+                <li>Results of release 1.x and 2.x are not necessarily comparable to each other, as 2.x uses tool calling to invoke actions, whereas 1.x parses action from the output strings. Read more about the changes in the <a href="https://mini-swe-agent.com/latest/advanced/v2_migration/">mini-SWE-agent v2 migration guide</a>.</li>
+                <li>For all results of release 1.x and earlier, the LM temperature is set to 0.0 if the temperature parameter is supported. For all results of release 2.x and later, the temperature parameter is not set. </li>
                 <li>
-                    Small changes in the setup and configuration are captured by the version number in the leaderboard. 
+                    Other than the aforementioned notes, small changes in the setup and configuration are captured by the version number in the leaderboard. 
                     Version numbers correspond to tags in the mini-SWE-agent repository. 
                     Since the mini-SWE-agent repository contains other components as well, a new version number does not necessarily mean that anything of relevance has changed for the bash-only leaderboard setting.
                     We do <em>not</em> aim to tune the configuration and setup to reach higher and higher scores. 
-                    Instead, we only make general fixes to the framework, as well as clarifications in the prompt to provide a maximally fair evaluation setup for the LMs. </li>
+                    Instead, we only make general fixes to the framework, as well as clarifications in the prompt to provide a maximally fair evaluation setup for the LMs. 
+                    Generally, everything in the minor or patch release version number should be a minor change for the purpose of the bash-only leaderboard.
+                </li>
+                <li><a href="https://mini-swe-agent.com/latest/usage/swebench/">This guide</a> shows how to run the evaluation yourself.</li>
             </ul>
         </details>