BugDetectionBench/review-bugs.ts at main · moritzWa/BugDetectionBench · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
import fs from "fs";
import { promisify } from "util";
import { reviewBug } from "./review-utils";

const readFileAsync = promisify(fs.readFile);
const writeFileAsync = promisify(fs.writeFile);

// Define a type for the bug object
interface Bug {
  prNumber: number;
  repo: string;
  LLMReviewed: boolean;
  LLMReviewPassed?: boolean;
  reReviewedAfterJan27th?: boolean;
  // Add other properties if needed
}

async function main() {
  try {
    // Clear previous prompts file
    await fs.promises.writeFile("review-prompts.txt", "");

    // Read existing bugs
    const bugsJson = await readFileAsync("bugs.json", "utf8");
    const bugs: Bug[] = JSON.parse(bugsJson);

    // Filter out bugs that have already been reviewed and failed after Jan 27th
    const bugsToReview = bugs.filter(bug =>
      !(bug.reReviewedAfterJan27th && bug.LLMReviewed && bug.LLMReviewPassed === false)
    );

    const totalBugs = bugsToReview.length;
    console.log(`Total bugs to review: ${totalBugs} (${bugs.length - totalBugs} skipped as already failed)`);

    const batchSize = 5;

    for (let i = 0; i < bugsToReview.length; i += batchSize) {
      const batch = bugsToReview.slice(i, i + batchSize);

      await Promise.all(
        batch.map(async (bug: Bug, index: number) => {
          console.log(
            `Reviewing bug ${i + index + 1}/${totalBugs} in PR #${bug.prNumber} from ${bug.repo}`
          );

          // Review the bug using shared functionality
          const shouldInclude = await reviewBug(bug);

          // Update bug object
          bug.LLMReviewed = true;
          bug.LLMReviewPassed = shouldInclude ?? false;
          bug.reReviewedAfterJan27th = true;

          console.log(
            `Review result for PR #${bug.prNumber} (${i + index + 1}/${totalBugs}): ${
              shouldInclude ? "PASS" : "FAIL"
            }`
          );
        })
      );

      // Need to update the original bugs array with our changes
      for (const reviewedBug of batch) {
        const originalBug = bugs.find(b => b.prNumber === reviewedBug.prNumber && b.repo === reviewedBug.repo);
        if (originalBug) {
          originalBug.LLMReviewed = reviewedBug.LLMReviewed;
          originalBug.LLMReviewPassed = reviewedBug.LLMReviewPassed;
          originalBug.reReviewedAfterJan27th = reviewedBug.reReviewedAfterJan27th;
        }
      }

      // Save after each batch in case of interruption
      await writeFileAsync("bugs.json", JSON.stringify(bugs, null, 2));

      // Add a small delay to avoid rate limits
      await new Promise((resolve) => setTimeout(resolve, 1000));
    }

    console.log("Review completed!");
  } catch (error) {
    console.error("Error:", error);
  }
}

main();