add basic ci functionality for serial cpu tests

Author: Dando18

.github/workflows/serial-tests.yml

@@ -0,0 +1,32 @@
+name: Serial CPU Tests
+
+on:
+  push:
+    branches:
+      - develop
+  pull_request:
+    branches:
+      - develop
+
+jobs:
+  serial-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        problem: [00_dense_la_lu_decomp, 01_dense_la_solve, 02_dense_la_gemm, 03_dense_la_axpy, 04_dense_la_gemv, 05_fft_inverse_fft, 06_fft_dft, 07_fft_fft_conjugate, 08_fft_split_fft, 09_fft_fft_out_of_place, 10_geometry_convex_hull, 11_geometry_convex_hull_perimeter, 12_geometry_smallest_triangle, 13_geometry_closest_pair_2d, 14_geometry_closest_pair_1d, 15_graph_edge_count, 16_graph_largest_component, 17_graph_highest_degree, 18_graph_count_components, 19_graph_shortest_path, 20_histogram_pixel_histogram, 21_histogram_bin_0-100, 22_histogram_count_quadrants, 23_histogram_first_letter_counts, 24_histogram_count_quartile, 25_reduce_xor, 26_reduce_product_of_inverses, 27_reduce_average, 28_reduce_smallest_odd_number, 29_reduce_sum_of_min_of_pairs, 30_scan_prefix_sum, 31_scan_scan_with_min_function, 32_scan_sum_of_prefix_sum_array, 33_scan_reverse_prefix_sum,, 34_scan_largest_contiguous_subarray_sum, 35_search_search_for_last_struct_by_key, 36_search_check_if_array_contains_value, 37_search_find_the_closest_number_to_pi,, 38_search_find_the_first_even_number, 39_search_xor_contains, 40_sort_sort_an_array_of_complex_numbers_by_magnitude, 41_sort_k-th_smallest_element, 42_sort_sorted_ranks, 43_sort_sort_an_array_of_structs_by_key,, 44_sort_sort_non-zero_elements, 45_sparse_la_sparse_solve, 46_sparse_la_spmm, 47_sparse_la_spmv, 48_sparse_la_sparse_axpy, 49_sparse_la_sparse_lu_decomp, 50_stencil_xor_kernel, 51_stencil_edge_kernel, 52_stencil_1d_jacobi_3-point_stencil, 53_stencil_2d_jacobi_5-point_stencil, 54_stencil_game_of_life, 55_transform_relu, 56_transform_negate_odds, 57_transform_inverse_offset, 58_transform_squaring, 59_transform_map_function]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install tqdm
+
+      - name: Run CPU test for ${{ matrix.problem }}
+        run: bash test/test-cpu.bash "${{ matrix.problem }}"

prompts/create-serial-tests.py

@@ -24,7 +24,7 @@ def get_return_type(code: str) -> str:
     # then return the type
     lines = code.split('\n')
     for line in lines:
-        if line.strip().endswith(') {'):
+        if "NO_INLINE correct" in line and line.strip().endswith(') {'):
             return line.split()[0]
 
 def main():
@@ -45,7 +45,8 @@ def main():
             continue
 
         baseline = get_file_contents(baseline_fpath)
-        impl = get_substr_after_first_of(baseline, ') {')
+        func_start = get_substr_after_first_of(baseline, 'NO_INLINE correct')
+        impl = get_substr_after_first_of(func_start, ') {')
         return_type = get_return_type(baseline)
         prompt['outputs'] = [
             impl, 

test/README.md

@@ -0,0 +1,3 @@
+# Tests
+
+Testing of the benchmark. Currently only tests the sequential CPU capabilities.

test/test-cpu.bash

@@ -0,0 +1,32 @@
+#!/bin/bash
+# Uses the baseline implementations to test the CPU capabilities of the system.
+
+# usage: bash test/test-cpu.bash <?problem>
+if [ $# -eq 0 ]; then
+    echo "No problem specified. Using default: 'all'."
+    PROBLEM_ARG=""
+else
+    PROBLEM_ARG="--problem $1"
+fi
+
+# First, use the baseline implementations to mimic LLM outputs.
+python prompts/create-serial-tests.py drivers/cpp/benchmarks prompts/generation-prompts.json serial-generations.json
+
+# make sure the model drivers are built
+cd drivers
+cd cpp
+make
+cd ..
+
+# Run the drivers using these generations
+python run-all.py \
+    ../serial-generations.json \
+    --output results.json \
+    --launch-configs launch-configs.json \
+    --problem-sizes problem-sizes.json \
+    --yes-to-all \
+    --include-models serial \
+    ${PROBLEM_ARG} \
+    --build-timeout 60 \
+    --run-timeout 120 \
+    --log info