Skip to content

Commit d33d7f3

Browse files
stellaprinsstellaprins
authored
Add CI for regression tests (#25)
* add regression tests workflow * add optional argument to display results in runRegressionTests function * add test_info array and save error strings for regression tests * add show_results input to runRegressionTests * refactor regression test functions and add artifact saving * use show_test_results in unit tests workflow * reorganize regression test workflow * update runRegressionTests function to return test_struct * save regression test results in test_struct.mat * fix runRegressionTests input * add path for Generate data (regression tests) * run on all regression and unit tests * replace runUnitTests_artifact with save_artifact and remove unused functions * swap around the list of failed tests and summary (N failed / passed) I realised that when you open the CI output (if it has failed) it directs you to the end of the output, so it makes more sense to put this information at the end after all
1 parent 23d93e5 commit d33d7f3

6 files changed

Lines changed: 183 additions & 151 deletions

File tree

‎.github/workflows/run_tests.yml‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
name: Run Tests
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
branches:
9+
- main
10+
11+
jobs:
12+
unit-tests:
13+
name: Unit tests
14+
runs-on: ubuntu-latest
15+
steps:
16+
- name: Check out repository
17+
uses: actions/checkout@v4
18+
19+
- name: Set up MATLAB
20+
uses: matlab-actions/setup-matlab@v2
21+
with:
22+
products: Signal_Processing_Toolbox
23+
cache: true
24+
25+
- name: Run unit tests
26+
uses: matlab-actions/run-command@v2
27+
with:
28+
command: |
29+
addpath("${{ github.workspace }}/k-Wave");
30+
cd("${{ github.workspace }}/k-Wave/testing/unit");
31+
test_struct = runUnitTests("",false);
32+
save('test_struct.mat', 'test_struct');
33+
startup-options: -nojvm -logfile output.log
34+
35+
- name: Create artifact
36+
uses: matlab-actions/run-command@v2
37+
with:
38+
command: |
39+
addpath("${{ github.workspace }}/k-Wave/testing");
40+
cd("${{ github.workspace }}/k-Wave/testing/unit");
41+
load('test_struct.mat', 'test_struct');
42+
save_artifact(test_struct);
43+
startup-options: -nojvm
44+
45+
- name: Upload artifact
46+
uses: actions/upload-artifact@v4
47+
with:
48+
name: unit_test_results
49+
path: ${{ github.workspace }}/k-Wave/testing/unit/test_results.json
50+
51+
- name: Test results
52+
uses: matlab-actions/run-command@v2
53+
with:
54+
command: |
55+
addpath("${{ github.workspace }}/k-Wave/testing");
56+
cd("${{ github.workspace }}/k-Wave/testing/unit");
57+
load('test_struct.mat', 'test_struct');
58+
show_test_results(test_struct);
59+
disp(' ');
60+
disp('NOTE:');
61+
disp('Test output details are in the "Run unit tests" section of the workflow.');
62+
disp('You can also download a JSON summary from the "Upload Artifact" section in your CI logs or dashboard.');
63+
startup-options: -nojvm
64+
65+
regression-tests:
66+
name: Regression tests
67+
runs-on: ubuntu-latest
68+
steps:
69+
- name: Check out repository
70+
uses: actions/checkout@v4
71+
72+
- name: Set up MATLAB
73+
uses: matlab-actions/setup-matlab@v2
74+
with:
75+
products: Signal_Processing_Toolbox
76+
cache: true
77+
78+
- name: Generate data
79+
uses: matlab-actions/run-command@v2
80+
with:
81+
command: |
82+
addpath("${{ github.workspace }}/k-Wave");
83+
cd("${{ github.workspace }}/k-Wave/testing/regression");
84+
generateRegressionData
85+
startup-options: -nojvm
86+
87+
- name: Run regression tests
88+
uses: matlab-actions/run-command@v2
89+
with:
90+
command: |
91+
addpath("${{ github.workspace }}/k-Wave");
92+
cd("${{ github.workspace }}/k-Wave/testing/regression");
93+
test_struct = runRegressionTests("${{ github.workspace }}/k-Wave/testing/regression",false);
94+
save('test_struct.mat', 'test_struct');
95+
startup-options: -nojvm
96+
97+
- name: Create artifact
98+
uses: matlab-actions/run-command@v2
99+
with:
100+
command: |
101+
addpath("${{ github.workspace }}/k-Wave/testing");
102+
cd("${{ github.workspace }}/k-Wave/testing/regression");
103+
load('test_struct.mat', 'test_struct');
104+
save_artifact(test_struct);
105+
startup-options: -nojvm
106+
107+
- name: Upload artifact
108+
uses: actions/upload-artifact@v4
109+
with:
110+
name: regression_test_results
111+
path: ${{ github.workspace }}/k-Wave/testing/regression/test_results.json
112+
113+
- name: Test results
114+
uses: matlab-actions/run-command@v2
115+
with:
116+
command: |
117+
addpath("${{ github.workspace }}/k-Wave/testing");
118+
cd("${{ github.workspace }}/k-Wave/testing/regression");
119+
load('test_struct.mat', 'test_struct');
120+
show_test_results(test_struct);
121+
disp(' ');
122+
disp('NOTE:');
123+
disp('Test output details are in the "Run regression tests" section of the workflow.');
124+
disp('You can also download a JSON summary from the "Upload Artifact" section in your CI logs or dashboard.');
125+
startup-options: -nojvm

‎.github/workflows/unit_tests.yml‎

Lines changed: 0 additions & 60 deletions
This file was deleted.

‎k-Wave/testing/regression/generateRegressionData.m‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -410,6 +410,7 @@
410410
load([tempdir 'generate_regression_data_TEMP_VARS']);
411411

412412
% move to the testing directory
413+
if ~exist(testing_dir, 'dir'), mkdir(testing_dir); end
413414
cd(testing_dir);
414415

415416
% generate computer info and add precision

‎k-Wave/testing/regression/runRegressionTests.m‎

Lines changed: 32 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
function runRegressionTests(data_folder)
1+
function test_struct = runRegressionTests(data_folder, show_results)
22
%RUNREGRESSIONTESTS Run MATLAB regression tests.
33
%
44
% DESCRIPTION:
@@ -18,6 +18,7 @@ function runRegressionTests(data_folder)
1818
%
1919
% INPUTS:
2020
% data_folder - string of folder containing regression data
21+
% show_results - logical flag to display results (default: true)
2122
%
2223
% ABOUT:
2324
% author - Bradley Treeby
@@ -44,6 +45,11 @@ function runRegressionTests(data_folder)
4445
%#ok<*NASGU>
4546
%#ok<*IDISVAR>
4647

48+
% Set defaults for optional arguments
49+
if nargin < 2 || isempty(show_results)
50+
show_results = true;
51+
end
52+
4753
% start the timer
4854
regression_start_time = datetime('now');
4955

@@ -77,6 +83,9 @@ function runRegressionTests(data_folder)
7783
% keep a list of whether the test passed or failed
7884
test_result = false(num_files, 1);
7985

86+
% preallocate cell array for test_info with empty strings by default
87+
test_info = repmat({''}, num_files, 1);
88+
8089
% =========================================================================
8190
% RUN TESTS
8291
% =========================================================================
@@ -93,7 +102,7 @@ function runRegressionTests(data_folder)
93102
save(temp_var_filename, ...
94103
'DOUBLE_COMPARISON_THRESHOLD', 'SINGLE_COMPARISON_THRESHOLD', ...
95104
'regression_test_folder', 'temp_var_filename', 'regression_start_time', ...
96-
'filename_index', 'filenames', 'fn', 'num_files', 'test_result');
105+
'filename_index', 'filenames', 'fn', 'num_files', 'test_result', 'show_results');
97106

98107
% display the filename
99108
disp(' ');
@@ -238,10 +247,14 @@ function runRegressionTests(data_folder)
238247
test_pass = true;
239248
fprintf('passed');
240249
end
250+
% save the error string
251+
error_str = sprintf(' (L_inf = %e)\n', L_inf);
241252

242253
% display the error
243-
fprintf(' (L_inf = %e)\n', L_inf);
244-
254+
fprintf('%s', error_str);
255+
256+
% save L_inf as test_info
257+
test_info{filename_index} = error_str;
245258
end
246259

247260
% clear the variables just in case
@@ -256,69 +269,27 @@ function runRegressionTests(data_folder)
256269

257270
% store test result
258271
test_result(filename_index) = test_pass_overall;
259-
260272
end
261273

262274
% =========================================================================
263-
% DISPLAY SUMMARY
275+
% CREATE OUTPUT
264276
% =========================================================================
265277

266-
% get information about PC
278+
completion_time = scaleTime(seconds(datetime('now') - regression_start_time));
267279
comp_info = getComputerInfo;
280+
info = comp_info;
281+
info.completion_time = completion_time;
268282

269-
% get k-Wave version
270-
eval('cur_dir = pwd; cd(getkWavePath(''private'')); kwave_ver = getkWaveVersion; cd(cur_dir);');
271-
272-
% display test header
273-
disp(' ');
274-
disp('-------------------------------------------------------------------------------------');
275-
disp(' _ __ __ _____ _ ');
276-
disp(' | | __ \ \ / /_ ___ _____ |_ _|__ ___| |_ ___ _ __ ');
277-
disp(' | |/ /___\ \ /\ / / _` \ \ / / _ \ | |/ _ \/ __| __/ _ \ ''__|');
278-
disp(' | <_____\ V V / (_| |\ V / __/ | | __/\__ \ || __/ | ');
279-
disp(' |_|\_\ \_/\_/ \__,_| \_/ \___| |_|\___||___/\__\___|_| ');
280-
disp(' ');
281-
disp('-------------------------------------------------------------------------------------');
282-
disp(' ');
283-
disp(['DATE: ' comp_info.date]);
284-
disp(['HOST NAME: ' comp_info.computer_name]);
285-
disp(['USER NAME: ' comp_info.user_name]);
286-
disp(['O/S TYPE: ' comp_info.operating_system_type]);
287-
disp(['O/S: ' comp_info.operating_system]);
288-
disp(['MATLAB VERSION: ' comp_info.matlab_version]);
289-
disp(['TESTED K-WAVE VERSION: ' kwave_ver]);
290-
disp(['TESTS COMPLETED IN: ' scaleTime(seconds(datetime('now') - regression_start_time))]);
291-
disp(' ');
292-
293-
% display individual test results
294-
disp('REGRESSION TEST RESULTS:');
295-
for filename_index = 1:length(filenames)
296-
297-
% trim the filename and add number
298-
fn = filenames{filename_index};
299-
fn = [num2str(filename_index, '%02.f') ' ' fn(1:end - 4), ':'];
300-
301-
% add some spaces to align results
302-
fn = sprintf('%-50s', fn);
303-
304-
% append the test result
305-
if test_result(filename_index)
306-
disp([' ' fn 'passed']);
307-
else
308-
disp([' ' fn 'failed']);
309-
end
310-
end
311-
312-
% display test summary
313-
disp(' ');
314-
if all(test_result)
315-
disp('ALL REGRESSION TESTS PASSED!');
316-
else
317-
disp('REGRESSION TEST FAILED...');
318-
end
283+
% create results struct
284+
test_struct = struct( ...
285+
'info', info, ...
286+
'results', struct('test', filenames(:), 'pass', num2cell(test_result(:)), 'test_info', test_info(:)) ...
287+
);
319288

320-
disp(' ');
321-
disp('-------------------------------------------------------------------------------------');
289+
% =========================================================================
290+
% SHOW RESULTS
291+
% =========================================================================
322292

323-
% remove temp data
324-
delete(temp_var_filename);
293+
if show_results
294+
show_test_results(test_struct);
295+
end

k-Wave/testing/unit/runUnitTests_artifact.m renamed to k-Wave/testing/save_artifact.m

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
function runUnitTests_artifact(test_struct)
2-
%RUNUNITTESTS_ARTIFACT
1+
function save_artifact(test_struct)
2+
%SAVE_ARTIFACT
33
%
44
% DESCRIPTION:
5-
% runUnitTests_actions processes the provided test_struct, saves the results
5+
% save_artifact processes the provided test_struct, saves the results
66
% as a test_results.json artifact.
77
%
88

0 commit comments

Comments
 (0)