Skip to content

Commit 40e044f

Browse files
vringarvringar
committed
feat(ci): use pytest-split for optimal test distribution
1 parent d327d69 commit 40e044f

4 files changed

Lines changed: 178 additions & 20 deletions

File tree

‎.github/workflows/run-tests.yaml‎

Lines changed: 42 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -27,37 +27,60 @@ jobs:
2727
runs-on: ubuntu-latest
2828
strategy:
2929
matrix:
30-
test-groups:
31-
[
32-
"test/test_[a-e]*",
33-
"test/test_[f-h]*",
34-
"test/test_[i-o,q-r,t-z]*",
35-
"test/test_[p]*",
36-
"test/test_[s]*",
37-
"test/storage/*",
38-
"test/extension/*",
39-
]
30+
group: [1, 2, 3, 4, 5, 6, 7]
4031
fail-fast: false
4132
steps:
4233
- uses: actions/checkout@v4
4334
- uses: ./.github/actions/setup
4435
- run: ./scripts/ci.sh
4536
env:
4637
DISPLAY: ":99.0"
47-
TESTS: ${{ matrix.test-groups }}
48-
- name: Test Report
49-
uses: dorny/test-reporter@v1
38+
GROUP: ${{ matrix.group }}
39+
SPLITS: 7
40+
- name: Test failure summary
41+
if: always()
5042
continue-on-error: true
51-
if: always() # run this step even if previous step failed
52-
with:
53-
name: ${{matrix.test-groups}} # Name of the check run which will be created
54-
path: junit-report.xml # Path to test results
55-
reporter: java-junit # Format of test results
43+
run: |
44+
python - <<'PYEOF'
45+
import xml.etree.ElementTree as ET, os, sys
46+
if not os.path.exists("junit-report.xml"):
47+
sys.exit(0)
48+
tree = ET.parse("junit-report.xml")
49+
root = tree.getroot()
50+
# Collect suite-level stats
51+
suites = list(root.iter("testsuite"))
52+
total = sum(int(s.get("tests", 0)) for s in suites)
53+
fails = sum(int(s.get("failures", 0)) for s in suites)
54+
errs = sum(int(s.get("errors", 0)) for s in suites)
55+
bad = fails + errs
56+
if bad == 0:
57+
sys.exit(0)
58+
with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
59+
f.write(f"### Group ${{ matrix.group }}: {bad}/{total} test(s) failed\n\n")
60+
for tc in root.iter("testcase"):
61+
for tag in ("failure", "error"):
62+
elem = tc.find(tag)
63+
if elem is None:
64+
continue
65+
name = tc.get("name", "?")
66+
cls = tc.get("classname", "")
67+
short = cls.rsplit(".", 1)[-1] if cls else ""
68+
etype = elem.get("type", tag)
69+
msg = elem.get("message", "")
70+
trace = (elem.text or "").rstrip()
71+
# Summary line shows test + exception type
72+
f.write(f"<details>\n<summary>")
73+
f.write(f"<code>{short}::{name}</code> — {etype}")
74+
if msg:
75+
oneliner = msg.split("\n")[0][:120]
76+
f.write(f": {oneliner}")
77+
f.write(f"</summary>\n\n```\n{trace}\n```\n</details>\n\n")
78+
PYEOF
5679
- name: Upload coverage artifact
5780
uses: actions/upload-artifact@v4
5881
if: always()
5982
with:
60-
name: coverage-${{ strategy.job-index }}
83+
name: coverage-${{ matrix.group }}
6184
path: coverage.xml
6285

6386
upload-coverage:

‎.test_durations‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
{
2+
"test/extension/test_logging.py::test_extension_logging": 13.18,
3+
"test/extension/test_startup_timeout.py::test_extension_startup_timeout": 68.22,
4+
"test/storage/test_arrow_cache.py::test_arrow_cache": 9.49,
5+
"test/storage/test_storage_controller.py::test_arrow_provider": 15.25,
6+
"test/storage/test_storage_controller.py::test_startup_and_shutdown": 15.16,
7+
"test/storage/test_storage_providers.py::test_basic_access[memory_arrow]": 0.0,
8+
"test/storage/test_storage_providers.py::test_basic_access[memory_structured]": 0.0,
9+
"test/storage/test_storage_providers.py::test_basic_access[sqlite]": 0.01,
10+
"test/storage/test_storage_providers.py::test_basic_unstructured_storing[leveldb]": 0.24,
11+
"test/storage/test_storage_providers.py::test_basic_unstructured_storing[local_gzip]": 0.0,
12+
"test/storage/test_storage_providers.py::test_basic_unstructured_storing[memory_unstructured]": 0.0,
13+
"test/storage/test_storage_providers.py::test_local_arrow_storage_provider": 0.1,
14+
"test/test_callback.py::test_local_callbacks": 23.17,
15+
"test/test_callstack_instrument.py::test_http_stacktrace": 0.12,
16+
"test/test_crawl.py::test_browser_profile_coverage": 259.65,
17+
"test/test_custom_function_command.py::test_custom_function": 23.16,
18+
"test/test_dataclass_validations.py::test_browser_type": 0.0,
19+
"test/test_dataclass_validations.py::test_display_mode": 0.0,
20+
"test/test_dataclass_validations.py::test_failure_limit": 0.0,
21+
"test/test_dataclass_validations.py::test_log_file_extension": 0.0,
22+
"test/test_dataclass_validations.py::test_num_browser_crawl_config": 0.12,
23+
"test/test_dataclass_validations.py::test_save_content_type": 0.0,
24+
"test/test_dataclass_validations.py::test_tp_cookies_opt": 0.0,
25+
"test/test_dns_instrument.py::test_name_resolution": 23.04,
26+
"test/test_extension.py::TestExtension::test_canvas_fingerprinting": 23.16,
27+
"test/test_extension.py::TestExtension::test_document_cookie_instrumentation": 23.16,
28+
"test/test_extension.py::TestExtension::test_extension_gets_correct_visit_id": 46.3,
29+
"test/test_extension.py::TestExtension::test_js_call_stack": 23.16,
30+
"test/test_extension.py::TestExtension::test_js_time_stamp": 23.16,
31+
"test/test_extension.py::TestExtension::test_property_enumeration": 23.16,
32+
"test/test_extension.py::TestExtension::test_webrtc_localip": 23.16,
33+
"test/test_extension.py::test_audio_fingerprinting": 0.0,
34+
"test/test_http_instrumentation.py::TestHTTPInstrument::test_service_worker_requests": 23.18,
35+
"test/test_http_instrumentation.py::TestHTTPInstrument::test_worker_script_requests": 23.19,
36+
"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_binary_post_data": 23.18,
37+
"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_file_upload": 0.14,
38+
"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_post_data_ajax": 23.04,
39+
"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_post_data_ajax_no_key_value": 23.18,
40+
"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_post_data_ajax_no_key_value_base64_encoded": 23.04,
41+
"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_post_data_multipart_formdata": 23.18,
42+
"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_post_data_text_plain": 23.04,
43+
"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_post_data_x_www_form_urlencoded": 23.18,
44+
"test/test_http_instrumentation.py::TestPOSTInstrument::test_record_post_formdata": 23.04,
45+
"test/test_http_instrumentation.py::test_cache_hits_recorded": 38.19,
46+
"test/test_http_instrumentation.py::test_content_saving": 23.04,
47+
"test/test_http_instrumentation.py::test_document_saving": 33.18,
48+
"test/test_http_instrumentation.py::test_javascript_saving": 23.19,
49+
"test/test_http_instrumentation.py::test_page_visit[False]": 23.04,
50+
"test/test_http_instrumentation.py::test_page_visit[True]": 38.18,
51+
"test/test_js_instrument.py::TestJSInstrument::test_instrument_object": 23.2,
52+
"test/test_js_instrument.py::TestJSInstrumentByPython::test_instrument_object": 23.04,
53+
"test/test_js_instrument.py::TestJSInstrumentExistingWindowProperty::test_instrument_object": 23.2,
54+
"test/test_js_instrument.py::TestJSInstrumentMockWindowProperty::test_instrument_object": 33.2,
55+
"test/test_js_instrument.py::TestJSInstrumentNonExistingWindowProperty::test_instrument_object": 23.2,
56+
"test/test_js_instrument.py::TestJSInstrumentRecursiveProperties::test_instrument_object": 23.04,
57+
"test/test_js_instrument_py.py::test_api_collection_fingerprinting": 0.0,
58+
"test/test_js_instrument_py.py::test_api_instances_on_window": 0.0,
59+
"test/test_js_instrument_py.py::test_api_instances_on_window_with_properties": 0.0,
60+
"test/test_js_instrument_py.py::test_api_module_specific_properties": 0.0,
61+
"test/test_js_instrument_py.py::test_api_passing_partial_log_settings": 0.01,
62+
"test/test_js_instrument_py.py::test_api_two_keys_in_shortcut": 0.0,
63+
"test/test_js_instrument_py.py::test_api_whole_module": 0.0,
64+
"test/test_js_instrument_py.py::test_complete_pass": 0.01,
65+
"test/test_js_instrument_py.py::test_merge_and_validate_multiple_overlap_properties": 0.0,
66+
"test/test_js_instrument_py.py::test_merge_and_validate_multiple_overlap_properties_to_instrument_properties_to_exclude": 0.0,
67+
"test/test_js_instrument_py.py::test_merge_diff_instrumented_names": 0.0,
68+
"test/test_js_instrument_py.py::test_merge_multiple_duped_properties": 0.0,
69+
"test/test_js_instrument_py.py::test_merge_multiple_duped_properties_different_log_settings": 0.0,
70+
"test/test_js_instrument_py.py::test_merge_when_log_settings_is_null": 0.0,
71+
"test/test_js_instrument_py.py::test_validate_bad__log_settings_invalid": 0.0,
72+
"test/test_js_instrument_py.py::test_validate_bad__log_settings_missing": 0.0,
73+
"test/test_js_instrument_py.py::test_validate_bad__missing_object": 0.0,
74+
"test/test_js_instrument_py.py::test_validate_bad__not_a_list": 0.0,
75+
"test/test_js_instrument_py.py::test_validate_good": 0.03,
76+
"test/test_js_instrument_py.py::test_validated_bad__missing_instrumentedName": 0.01,
77+
"test/test_mp_logger.py::test_child_process_logging": 13.01,
78+
"test/test_mp_logger.py::test_child_process_with_exception": 13.02,
79+
"test/test_mp_logger.py::test_multiple_instances": 28.21,
80+
"test/test_mp_logger.py::test_multiprocess": 14.1,
81+
"test/test_profile.py::test_crash_during_init": 38.07,
82+
"test/test_profile.py::test_crash_profile": 43.06,
83+
"test/test_profile.py::test_dump_profile_command": 88.12,
84+
"test/test_profile.py::test_load_tar_file": 0.13,
85+
"test/test_profile.py::test_profile_error": 48.06,
86+
"test/test_profile.py::test_profile_recovery[on_crash-stateful-with_seed_tar]": 103.71,
87+
"test/test_profile.py::test_profile_recovery[on_crash-stateful-without_seed_tar]": 113.69,
88+
"test/test_profile.py::test_profile_recovery[on_crash-stateless-with_seed_tar]": 108.57,
89+
"test/test_profile.py::test_profile_recovery[on_crash_during_launch-stateful-with_seed_tar]": 128.68,
90+
"test/test_profile.py::test_profile_recovery[on_crash_during_launch-stateful-without_seed_tar]": 133.74,
91+
"test/test_profile.py::test_profile_recovery[on_crash_during_launch-stateless-with_seed_tar]": 138.54,
92+
"test/test_profile.py::test_profile_recovery[on_normal_operation-stateful-with_seed_tar]": 93.54,
93+
"test/test_profile.py::test_profile_recovery[on_normal_operation-stateful-without_seed_tar]": 93.53,
94+
"test/test_profile.py::test_profile_recovery[on_normal_operation-stateless-with_seed_tar]": 93.39,
95+
"test/test_profile.py::test_profile_recovery[on_timeout-stateful-with_seed_tar]": 98.64,
96+
"test/test_profile.py::test_profile_recovery[on_timeout-stateful-without_seed_tar]": 103.69,
97+
"test/test_profile.py::test_profile_recovery[on_timeout-stateless-with_seed_tar]": 108.53,
98+
"test/test_profile.py::test_profile_saved_when_launch_crashes": 118.28,
99+
"test/test_profile.py::test_save_incomplete_profile_error": 22.28,
100+
"test/test_profile.py::test_saving": 36.03,
101+
"test/test_profile.py::test_seed_persistence": 23.04,
102+
"test/test_simple_commands.py::test_browse_http_table_valid[headless]": 43.19,
103+
"test/test_simple_commands.py::test_browse_http_table_valid[xvfb]": 53.2,
104+
"test/test_simple_commands.py::test_browse_site_visits_table_valid[headless]": 23.04,
105+
"test/test_simple_commands.py::test_browse_site_visits_table_valid[xvfb]": 33.05,
106+
"test/test_simple_commands.py::test_browse_wrapper_http_table_valid[headless]": 43.19,
107+
"test/test_simple_commands.py::test_browse_wrapper_http_table_valid[xvfb]": 53.21,
108+
"test/test_simple_commands.py::test_dump_page_source_valid[headless]": 23.04,
109+
"test/test_simple_commands.py::test_dump_page_source_valid[xvfb]": 43.2,
110+
"test/test_simple_commands.py::test_get_http_tables_valid[headless]": 23.04,
111+
"test/test_simple_commands.py::test_get_http_tables_valid[xvfb]": 33.19,
112+
"test/test_simple_commands.py::test_get_site_visits_table_valid[headless]": 33.2,
113+
"test/test_simple_commands.py::test_get_site_visits_table_valid[xvfb]": 38.2,
114+
"test/test_simple_commands.py::test_recursive_dump_page_source_valid[headless]": 23.04,
115+
"test/test_simple_commands.py::test_recursive_dump_page_source_valid[xvfb]": 43.19,
116+
"test/test_simple_commands.py::test_save_screenshot_valid[headless]": 23.07,
117+
"test/test_simple_commands.py::test_save_screenshot_valid[xvfb]": 43.21,
118+
"test/test_storage_vectors.py::test_js_profile_cookies": 33.41,
119+
"test/test_task_manager.py::test_assertion_error_propagation[False-expectation0]": 63.22,
120+
"test/test_task_manager.py::test_assertion_error_propagation[True-expectation1]": 73.23,
121+
"test/test_task_manager.py::test_failure_limit_exceeded": 23.04,
122+
"test/test_task_manager.py::test_failure_limit_reset": 38.18,
123+
"test/test_task_manager.py::test_failure_limit_value": 0.14,
124+
"test/test_timer.py::test_command_duration": 28.19,
125+
"test/test_webdriver_utils.py::test_parse_neterror": 0.15,
126+
"test/test_webdriver_utils.py::test_parse_neterror_integration": 18.04,
127+
"test/test_xvfb_browser.py::test_display_shutdown": 78.24
128+
}

‎environment.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ dependencies:
4040
- dataclasses-json==0.6.7
4141
- domain-utils==0.7.1
4242
- jsonschema==4.23.0
43+
- pytest-split==0.11.0
4344
- tranco==0.8.1
4445
- types-pyyaml==6.0.12.20241230
4546
- types-redis==4.6.0.20241004

‎scripts/ci.sh‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
11
#!/bin/bash
22

3-
python -m pytest --cov=openwpm --junit-xml=junit-report.xml --cov-report=xml $TESTS -s -v --durations=10;
3+
if [ -n "$GROUP" ] && [ -n "$SPLITS" ]; then
4+
# CI mode: use pytest-split for optimal distribution
5+
python -m pytest --splits "$SPLITS" --group "$GROUP" --splitting-algorithm least_duration --cov=openwpm --junit-xml=junit-report.xml --cov-report=xml -s -v --durations=10
6+
else
7+
# Local mode: run specific tests or all
8+
python -m pytest --cov=openwpm --junit-xml=junit-report.xml --cov-report=xml $TESTS -s -v --durations=10
9+
fi

0 commit comments

Comments
 (0)