From d162bbef6c00d05864ba107131a7a0bf9c50092d Mon Sep 17 00:00:00 2001 From: Razvan-Liviu Varzaru Date: Mon, 20 Jul 2026 10:25:00 +0300 Subject: [PATCH] MDBF-1185: Modernize MSAN/UBSAN CI with Clang 22 Phase in Clang 22 MSAN/UBSAN builders and update their worker assignments. Remove the obsolete Clang 20 builders and the MSAN debug configuration. Add an experimental mode to evaluate new suites and options with the Clang 22 builders. Extend Spider coverage to the non-debug MSAN build after it proved quick and stable locally. Evaluate cursor and view protocols across MSAN/UBSAN debug and non-debug builds. UBSAN mainly reproduced known minor failures. MSAN view exposed a failure that was also reproducible without MSAN, so run cursor and view tests on the faster non-debug MSAN builder. Use -Og for debug sanitizer builds to reduce instrumentation and improve CI runtime while retaining useful debugging information. Recognize DuckDB as a plugin, but disable it under MSAN/UBSAN until its linking issue is fixed. --- configuration/builders/sequences/helpers.py | 96 +++++++++++++++++++ .../builders/sequences/sanitizers.py | 78 +++++++++++---- .../steps/generators/cmake/options.py | 1 + constants.py | 6 +- master-migration/master.cfg | 74 +++++++++++--- master-migration/workers.yaml | 2 +- 6 files changed, 224 insertions(+), 33 deletions(-) diff --git a/configuration/builders/sequences/helpers.py b/configuration/builders/sequences/helpers.py index d907886b9..23ee37082 100644 --- a/configuration/builders/sequences/helpers.py +++ b/configuration/builders/sequences/helpers.py @@ -65,6 +65,102 @@ def get_mtr_normal_steps( return steps +def get_mtr_cursor_steps( + jobs, + path_to_test_runner: PurePath, + halt_on_failure: bool = True, + step_wrapping_fn=lambda step: step, + additional_mtr_options: list[MTROption] = [], + env_vars: list[tuple] = [], +): + steps = [] + steps.append( + step_wrapping_fn( + ShellStep( + MTRTest( + name="cursor", + save_logs_path=MTR_PATH_TO_SAVE_LOGS / "cursor", + workdir=path_to_test_runner, + testcase=MTRGenerator( + flags=[ + MTROption(MTR.VERBOSE_RESTART, True), + MTROption(MTR.FORCE, True), + MTROption(MTR.CURSOR_PROTOCOL, True), + MTROption(MTR.MAX_SAVE_CORE, 2), + MTROption(MTR.MAX_SAVE_DATADIR, 1), + MTROption(MTR.MAX_TEST_FAIL, 20), + MTROption(MTR.PARALLEL, jobs * 2), + MTROption(MTR.VARDIR, "/dev/shm/cursor"), + MTROption( + MTR.XML_REPORT, MTR_PATH_TO_SAVE_LOGS / "cursor.xml" + ), + ] + + additional_mtr_options, + suite_collection=TestSuiteCollection( + [ + SUITE.MAIN, + ] + ), + ), + ), + options=StepOptions( + haltOnFailure=halt_on_failure, descriptionDone="MTR cursor" + ), + env_vars=env_vars, + ) + ) + ) + return steps + + +def get_mtr_view_steps( + jobs, + path_to_test_runner: PurePath, + halt_on_failure: bool = True, + step_wrapping_fn=lambda step: step, + additional_mtr_options: list[MTROption] = [], + env_vars: list[tuple] = [], +): + steps = [] + steps.append( + step_wrapping_fn( + ShellStep( + MTRTest( + name="view", + save_logs_path=MTR_PATH_TO_SAVE_LOGS / "view", + workdir=path_to_test_runner, + testcase=MTRGenerator( + flags=[ + MTROption(MTR.VERBOSE_RESTART, True), + MTROption(MTR.FORCE, True), + MTROption(MTR.VIEW_PROTOCOL, True), + MTROption(MTR.MAX_SAVE_CORE, 2), + MTROption(MTR.MAX_SAVE_DATADIR, 1), + MTROption(MTR.MAX_TEST_FAIL, 20), + MTROption(MTR.PARALLEL, jobs * 2), + MTROption(MTR.VARDIR, "/dev/shm/view"), + MTROption( + MTR.XML_REPORT, MTR_PATH_TO_SAVE_LOGS / "view.xml" + ), + ] + + additional_mtr_options, + suite_collection=TestSuiteCollection( + [ + SUITE.MAIN, + ] + ), + ), + ), + options=StepOptions( + haltOnFailure=halt_on_failure, descriptionDone="MTR view" + ), + env_vars=env_vars, + ) + ) + ) + return steps + + def get_mtr_rocksdb_steps( jobs, path_to_test_runner: PurePath, diff --git a/configuration/builders/sequences/sanitizers.py b/configuration/builders/sequences/sanitizers.py index 8ed83f483..cb16aea6a 100644 --- a/configuration/builders/sequences/sanitizers.py +++ b/configuration/builders/sequences/sanitizers.py @@ -6,9 +6,11 @@ InContainer, ) from configuration.builders.sequences.helpers import ( + get_mtr_cursor_steps, get_mtr_normal_steps, get_mtr_s3_steps, get_mtr_spider_steps, + get_mtr_view_steps, mtr_reporter, save_mtr_logs, ) @@ -34,6 +36,7 @@ def asan_ubsan( config: DockerConfig, jobs: int, isDebugBuildType: bool, + isExperimental: bool, ): sequence = BuildSequence() @@ -73,10 +76,17 @@ def asan_ubsan( CMakeOption(WITH.UBSAN, True), CMakeOption(WITH.UNIT_TESTS, False), CMakeOption(PLUGIN.COLUMNSTORE_STORAGE_ENGINE, False), + CMakeOption(PLUGIN.DUCKDB_STORAGE_ENGINE, False), ] if isDebugBuildType: - flags.append(CMakeOption(CMAKE.BUILD_TYPE, BuildType.DEBUG)) - flags.append(CMakeOption(WITH.DBUG_TRACE, False)) + flags.extend( + [ + CMakeOption(CMAKE.BUILD_TYPE, BuildType.DEBUG), + CMakeOption(WITH.DBUG_TRACE, False), + CMakeOption(CMAKE.C_FLAGS, "-Og"), + CMakeOption(CMAKE.CXX_FLAGS, "-Og"), + ] + ) sequence.add_step( InContainer( @@ -164,21 +174,25 @@ def asan_ubsan( docker_environment=config, step=step ), ) - + [ - save_mtr_logs( - step_wrapping_fn=lambda step: InContainer( - docker_environment=config, step=step - ), - ), - mtr_reporter( - step_wrapping_fn=lambda step: InContainer( - docker_environment=config, step=step - ), - ), - ] ): sequence.add_step(step) + sequence.add_step( + save_mtr_logs( + step_wrapping_fn=lambda step: InContainer( + docker_environment=config, step=step + ), + ) + ) + + sequence.add_step( + mtr_reporter( + step_wrapping_fn=lambda step: InContainer( + docker_environment=config, step=step + ), + ), + ) + return sequence @@ -186,6 +200,7 @@ def msan( config: DockerConfig, jobs: int, isDebugBuildType: bool, + isExperimental: bool, ): sequence = BuildSequence() @@ -214,13 +229,20 @@ def msan( CMakeOption(WITH.ZLIB, "bundled"), CMakeOption(WITH.SYSTEMD, "no"), CMakeOption(PLUGIN.COLUMNSTORE_STORAGE_ENGINE, False), - CMakeOption(PLUGIN.SPIDER_STORAGE_ENGINE, isDebugBuildType), + CMakeOption(PLUGIN.SPIDER_STORAGE_ENGINE, isDebugBuildType or isExperimental), CMakeOption(PLUGIN.ROCKSDB_STORAGE_ENGINE, False), CMakeOption(PLUGIN.OQGRAPH_STORAGE_ENGINE, False), + CMakeOption(PLUGIN.DUCKDB_STORAGE_ENGINE, False), ] if isDebugBuildType: - flags.append(CMakeOption(CMAKE.BUILD_TYPE, BuildType.DEBUG)) - flags.append(CMakeOption(WITH.DBUG_TRACE, False)) + flags.extend( + [ + CMakeOption(CMAKE.BUILD_TYPE, BuildType.DEBUG), + CMakeOption(WITH.DBUG_TRACE, False), + CMakeOption(CMAKE.C_FLAGS, "-Og"), + CMakeOption(CMAKE.CXX_FLAGS, "-Og"), + ] + ) sequence.add_step( InContainer( @@ -279,7 +301,7 @@ def msan( path_to_test_runner=PurePath("bld", "mysql-test"), step_wrapping_fn=lambda step: InContainer(docker_environment=config, step=step), ) - if isDebugBuildType: + if isDebugBuildType or isExperimental: steps += get_mtr_spider_steps( jobs=jobs, env_vars=env_vars, @@ -290,6 +312,26 @@ def msan( docker_environment=config, step=step ), ) + if isExperimental and not isDebugBuildType: + steps += get_mtr_cursor_steps( + jobs=jobs, + env_vars=env_vars, + halt_on_failure=False, + path_to_test_runner=PurePath("bld", "mysql-test"), + step_wrapping_fn=lambda step: InContainer( + docker_environment=config, step=step + ), + ) + steps += get_mtr_view_steps( + jobs=jobs, + env_vars=env_vars, + halt_on_failure=False, + path_to_test_runner=PurePath("bld", "mysql-test"), + step_wrapping_fn=lambda step: InContainer( + docker_environment=config, step=step + ), + ) + # Save results steps += [ save_mtr_logs( step_wrapping_fn=lambda step: InContainer( diff --git a/configuration/steps/generators/cmake/options.py b/configuration/steps/generators/cmake/options.py index d3981c671..0c1f67f3c 100644 --- a/configuration/steps/generators/cmake/options.py +++ b/configuration/steps/generators/cmake/options.py @@ -73,6 +73,7 @@ class PLUGIN(StrEnum): ARCHIVE_STORAGE_ENGINE = "ARCHIVE" COLUMNSTORE_STORAGE_ENGINE = "COLUMNSTORE" CONNECT_STORAGE_ENGINE = "CONNECT" + DUCKDB_STORAGE_ENGINE = "DUCKDB" FEDERATED_STORAGE_ENGINE = "FEDERATED" FEDERATEDX_STORAGE_ENGINE = "FEDERATEDX" FEEDBACK = "FEEDBACK" diff --git a/constants.py b/constants.py index 1dc251925..d56e71b95 100644 --- a/constants.py +++ b/constants.py @@ -132,6 +132,7 @@ "amd64-fedora-44-valgrind", "amd64-freebsd-14", "amd64-msan-clang-20", + "amd64-msan-clang-22", "amd64-openeuler-2403", "amd64-openssl3-fips", "amd64-rhel-7", @@ -181,10 +182,9 @@ "amd64-debian-12", "amd64-debian-12-deb-autobake-migration", "amd64-debian-12-debug-embedded", - "amd64-msan-clang-20-debug", "amd64-rhel-10", - "amd64-ubasan-clang-20", - "amd64-ubasan-clang-20-debug", + "amd64-ubasan-clang-22", + "amd64-ubasan-clang-22-debug", "amd64-ubuntu-2404", "ppc64le-centos-stream10", "ppc64le-rhel-10", diff --git a/master-migration/master.cfg b/master-migration/master.cfg index b1d7dbff8..35a671403 100644 --- a/master-migration/master.cfg +++ b/master-migration/master.cfg @@ -8,7 +8,7 @@ import yaml import configuration.builders.definitions.connectors.conc as conc_c_builders import configuration.builders.definitions.connectors.concpp as conc_cc_builders import configuration.builders.definitions.connectors.conodbc as conc_odbc_builders -from configuration.builders.base import GenericBuilder +from configuration.builders.base import GenericBuilder, WorkerBase from configuration.builders.common import ( deb_release_builder, docker_config, @@ -32,6 +32,8 @@ from configuration.schedulers.connectors import ( from configuration.workers import worker from master_common import IS_CHECKCONFIG, base_master_config +from typing import Iterable + ####### VARIABLES cfg_dir = os.path.abspath(os.path.dirname(__file__)) base_dir = os.path.abspath(f"{cfg_dir}/../") @@ -162,7 +164,13 @@ c["builders"].append( ## ------------------------------------------------------------------- ## -def ubasan_builder(name: str, debug: bool) -> GenericBuilder: +def ubasan_builder( + name: str, + image: str, + debug: bool, + experimental: bool, + workers: Iterable[WorkerBase], +) -> GenericBuilder: tags_ubasan = ("Debian", "clang", "asan", "ubsan", "big") jobs = 12 if debug: @@ -173,22 +181,52 @@ def ubasan_builder(name: str, debug: bool) -> GenericBuilder: sequences=[ asan_ubsan( jobs=jobs, - config=docker_config(image="debian12-msan-clang-20", shm_size="24g"), + config=docker_config(image=image, shm_size="24g"), isDebugBuildType=debug, + isExperimental=experimental, ) ], ).get_config( - workers=DEFAULT_AMD64_WORKER_POOL, + workers=workers, tags=list(tags_ubasan), jobs=jobs, ) -for builder in ["amd64-ubasan-clang-20", "amd64-ubasan-clang-20-debug"]: - c["builders"].append(ubasan_builder(name=builder, debug=builder.endswith("debug"))) +UBASAN_WORKERS = WORKER_POOL.get_workers_for_arch( + arch="amd64", names=["hz-bbw6", "hz-bbw8", "hz-bbw9"] +) +UBASAN_DEBUG_WORKERS = WORKER_POOL.get_workers_for_arch( + arch="amd64", names=["hz-bbw6", "sg-bbw1"] +) + +for builder, image, experimental, workers in [ + ("amd64-ubasan-clang-22", "debian13-msan-clang-22", True, UBASAN_WORKERS), + ( + "amd64-ubasan-clang-22-debug", + "debian13-msan-clang-22", + True, + UBASAN_DEBUG_WORKERS, + ), +]: + c["builders"].append( + ubasan_builder( + name=builder, + image=image, + debug=builder.endswith("debug"), + experimental=experimental, + workers=workers, + ) + ) -def msan_builder(name: str, debug: bool) -> GenericBuilder: +def msan_builder( + name: str, + image: str, + debug: bool, + experimental: bool, + workers: Iterable[WorkerBase], +) -> GenericBuilder: tags_msan = ("Debian", "clang", "msan", "big") jobs = 12 if debug: @@ -200,19 +238,33 @@ def msan_builder(name: str, debug: bool) -> GenericBuilder: sequences=[ msan( jobs=jobs, - config=docker_config(image="debian12-msan-clang-20", shm_size="24g"), + config=docker_config(image=image, shm_size="24g"), isDebugBuildType=debug, + isExperimental=experimental, ) ], ).get_config( - workers=DEFAULT_AMD64_WORKER_POOL, + workers=workers, tags=list(tags_msan), jobs=jobs, ) -builder = "amd64-msan-clang-20-debug" -c["builders"].append(msan_builder(name=builder, debug=builder.endswith("debug"))) +MSAN_WORKERS = WORKER_POOL.get_workers_for_arch( + arch="amd64", names=["hz-bbw6", "hz-bbw8", "hz-bbw9"] +) +for builder, image, experimental, workers in [ + ("amd64-msan-clang-22", "debian13-msan-clang-22", True, MSAN_WORKERS), +]: + c["builders"].append( + msan_builder( + name=builder, + image=image, + debug=builder.endswith("debug"), + experimental=experimental, + workers=workers, + ) + ) ## ------------------------------------------------------------------- ## ## MTR EXTENDED COVERAGE BUILDERS ## diff --git a/master-migration/workers.yaml b/master-migration/workers.yaml index 6fd17a778..5293ee488 100644 --- a/master-migration/workers.yaml +++ b/master-migration/workers.yaml @@ -27,7 +27,7 @@ - name: sg-bbw1 arch: amd64 - total_jobs: 128 + total_jobs: 20 os_type: debian - name: bg-arm-rasmus-5