diff --git a/configuration/builders/sequences/helpers.py b/configuration/builders/sequences/helpers.py index d907886b..23ee3708 100644 --- a/configuration/builders/sequences/helpers.py +++ b/configuration/builders/sequences/helpers.py @@ -65,6 +65,102 @@ def get_mtr_normal_steps( return steps +def get_mtr_cursor_steps( + jobs, + path_to_test_runner: PurePath, + halt_on_failure: bool = True, + step_wrapping_fn=lambda step: step, + additional_mtr_options: list[MTROption] = [], + env_vars: list[tuple] = [], +): + steps = [] + steps.append( + step_wrapping_fn( + ShellStep( + MTRTest( + name="cursor", + save_logs_path=MTR_PATH_TO_SAVE_LOGS / "cursor", + workdir=path_to_test_runner, + testcase=MTRGenerator( + flags=[ + MTROption(MTR.VERBOSE_RESTART, True), + MTROption(MTR.FORCE, True), + MTROption(MTR.CURSOR_PROTOCOL, True), + MTROption(MTR.MAX_SAVE_CORE, 2), + MTROption(MTR.MAX_SAVE_DATADIR, 1), + MTROption(MTR.MAX_TEST_FAIL, 20), + MTROption(MTR.PARALLEL, jobs * 2), + MTROption(MTR.VARDIR, "/dev/shm/cursor"), + MTROption( + MTR.XML_REPORT, MTR_PATH_TO_SAVE_LOGS / "cursor.xml" + ), + ] + + additional_mtr_options, + suite_collection=TestSuiteCollection( + [ + SUITE.MAIN, + ] + ), + ), + ), + options=StepOptions( + haltOnFailure=halt_on_failure, descriptionDone="MTR cursor" + ), + env_vars=env_vars, + ) + ) + ) + return steps + + +def get_mtr_view_steps( + jobs, + path_to_test_runner: PurePath, + halt_on_failure: bool = True, + step_wrapping_fn=lambda step: step, + additional_mtr_options: list[MTROption] = [], + env_vars: list[tuple] = [], +): + steps = [] + steps.append( + step_wrapping_fn( + ShellStep( + MTRTest( + name="view", + save_logs_path=MTR_PATH_TO_SAVE_LOGS / "view", + workdir=path_to_test_runner, + testcase=MTRGenerator( + flags=[ + MTROption(MTR.VERBOSE_RESTART, True), + MTROption(MTR.FORCE, True), + MTROption(MTR.VIEW_PROTOCOL, True), + MTROption(MTR.MAX_SAVE_CORE, 2), + MTROption(MTR.MAX_SAVE_DATADIR, 1), + MTROption(MTR.MAX_TEST_FAIL, 20), + MTROption(MTR.PARALLEL, jobs * 2), + MTROption(MTR.VARDIR, "/dev/shm/view"), + MTROption( + MTR.XML_REPORT, MTR_PATH_TO_SAVE_LOGS / "view.xml" + ), + ] + + additional_mtr_options, + suite_collection=TestSuiteCollection( + [ + SUITE.MAIN, + ] + ), + ), + ), + options=StepOptions( + haltOnFailure=halt_on_failure, descriptionDone="MTR view" + ), + env_vars=env_vars, + ) + ) + ) + return steps + + def get_mtr_rocksdb_steps( jobs, path_to_test_runner: PurePath, diff --git a/configuration/builders/sequences/sanitizers.py b/configuration/builders/sequences/sanitizers.py index 8ed83f48..cb16aea6 100644 --- a/configuration/builders/sequences/sanitizers.py +++ b/configuration/builders/sequences/sanitizers.py @@ -6,9 +6,11 @@ InContainer, ) from configuration.builders.sequences.helpers import ( + get_mtr_cursor_steps, get_mtr_normal_steps, get_mtr_s3_steps, get_mtr_spider_steps, + get_mtr_view_steps, mtr_reporter, save_mtr_logs, ) @@ -34,6 +36,7 @@ def asan_ubsan( config: DockerConfig, jobs: int, isDebugBuildType: bool, + isExperimental: bool, ): sequence = BuildSequence() @@ -73,10 +76,17 @@ def asan_ubsan( CMakeOption(WITH.UBSAN, True), CMakeOption(WITH.UNIT_TESTS, False), CMakeOption(PLUGIN.COLUMNSTORE_STORAGE_ENGINE, False), + CMakeOption(PLUGIN.DUCKDB_STORAGE_ENGINE, False), ] if isDebugBuildType: - flags.append(CMakeOption(CMAKE.BUILD_TYPE, BuildType.DEBUG)) - flags.append(CMakeOption(WITH.DBUG_TRACE, False)) + flags.extend( + [ + CMakeOption(CMAKE.BUILD_TYPE, BuildType.DEBUG), + CMakeOption(WITH.DBUG_TRACE, False), + CMakeOption(CMAKE.C_FLAGS, "-Og"), + CMakeOption(CMAKE.CXX_FLAGS, "-Og"), + ] + ) sequence.add_step( InContainer( @@ -164,21 +174,25 @@ def asan_ubsan( docker_environment=config, step=step ), ) - + [ - save_mtr_logs( - step_wrapping_fn=lambda step: InContainer( - docker_environment=config, step=step - ), - ), - mtr_reporter( - step_wrapping_fn=lambda step: InContainer( - docker_environment=config, step=step - ), - ), - ] ): sequence.add_step(step) + sequence.add_step( + save_mtr_logs( + step_wrapping_fn=lambda step: InContainer( + docker_environment=config, step=step + ), + ) + ) + + sequence.add_step( + mtr_reporter( + step_wrapping_fn=lambda step: InContainer( + docker_environment=config, step=step + ), + ), + ) + return sequence @@ -186,6 +200,7 @@ def msan( config: DockerConfig, jobs: int, isDebugBuildType: bool, + isExperimental: bool, ): sequence = BuildSequence() @@ -214,13 +229,20 @@ def msan( CMakeOption(WITH.ZLIB, "bundled"), CMakeOption(WITH.SYSTEMD, "no"), CMakeOption(PLUGIN.COLUMNSTORE_STORAGE_ENGINE, False), - CMakeOption(PLUGIN.SPIDER_STORAGE_ENGINE, isDebugBuildType), + CMakeOption(PLUGIN.SPIDER_STORAGE_ENGINE, isDebugBuildType or isExperimental), CMakeOption(PLUGIN.ROCKSDB_STORAGE_ENGINE, False), CMakeOption(PLUGIN.OQGRAPH_STORAGE_ENGINE, False), + CMakeOption(PLUGIN.DUCKDB_STORAGE_ENGINE, False), ] if isDebugBuildType: - flags.append(CMakeOption(CMAKE.BUILD_TYPE, BuildType.DEBUG)) - flags.append(CMakeOption(WITH.DBUG_TRACE, False)) + flags.extend( + [ + CMakeOption(CMAKE.BUILD_TYPE, BuildType.DEBUG), + CMakeOption(WITH.DBUG_TRACE, False), + CMakeOption(CMAKE.C_FLAGS, "-Og"), + CMakeOption(CMAKE.CXX_FLAGS, "-Og"), + ] + ) sequence.add_step( InContainer( @@ -279,7 +301,7 @@ def msan( path_to_test_runner=PurePath("bld", "mysql-test"), step_wrapping_fn=lambda step: InContainer(docker_environment=config, step=step), ) - if isDebugBuildType: + if isDebugBuildType or isExperimental: steps += get_mtr_spider_steps( jobs=jobs, env_vars=env_vars, @@ -290,6 +312,26 @@ def msan( docker_environment=config, step=step ), ) + if isExperimental and not isDebugBuildType: + steps += get_mtr_cursor_steps( + jobs=jobs, + env_vars=env_vars, + halt_on_failure=False, + path_to_test_runner=PurePath("bld", "mysql-test"), + step_wrapping_fn=lambda step: InContainer( + docker_environment=config, step=step + ), + ) + steps += get_mtr_view_steps( + jobs=jobs, + env_vars=env_vars, + halt_on_failure=False, + path_to_test_runner=PurePath("bld", "mysql-test"), + step_wrapping_fn=lambda step: InContainer( + docker_environment=config, step=step + ), + ) + # Save results steps += [ save_mtr_logs( step_wrapping_fn=lambda step: InContainer( diff --git a/configuration/steps/generators/cmake/options.py b/configuration/steps/generators/cmake/options.py index d3981c67..0c1f67f3 100644 --- a/configuration/steps/generators/cmake/options.py +++ b/configuration/steps/generators/cmake/options.py @@ -73,6 +73,7 @@ class PLUGIN(StrEnum): ARCHIVE_STORAGE_ENGINE = "ARCHIVE" COLUMNSTORE_STORAGE_ENGINE = "COLUMNSTORE" CONNECT_STORAGE_ENGINE = "CONNECT" + DUCKDB_STORAGE_ENGINE = "DUCKDB" FEDERATED_STORAGE_ENGINE = "FEDERATED" FEDERATEDX_STORAGE_ENGINE = "FEDERATEDX" FEEDBACK = "FEEDBACK" diff --git a/constants.py b/constants.py index 1dc25192..d56e71b9 100644 --- a/constants.py +++ b/constants.py @@ -132,6 +132,7 @@ "amd64-fedora-44-valgrind", "amd64-freebsd-14", "amd64-msan-clang-20", + "amd64-msan-clang-22", "amd64-openeuler-2403", "amd64-openssl3-fips", "amd64-rhel-7", @@ -181,10 +182,9 @@ "amd64-debian-12", "amd64-debian-12-deb-autobake-migration", "amd64-debian-12-debug-embedded", - "amd64-msan-clang-20-debug", "amd64-rhel-10", - "amd64-ubasan-clang-20", - "amd64-ubasan-clang-20-debug", + "amd64-ubasan-clang-22", + "amd64-ubasan-clang-22-debug", "amd64-ubuntu-2404", "ppc64le-centos-stream10", "ppc64le-rhel-10", diff --git a/master-migration/master.cfg b/master-migration/master.cfg index b1d7dbff..35a67140 100644 --- a/master-migration/master.cfg +++ b/master-migration/master.cfg @@ -8,7 +8,7 @@ import yaml import configuration.builders.definitions.connectors.conc as conc_c_builders import configuration.builders.definitions.connectors.concpp as conc_cc_builders import configuration.builders.definitions.connectors.conodbc as conc_odbc_builders -from configuration.builders.base import GenericBuilder +from configuration.builders.base import GenericBuilder, WorkerBase from configuration.builders.common import ( deb_release_builder, docker_config, @@ -32,6 +32,8 @@ from configuration.schedulers.connectors import ( from configuration.workers import worker from master_common import IS_CHECKCONFIG, base_master_config +from typing import Iterable + ####### VARIABLES cfg_dir = os.path.abspath(os.path.dirname(__file__)) base_dir = os.path.abspath(f"{cfg_dir}/../") @@ -162,7 +164,13 @@ c["builders"].append( ## ------------------------------------------------------------------- ## -def ubasan_builder(name: str, debug: bool) -> GenericBuilder: +def ubasan_builder( + name: str, + image: str, + debug: bool, + experimental: bool, + workers: Iterable[WorkerBase], +) -> GenericBuilder: tags_ubasan = ("Debian", "clang", "asan", "ubsan", "big") jobs = 12 if debug: @@ -173,22 +181,52 @@ def ubasan_builder(name: str, debug: bool) -> GenericBuilder: sequences=[ asan_ubsan( jobs=jobs, - config=docker_config(image="debian12-msan-clang-20", shm_size="24g"), + config=docker_config(image=image, shm_size="24g"), isDebugBuildType=debug, + isExperimental=experimental, ) ], ).get_config( - workers=DEFAULT_AMD64_WORKER_POOL, + workers=workers, tags=list(tags_ubasan), jobs=jobs, ) -for builder in ["amd64-ubasan-clang-20", "amd64-ubasan-clang-20-debug"]: - c["builders"].append(ubasan_builder(name=builder, debug=builder.endswith("debug"))) +UBASAN_WORKERS = WORKER_POOL.get_workers_for_arch( + arch="amd64", names=["hz-bbw6", "hz-bbw8", "hz-bbw9"] +) +UBASAN_DEBUG_WORKERS = WORKER_POOL.get_workers_for_arch( + arch="amd64", names=["hz-bbw6", "sg-bbw1"] +) + +for builder, image, experimental, workers in [ + ("amd64-ubasan-clang-22", "debian13-msan-clang-22", True, UBASAN_WORKERS), + ( + "amd64-ubasan-clang-22-debug", + "debian13-msan-clang-22", + True, + UBASAN_DEBUG_WORKERS, + ), +]: + c["builders"].append( + ubasan_builder( + name=builder, + image=image, + debug=builder.endswith("debug"), + experimental=experimental, + workers=workers, + ) + ) -def msan_builder(name: str, debug: bool) -> GenericBuilder: +def msan_builder( + name: str, + image: str, + debug: bool, + experimental: bool, + workers: Iterable[WorkerBase], +) -> GenericBuilder: tags_msan = ("Debian", "clang", "msan", "big") jobs = 12 if debug: @@ -200,19 +238,33 @@ def msan_builder(name: str, debug: bool) -> GenericBuilder: sequences=[ msan( jobs=jobs, - config=docker_config(image="debian12-msan-clang-20", shm_size="24g"), + config=docker_config(image=image, shm_size="24g"), isDebugBuildType=debug, + isExperimental=experimental, ) ], ).get_config( - workers=DEFAULT_AMD64_WORKER_POOL, + workers=workers, tags=list(tags_msan), jobs=jobs, ) -builder = "amd64-msan-clang-20-debug" -c["builders"].append(msan_builder(name=builder, debug=builder.endswith("debug"))) +MSAN_WORKERS = WORKER_POOL.get_workers_for_arch( + arch="amd64", names=["hz-bbw6", "hz-bbw8", "hz-bbw9"] +) +for builder, image, experimental, workers in [ + ("amd64-msan-clang-22", "debian13-msan-clang-22", True, MSAN_WORKERS), +]: + c["builders"].append( + msan_builder( + name=builder, + image=image, + debug=builder.endswith("debug"), + experimental=experimental, + workers=workers, + ) + ) ## ------------------------------------------------------------------- ## ## MTR EXTENDED COVERAGE BUILDERS ## diff --git a/master-migration/workers.yaml b/master-migration/workers.yaml index 6fd17a77..5293ee48 100644 --- a/master-migration/workers.yaml +++ b/master-migration/workers.yaml @@ -27,7 +27,7 @@ - name: sg-bbw1 arch: amd64 - total_jobs: 128 + total_jobs: 20 os_type: debian - name: bg-arm-rasmus-5