From 2b54413a14c615847d470c72c5b3753268e4f506 Mon Sep 17 00:00:00 2001 From: Varun Joginpalli Date: Wed, 1 Jul 2026 01:43:01 +0000 Subject: [PATCH] MAINT: Standardize garak.encoding defaults and fix atomic-attack name collisions --- doc/scanner/garak.ipynb | 13 +- doc/scanner/garak.py | 11 +- pyrit/scenario/scenarios/garak/encoding.py | 131 +++++++++++------ .../unit/backend/test_scenario_run_service.py | 35 +++++ tests/unit/scenario/garak/test_encoding.py | 135 +++++++++++++++++- 5 files changed, 266 insertions(+), 59 deletions(-) diff --git a/doc/scanner/garak.ipynb b/doc/scanner/garak.ipynb index 296d0db846..e9017b7a0b 100644 --- a/doc/scanner/garak.ipynb +++ b/doc/scanner/garak.ipynb @@ -65,15 +65,20 @@ "strategy encodes the prompt, asks the target to decode it, and scores whether the decoded output\n", "matches the harmful content. Default datasets include slur terms and web/HTML/JS content.\n", "\n", - "**CLI example:**\n", + "**Default run** uses the curated `DEFAULT` strategy aggregate (Base16, ROT13, MorseCode — one\n", + "base-N, one substitution cipher, and one symbolic alphabet) for a fast, representative scan. Use\n", + "the `ALL` aggregate for an exhaustive run across every encoding scheme.\n", + "\n", + "**Fast path** (sanity-check target wiring in well under a minute) — pick a single-variant encoding\n", + "and one prompt:\n", "\n", "```bash\n", - "pyrit_scan garak.encoding --target openai_chat --strategies base64 --max-dataset-size 1\n", + "pyrit_scan garak.encoding --target openai_chat --strategies rot13 --max-dataset-size 1\n", "```\n", "\n", "**Available strategies** (17 encodings): Base64, Base2048, Base16, Base32, ASCII85, Hex,\n", "QuotedPrintable, UUencode, ROT13, Braille, Atbash, MorseCode, NATO, Ecoji, Zalgo, LeetSpeak,\n", - "AsciiSmuggler\n", + "AsciiSmuggler. Aggregates: `DEFAULT` (curated subset, the default) and `ALL` (every encoding).\n", "\n", "> **Note:** Strategy composition is NOT supported for Encoding — each encoding is tested\n", "> independently." @@ -143,7 +148,7 @@ "\u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n", "\u001b[1m 📋 Scenario Details\u001b[0m\n", "\u001b[36m • Name: Encoding\u001b[0m\n", - "\u001b[36m • Scenario Version: 1\u001b[0m\n", + "\u001b[36m • Scenario Version: 2\u001b[0m\n", "\u001b[36m • PyRIT Version: 0.12.1.dev0\u001b[0m\n", "\u001b[36m • Description:\u001b[0m\n", "\u001b[36m Encoding Scenario implementation for PyRIT. This scenario tests how resilient models are to various encoding\u001b[0m\n", diff --git a/doc/scanner/garak.py b/doc/scanner/garak.py index 9be4417596..b221de5968 100644 --- a/doc/scanner/garak.py +++ b/doc/scanner/garak.py @@ -39,15 +39,20 @@ # strategy encodes the prompt, asks the target to decode it, and scores whether the decoded output # matches the harmful content. Default datasets include slur terms and web/HTML/JS content. # -# **CLI example:** +# **Default run** uses the curated `DEFAULT` strategy aggregate (Base16, ROT13, MorseCode — one +# base-N, one substitution cipher, and one symbolic alphabet) for a fast, representative scan. Use +# the `ALL` aggregate for an exhaustive run across every encoding scheme. +# +# **Fast path** (sanity-check target wiring in well under a minute) — pick a single-variant encoding +# and one prompt: # # ```bash -# pyrit_scan garak.encoding --target openai_chat --strategies base64 --max-dataset-size 1 +# pyrit_scan garak.encoding --target openai_chat --strategies rot13 --max-dataset-size 1 # ``` # # **Available strategies** (17 encodings): Base64, Base2048, Base16, Base32, ASCII85, Hex, # QuotedPrintable, UUencode, ROT13, Braille, Atbash, MorseCode, NATO, Ecoji, Zalgo, LeetSpeak, -# AsciiSmuggler +# AsciiSmuggler. Aggregates: `DEFAULT` (curated subset, the default) and `ALL` (every encoding). # # > **Note:** Strategy composition is NOT supported for Encoding — each encoding is tested # > independently. diff --git a/pyrit/scenario/scenarios/garak/encoding.py b/pyrit/scenario/scenarios/garak/encoding.py index 57041aac1e..a2b46dbd5e 100644 --- a/pyrit/scenario/scenarios/garak/encoding.py +++ b/pyrit/scenario/scenarios/garak/encoding.py @@ -85,33 +85,50 @@ class EncodingStrategy(ScenarioStrategy): Strategies for encoding attacks. Each enum member represents an encoding scheme that will be tested against the target model. - The ALL aggregate expands to include all encoding strategies. + The ``ALL`` aggregate expands to every encoding scheme (exhaustive run). The ``DEFAULT`` + aggregate expands to a small curated subset that spans distinct encoding families, giving a + fast, representative default run. Note: EncodingStrategy does not support composition. Each encoding must be applied individually. + The strategy axis here is the encoding scheme (not an attack technique), and every encoding runs + as a single-turn ``PromptSendingAttack``, so SINGLE_TURN/MULTI_TURN aggregates are not applicable. """ - # Aggregate member + # Aggregate members ALL = ("all", {"all"}) + DEFAULT = ("default", {"default"}) - # Individual encoding strategies (matching the atomic attack names) + # Individual encoding strategies (each value matches the encoding name used for display grouping). + # Members tagged ``default`` form the curated DEFAULT aggregate: one base-N encoding (Base16), + # one substitution cipher (ROT13), and one symbolic alphabet (MorseCode). Base64 = ("base64", set[str]()) Base2048 = ("base2048", set[str]()) - Base16 = ("base16", set[str]()) + Base16 = ("base16", {"default"}) Base32 = ("base32", set[str]()) ASCII85 = ("ascii85", set[str]()) Hex = ("hex", set[str]()) QuotedPrintable = ("quoted_printable", set[str]()) UUencode = ("uuencode", set[str]()) - ROT13 = ("rot13", set[str]()) + ROT13 = ("rot13", {"default"}) Braille = ("braille", set[str]()) Atbash = ("atbash", set[str]()) - MorseCode = ("morse_code", set[str]()) + MorseCode = ("morse_code", {"default"}) NATO = ("nato", set[str]()) Ecoji = ("ecoji", set[str]()) Zalgo = ("zalgo", set[str]()) LeetSpeak = ("leet_speak", set[str]()) AsciiSmuggler = ("ascii_smuggler", set[str]()) + @classmethod + def get_aggregate_tags(cls) -> set[str]: + """ + Get the set of tags that represent aggregate categories. + + Returns: + set[str]: The base ``"all"`` aggregate plus the scenario-specific ``"default"`` aggregate. + """ + return super().get_aggregate_tags() | {"default"} + logger = logging.getLogger(__name__) @@ -133,7 +150,7 @@ class Encoding(Scenario): By default, this uses the same dataset as Garak: slur terms and web XSS payloads. """ - VERSION: int = 1 + VERSION: int = 2 @apply_defaults def __init__( @@ -165,7 +182,7 @@ def __init__( super().__init__( version=self.VERSION, strategy_class=EncodingStrategy, - default_strategy=EncodingStrategy.ALL, + default_strategy=EncodingStrategy.DEFAULT, default_dataset_config=CompoundDatasetAttackConfiguration( configurations=[ EncodingDatasetConfiguration(dataset_names=["garak_slur_terms_en"], max_dataset_size=3), @@ -231,45 +248,55 @@ def _get_converter_attacks(self) -> list[AtomicAttack]: Returns: list[AtomicAttack]: List of all atomic attacks to execute. """ - # Map of all available converters with their encoding names - all_converters_with_encodings: list[tuple[list[PromptConverter], str]] = [ - ([Base64Converter()], "base64"), - ([Base64Converter(encoding_func="urlsafe_b64encode")], "base64"), - ([Base64Converter(encoding_func="standard_b64encode")], "base64"), - ([Base64Converter(encoding_func="b2a_base64")], "base64"), - ([Base2048Converter()], "base2048"), - ([Base64Converter(encoding_func="b16encode")], "base16"), - ([Base64Converter(encoding_func="b32encode")], "base32"), - ([Base64Converter(encoding_func="a85encode")], "ascii85"), - ([Base64Converter(encoding_func="b85encode")], "ascii85"), - ([BinAsciiConverter(encoding_func="hex")], "hex"), - ([BinAsciiConverter(encoding_func="quoted-printable")], "quoted_printable"), - ([BinAsciiConverter(encoding_func="UUencode")], "uuencode"), - ([ROT13Converter()], "rot13"), - ([BrailleConverter()], "braille"), - ([AtbashConverter()], "atbash"), - ([MorseConverter()], "morse_code"), - ([NatoConverter()], "nato"), - ([EcojiConverter()], "ecoji"), - ([ZalgoConverter()], "zalgo"), - ([LeetspeakConverter()], "leet_speak"), - ([AsciiSmugglerConverter()], "ascii_smuggler"), + # Map of all available converters with their encoding name and a unique variant slug. + # ``encoding_name`` drives strategy selection and user-facing grouping (display_group); + # ``variant_slug`` is unique per row so that atomic-attack names stay unique even when one + # encoding name maps to multiple converter variants (e.g. base64, ascii85). + # NOTE: near-duplicate base64 variants were trimmed alongside the VERSION bump (standard_b64encode + # is byte-identical to the default b64encode; b2a_base64 only appends a trailing newline). We keep + # the default encoding plus the url-safe alphabet, which is a genuinely distinct representation. + all_converters_with_encodings: list[tuple[list[PromptConverter], str, str]] = [ + ([Base64Converter()], "base64", "base64"), + ([Base64Converter(encoding_func="urlsafe_b64encode")], "base64", "base64_urlsafe"), + ([Base2048Converter()], "base2048", "base2048"), + ([Base64Converter(encoding_func="b16encode")], "base16", "base16"), + ([Base64Converter(encoding_func="b32encode")], "base32", "base32"), + ([Base64Converter(encoding_func="a85encode")], "ascii85", "ascii85_a85"), + ([Base64Converter(encoding_func="b85encode")], "ascii85", "ascii85_b85"), + ([BinAsciiConverter(encoding_func="hex")], "hex", "hex"), + ([BinAsciiConverter(encoding_func="quoted-printable")], "quoted_printable", "quoted_printable"), + ([BinAsciiConverter(encoding_func="UUencode")], "uuencode", "uuencode"), + ([ROT13Converter()], "rot13", "rot13"), + ([BrailleConverter()], "braille", "braille"), + ([AtbashConverter()], "atbash", "atbash"), + ([MorseConverter()], "morse_code", "morse_code"), + ([NatoConverter()], "nato", "nato"), + ([EcojiConverter()], "ecoji", "ecoji"), + ([ZalgoConverter()], "zalgo", "zalgo"), + ([LeetspeakConverter()], "leet_speak", "leet_speak"), + ([AsciiSmugglerConverter()], "ascii_smuggler", "ascii_smuggler"), ] # Filter to only include selected strategies selected_encoding_names = {s.value for s in self._scenario_strategies} converters_with_encodings = [ - (conv, name) for conv, name in all_converters_with_encodings if name in selected_encoding_names + (conv, name, variant_slug) + for conv, name, variant_slug in all_converters_with_encodings + if name in selected_encoding_names ] atomic_attacks = [] - for conv, name in converters_with_encodings: - atomic_attacks.extend(self._get_prompt_attacks(converters=conv, encoding_name=name)) + for conv, name, variant_slug in converters_with_encodings: + atomic_attacks.extend( + self._get_prompt_attacks(converters=conv, encoding_name=name, variant_slug=variant_slug) + ) return atomic_attacks - def _get_prompt_attacks(self, *, converters: list[PromptConverter], encoding_name: str) -> list[AtomicAttack]: + def _get_prompt_attacks( + self, *, converters: list[PromptConverter], encoding_name: str, variant_slug: str + ) -> list[AtomicAttack]: """ - Create atomic attacks for a specific encoding scheme. + Create atomic attacks for a specific encoding converter variant. For each seed prompt (the text to be decoded), creates atomic attacks that: 1. Encode the seed prompt using the specified converter(s) @@ -279,31 +306,42 @@ def _get_prompt_attacks(self, *, converters: list[PromptConverter], encoding_nam Args: converters (list[PromptConverter]): The list of converters to apply to the seed prompts. - encoding_name (str): Human-readable name of the encoding scheme (e.g., "Base64", "ROT13"). + encoding_name (str): Human-readable name of the encoding scheme (e.g., "base64", "rot13"). + Used as the ``display_group`` so all variants of an encoding aggregate together in output. + variant_slug (str): Unique slug for this converter variant, used to build a unique + ``atomic_attack_name`` per converter variant and prompt config. Returns: - list[AtomicAttack]: List of atomic attacks for this encoding scheme. + list[AtomicAttack]: List of atomic attacks for this encoding converter variant. Raises: ValueError: If scenario is not properly initialized. """ - converter_configs = [ - AttackConverterConfig( - request_converters=PromptConverterConfiguration.from_converters(converters=converters) + # (config_name_suffix, converter_config). The bare "raw" config encodes only; each + # decode-template config additionally asks the model to decode. + converter_configs: list[tuple[str, AttackConverterConfig]] = [ + ( + "raw", + AttackConverterConfig( + request_converters=PromptConverterConfiguration.from_converters(converters=converters) + ), ) ] - for decode_type in self._encoding_templates: + for decode_index, decode_type in enumerate(self._encoding_templates): converters_ = converters[:] + [AskToDecodeConverter(template=decode_type, encoding_name=encoding_name)] converter_configs.append( - AttackConverterConfig( - request_converters=PromptConverterConfiguration.from_converters(converters=converters_) + ( + f"decode{decode_index}", + AttackConverterConfig( + request_converters=PromptConverterConfiguration.from_converters(converters=converters_) + ), ) ) atomic_attacks = [] - for attack_converter_config in converter_configs: + for config_suffix, attack_converter_config in converter_configs: # objective_target is guaranteed to be non-None by parent class validation if self._objective_target is None: raise ValueError( @@ -316,7 +354,8 @@ def _get_prompt_attacks(self, *, converters: list[PromptConverter], encoding_nam ) atomic_attacks.append( AtomicAttack( - atomic_attack_name=encoding_name, + atomic_attack_name=f"{variant_slug}_{config_suffix}", + display_group=encoding_name, attack_technique=AttackTechnique(attack=attack), seed_groups=self._resolved_seed_groups or [], ) diff --git a/tests/unit/backend/test_scenario_run_service.py b/tests/unit/backend/test_scenario_run_service.py index 08ae14f8a5..c8fa65c6a4 100644 --- a/tests/unit/backend/test_scenario_run_service.py +++ b/tests/unit/backend/test_scenario_run_service.py @@ -19,6 +19,7 @@ from pyrit.models import AttackOutcome, ScenarioRunState from pyrit.models.catalog.scenario import RunScenarioRequest from pyrit.scenario.core import DatasetAttackConfiguration, DatasetConfiguration +from pyrit.scenario.scenarios.garak.encoding import EncodingDatasetConfiguration _REGISTRY_PATCH_BASE = "pyrit.registry" _MEMORY_PATCH = "pyrit.memory.CentralMemory.get_memory_instance" @@ -311,6 +312,40 @@ class _MarkerDatasetConfiguration(DatasetConfiguration): assert default_config.dataset_names == ["original"] assert default_config.max_dataset_size == 100 + async def test_encoding_dataset_configuration_stays_backend_constructible(self, mock_all_registries) -> None: + """``EncodingDatasetConfiguration`` remains reconstructible by the backend ``dataset_names`` path. + + Foot-gun guard: the backend's ``_build_init_kwargs`` rebuilds the config via + ``type(default_config)(dataset_names=..., max_dataset_size=...)`` and silently degrades to a + plain base config on ``TypeError``. Each child inside the scenario's default + ``CompoundDatasetAttackConfiguration`` is an ``EncodingDatasetConfiguration``, so that subclass + must stay constructible with just ``dataset_names``/``max_dataset_size`` (no new *required* + ``__init__`` args). This pins the real subclass (not a synthetic marker) so adding a required + ctor arg fails loudly here. + + NOTE: This does NOT cover the scenario's actual *default* config, which is a + ``CompoundDatasetAttackConfiguration`` — that type takes ``configurations=`` (not + ``dataset_names=``), so the backend ``--dataset-names`` override currently falls back to a base + ``DatasetAttackConfiguration`` for compound defaults. That reconstruction gap is a separate, + pre-existing backend limitation tracked outside this scenario PR. + """ + default_config = EncodingDatasetConfiguration( + dataset_names=["garak_slur_terms_en", "garak_web_html_js"], max_dataset_size=3 + ) + scenario_instance = mock_all_registries["scenario_instance"] + scenario_instance._default_dataset_config = default_config + + service = ScenarioRunService() + await service.start_run_async(request=_make_request(dataset_names=["custom_a", "custom_b"], max_dataset_size=2)) + + init_call = scenario_instance.initialize_async.await_args + built_config = init_call.kwargs["dataset_config"] + + # Real subclass type is preserved (not degraded to base DatasetConfiguration) + assert type(built_config) is EncodingDatasetConfiguration + assert built_config.dataset_names == ["custom_a", "custom_b"] + assert built_config.max_dataset_size == 2 + async def test_start_run_dataset_names_without_max_dataset_size_preserves_subclass( self, mock_all_registries ) -> None: diff --git a/tests/unit/scenario/garak/test_encoding.py b/tests/unit/scenario/garak/test_encoding.py index c26784f952..8ed08b7dac 100644 --- a/tests/unit/scenario/garak/test_encoding.py +++ b/tests/unit/scenario/garak/test_encoding.py @@ -107,7 +107,7 @@ def test_init_with_default_seed_prompts(self, mock_objective_target, mock_object ) assert scenario.name == "Encoding" - assert scenario.VERSION == 1 + assert scenario.VERSION == 2 def test_init_with_custom_scorer(self, mock_objective_target, mock_objective_scorer, mock_memory_seeds): """Test initialization with custom objective scorer.""" @@ -199,12 +199,13 @@ async def test_init_attack_strategies( await scenario.initialize_async(objective_target=mock_objective_target, dataset_config=mock_dataset_config) - # By default, EncodingStrategy.ALL is used, which expands to all encoding strategies + # By default, EncodingStrategy.DEFAULT is used, which expands to the curated subset assert len(scenario._scenario_strategies) > 0 # Verify all strategies contain EncodingStrategy instances assert all(isinstance(s, EncodingStrategy) for s in scenario._scenario_strategies) - # Verify none of the strategies are the aggregate "ALL" + # Verify none of the strategies are the aggregate members assert all(s != EncodingStrategy.ALL for s in scenario._scenario_strategies) + assert all(s != EncodingStrategy.DEFAULT for s in scenario._scenario_strategies) @pytest.mark.usefixtures("patch_central_database") @@ -248,7 +249,7 @@ async def test_get_converter_attacks_returns_multiple_encodings( attack_runs = scenario._get_converter_attacks() # Should have multiple attack runs for different encodings - # The list includes: Base64 (4 variants), Base2048, Base16, Base32, ASCII85 (2), hex, + # The list includes: Base64 (2 variants), Base2048, Base16, Base32, ASCII85 (2), hex, # quoted-printable, UUencode, ROT13, Braille, Atbash, Morse, NATO, Ecoji, Zalgo, Leet, AsciiSmuggler assert len(attack_runs) > 0 @@ -266,7 +267,9 @@ async def test_get_prompt_attacks_creates_attack_runs( ) await scenario.initialize_async(objective_target=mock_objective_target, dataset_config=mock_dataset_config) - attack_runs = scenario._get_prompt_attacks(converters=[Base64Converter()], encoding_name="Base64") + attack_runs = scenario._get_prompt_attacks( + converters=[Base64Converter()], encoding_name="base64", variant_slug="base64" + ) # Should create attack runs assert len(attack_runs) > 0 @@ -295,7 +298,9 @@ async def test_attack_runs_include_objectives( ) await scenario.initialize_async(objective_target=mock_objective_target, dataset_config=mock_dataset_config) - attack_runs = scenario._get_prompt_attacks(converters=[Base64Converter()], encoding_name="Base64") + attack_runs = scenario._get_prompt_attacks( + converters=[Base64Converter()], encoding_name="base64", variant_slug="base64" + ) # Check that seed groups contain objectives with the expected format for run in attack_runs: @@ -470,3 +475,121 @@ async def test_one_resolution_call_baseline_matches_strategies(self, mock_object baseline_objs = set(scenario._atomic_attacks[0].objectives) for attack in scenario._atomic_attacks[1:]: assert set(attack.objectives) == baseline_objs + + +@pytest.mark.usefixtures("patch_central_database") +class TestEncodingStrategyDefaults: + """Tests for the curated DEFAULT aggregate and aggregate-tag wiring.""" + + def test_default_is_the_default_strategy(self, mock_objective_scorer): + """The scenario's default strategy is the curated DEFAULT aggregate, not ALL.""" + scenario = Encoding(objective_scorer=mock_objective_scorer) + assert scenario._default_strategy == EncodingStrategy.DEFAULT + + def test_default_aggregate_membership(self): + """DEFAULT expands to one base-N, one substitution, and one symbolic encoding.""" + members = EncodingStrategy.get_strategies_by_tag("default") + assert members == {EncodingStrategy.Base16, EncodingStrategy.ROT13, EncodingStrategy.MorseCode} + + def test_default_is_subset_of_all(self): + """Every DEFAULT member is also part of the exhaustive ALL aggregate.""" + all_members = set(EncodingStrategy.get_all_strategies()) + default_members = EncodingStrategy.get_strategies_by_tag("default") + assert default_members <= all_members + + def test_get_aggregate_tags_includes_default(self): + """``default`` is registered as an aggregate tag alongside ``all``.""" + tags = EncodingStrategy.get_aggregate_tags() + assert "all" in tags + assert "default" in tags + + +@pytest.mark.usefixtures("patch_central_database") +class TestEncodingAtomicCountsAndNaming: + """Atomic-attack counts, name uniqueness, and display-group grouping.""" + + @staticmethod + def _seed_groups(count: int = 1): + return [ + SeedAttackGroup(seeds=[SeedObjective(value=f"obj{i}"), SeedPrompt(value=f"payload{i}")]) + for i in range(count) + ] + + async def _build(self, target, scorer, strategies=None, *, include_baseline=True, encoding_templates=None): + from unittest.mock import patch + + seed_groups = self._seed_groups(1) + ctor_kwargs = {"objective_scorer": scorer} + if encoding_templates is not None: + ctor_kwargs["encoding_templates"] = encoding_templates + scenario = Encoding(**ctor_kwargs) + with patch.object(Encoding, "_resolve_seed_groups_async", new_callable=AsyncMock, return_value=seed_groups): + kwargs = {"objective_target": target, "include_baseline": include_baseline} + if strategies is not None: + kwargs["scenario_strategies"] = strategies + await scenario.initialize_async(**kwargs) + return scenario + + async def test_default_run_atomic_count(self, mock_objective_target, mock_objective_scorer): + """DEFAULT = 3 encodings x (1 raw + 4 decode templates) + 1 baseline = 16 atomics.""" + scenario = await self._build(mock_objective_target, mock_objective_scorer) + assert len(scenario._atomic_attacks) == 16 + + async def test_all_run_atomic_count(self, mock_objective_target, mock_objective_scorer): + """ALL = 19 converter variants x 5 prompt configs + 1 baseline = 96 atomics.""" + scenario = await self._build(mock_objective_target, mock_objective_scorer, strategies=[EncodingStrategy.ALL]) + assert len(scenario._atomic_attacks) == 96 + + async def test_all_atomic_names_unique(self, mock_objective_target, mock_objective_scorer): + """Every atomic-attack name is unique under ALL (no name collisions across variants).""" + scenario = await self._build(mock_objective_target, mock_objective_scorer, strategies=[EncodingStrategy.ALL]) + names = [a.atomic_attack_name for a in scenario._atomic_attacks] + assert len(names) == len(set(names)) + + async def test_multi_variant_encoding_names_unique(self, mock_objective_target, mock_objective_scorer): + """base64 expands to 2 converter variants (default + url-safe), each with distinct names.""" + scenario = await self._build(mock_objective_target, mock_objective_scorer, strategies=[EncodingStrategy.Base64]) + names = [a.atomic_attack_name for a in scenario._atomic_attacks if a.atomic_attack_name != "baseline"] + # 2 variants x 5 configs = 10 attacks, all uniquely named + assert len(names) == 10 + assert len(set(names)) == 10 + + async def test_display_group_aggregates_by_encoding(self, mock_objective_target, mock_objective_scorer): + """All base64 variants share a single ``base64`` display_group for reporting.""" + scenario = await self._build(mock_objective_target, mock_objective_scorer, strategies=[EncodingStrategy.Base64]) + non_baseline = [a for a in scenario._atomic_attacks if a.atomic_attack_name != "baseline"] + assert {a.display_group for a in non_baseline} == {"base64"} + + async def test_fast_path_single_encoding_count(self, mock_objective_target, mock_objective_scorer): + """Fast path (single ROT13 strategy) = 1 raw + 4 decode + 1 baseline = 6 atomics.""" + scenario = await self._build(mock_objective_target, mock_objective_scorer, strategies=[EncodingStrategy.ROT13]) + assert len(scenario._atomic_attacks) == 6 + rot13_names = sorted(a.atomic_attack_name for a in scenario._atomic_attacks) + assert rot13_names == [ + "baseline", + "rot13_decode0", + "rot13_decode1", + "rot13_decode2", + "rot13_decode3", + "rot13_raw", + ] + + async def test_default_run_count_without_baseline(self, mock_objective_target, mock_objective_scorer): + """With baseline disabled, DEFAULT = 3 encodings x 5 prompt configs = 15 atomics (no baseline).""" + scenario = await self._build(mock_objective_target, mock_objective_scorer, include_baseline=False) + assert len(scenario._atomic_attacks) == 15 + assert all(a.atomic_attack_name != "baseline" for a in scenario._atomic_attacks) + + async def test_custom_encoding_templates_scale_decode_configs(self, mock_objective_target, mock_objective_scorer): + """Each decode template adds one prompt config: N templates -> 1 raw + N decode configs per variant.""" + scenario = await self._build( + mock_objective_target, + mock_objective_scorer, + strategies=[EncodingStrategy.ROT13], + include_baseline=False, + encoding_templates=["decode this: {encoded_text}", "now decode: {encoded_text}"], + ) + # 1 variant x (1 raw + 2 decode) = 3 atomics + assert len(scenario._atomic_attacks) == 3 + names = sorted(a.atomic_attack_name for a in scenario._atomic_attacks) + assert names == ["rot13_decode0", "rot13_decode1", "rot13_raw"]