fix(protect): align metric names across Python and TypeScript SDKs (TH-7040)

python/fi/evals/guardrails/backends/turing.py

@@ -105,8 +105,8 @@ def classify(
                 response = self._protect.protect(
                     inputs=content,
                     protect_rules=[
-                        {"metric": "content_moderation"},
-                        {"metric": "security"},
+                        {"metric": "toxicity"},
+                        {"metric": "prompt_injection"},
                         {"metric": "bias_detection"},
                     ],
                     timeout=30000,
@@ -195,12 +195,13 @@ def _extract_categories(
         """
         categories = set()
 
-        # Map rule names to categories
         rule_to_category = {
-            "content_moderation": "toxicity",
-            "security": "prompt_injection",
+            "toxicity": "toxicity",
+            "prompt_injection": "prompt_injection",
             "bias_detection": "hate_speech",
             "data_privacy_compliance": "pii",
+            "content_moderation": "toxicity",
+            "security": "prompt_injection",
             "ProtectFlash": "harmful_content",
         }
 

python/fi/evals/protect.py

@@ -9,12 +9,10 @@
 from fi.api.types import HttpMethod, RequestConfig
 from fi.evals.evaluator import EvalResponseHandler, Evaluator
 from fi.evals.templates import (
+    BiasDetection,
     DataPrivacyCompliance,
     PromptInjection,
-    Sexist,
-    Tone,
     Toxicity,
-    BiasDetection,
 )
 from fi.evals.protect_input_adapter import ProtectInputAdapter
 from fi.utils.routes import Routes
@@ -52,19 +50,26 @@ def __init__(self,
             fi_base_url=fi_base_url
         )
 
-        # Map metric names to their corresponding template classes
+        # Keep in sync with typescript/ai-evaluation/src/protect.ts.
         self.metric_map = {
             "toxicity": Toxicity,
             "bias_detection": BiasDetection,
             "prompt_injection": PromptInjection,
             "data_privacy_compliance": DataPrivacyCompliance,
-            # Deprecated aliases (still supported)
             "content_moderation": Toxicity,
             "security": PromptInjection,
+            "Toxicity": Toxicity,
+            "Sexism": BiasDetection,
+            "Prompt Injection": PromptInjection,
+            "Data Privacy": DataPrivacyCompliance,
         }
         self._deprecated_metrics = {
             "content_moderation": "toxicity",
             "security": "prompt_injection",
+            "Toxicity": "toxicity",
+            "Sexism": "bias_detection",
+            "Prompt Injection": "prompt_injection",
+            "Data Privacy": "data_privacy_compliance",
         }
 
     def _sanitize_reason(self, text: Optional[str]) -> Optional[str]:
@@ -94,14 +99,12 @@ def _check_rule_sync(
         # print(f"Starting rule check for {rule['metric']} in thread {thread_name} at {start_time}")
 
         template_class = self.metric_map[rule["metric"]]
-        if rule["metric"] == "Data Privacy":
+        if template_class is DataPrivacyCompliance:
             template = template_class(
                 config={"call_type": "protect", "check_internet": False}
             )
-            # template = template_class(config={"check_internet": False})
         else:
             template = template_class(config={"call_type": "protect"})
-            # template = template_class(config={})
 
         payload = {
             "inputs": [test_case.model_dump()],
@@ -343,7 +346,7 @@ def protect(
             inputs: Text or list of texts to check for harmful content
             timeout: Time limit for evaluation in milliseconds (default: 30000)
             protect_rules: Rules to check against. Each rule needs:
-                metric: What to check (e.g. 'content_moderation', 'bias_detection')
+                metric: What to check (e.g. 'toxicity', 'bias_detection')
                 contains: Values to look for
                 type: 'any' or 'all' matching required
                 action: Message to show if rule fails
@@ -375,12 +378,12 @@ def protect(
         if use_flash and not SUPPORT_PROTECT_FLASH:
             # Provide a sensible default so behavior is still helpful.
             if not protect_rules:
-                protect_rules = [{"metric": "content_moderation"}]
+                protect_rules = [{"metric": "toxicity"}]
             use_flash = False  # force normal path
 
         # When using ProtectFlash and no protect_rules provided, create default rules
         if use_flash and not protect_rules:
-            protect_rules = [{"metric": "content_moderation"}]
+            protect_rules = [{"metric": "toxicity"}]
         elif use_flash and protect_rules:
             print("Note: When using ProtectFlash, Rules are not considered as it performs binary harmful/not harmful classification only.")
 
@@ -514,7 +517,6 @@ def protect(
             raise InvalidValueType(value_name="protect_rules", value=protect_rules_copy, correct_type="non-empty list")
 
         valid_metrics = set(self.metric_map.keys())
-        valid_types = {"any", "all"}
 
         for i, rule in enumerate(protect_rules_copy):
 
@@ -546,35 +548,13 @@ def protect(
                     stacklevel=2,
                 )
 
-            is_tone_metric = rule["metric"] == "Tone"
+            if "contains" in rule:
+                raise SDKException(f"'contains' should not be specified for {rule['metric']} metric at index {i}.")
+            if "type" in rule:
+                raise SDKException(f"'type' should not be specified for {rule['metric']} metric at index {i}.")
 
-            if is_tone_metric:
-                if "contains" not in rule:
-                    raise MissingRequiredKey(field_name=f"Rule for Tone metric at index {i}", missing_key="contains")
-                if not isinstance(rule["contains"], list):
-                    raise InvalidValueType(value_name=f"'contains' in Tone rule at index {i}", value=rule["contains"], correct_type="list")
-                if not rule["contains"]:
-                    raise InvalidValueType(value_name=f"'contains' in Tone rule at index {i}", value=rule["contains"], correct_type="non-empty list")
-
-                # Type for Tone metric
-                if "type" not in rule:
-                    rule["type"] = "any" # Default if not present
-                elif rule["type"] not in valid_types:
-                    raise InvalidValueType(
-                        value_name=f"'type' in Tone rule at index {i}", 
-                        value=rule["type"], 
-                        correct_type=f"one of {valid_types}"
-                    )
-            else: # For non-Tone metrics
-                if "contains" in rule:
-                    # This indicates an invalid configuration for a non-Tone metric
-                    raise SDKException(f"'contains' should not be specified for {rule['metric']} metric at index {i}. Provide it only for 'Tone' metric.")
-                if "type" in rule:
-                    raise SDKException(f"'type' should not be specified for {rule['metric']} metric at index {i}. Provide it only for 'Tone' metric.")
-
-                # Set default values for internal processing of non-Tone metrics
-                rule["contains"] = ["Failed"] # Predefined internal value to check against for non-Tone metrics
-                rule["type"] = "any" # Default type for non-Tone metrics
+            rule["contains"] = ["Failed"]
+            rule["type"] = "any"
 
             # Validate action 
             if "action" not in rule:

python/tests/evals/local/test_protect_metric_map.py

@@ -0,0 +1,96 @@
+"""Unit tests for Protect's metric_map contract.
+
+Pins the canonical + legacy-alias set that must stay in sync between
+the Python and TypeScript SDKs and match the templates the backend
+accepts.
+"""
+
+import warnings
+
+import pytest
+
+from fi.evals.protect import Protect
+from fi.evals.templates import (
+    BiasDetection,
+    DataPrivacyCompliance,
+    PromptInjection,
+    Toxicity,
+)
+
+
+@pytest.fixture(autouse=True)
+def _stub_env(monkeypatch):
+    monkeypatch.setenv("FI_API_KEY", "test-key")
+    monkeypatch.setenv("FI_SECRET_KEY", "test-secret")
+
+
+def _client():
+    return Protect()
+
+
+CANONICAL = {
+    "toxicity": Toxicity,
+    "bias_detection": BiasDetection,
+    "prompt_injection": PromptInjection,
+    "data_privacy_compliance": DataPrivacyCompliance,
+}
+LEGACY_ALIASES = {
+    "content_moderation": Toxicity,
+    "security": PromptInjection,
+    "Toxicity": Toxicity,
+    "Sexism": BiasDetection,
+    "Prompt Injection": PromptInjection,
+    "Data Privacy": DataPrivacyCompliance,
+}
+
+
+@pytest.mark.parametrize("name,template", list(CANONICAL.items()))
+def test_canonical_metric_resolves_to_expected_template(name, template):
+    assert _client().metric_map[name] is template
+
+
+@pytest.mark.parametrize("alias,template", list(LEGACY_ALIASES.items()))
+def test_legacy_alias_resolves_to_same_template_as_canonical(alias, template):
+    assert _client().metric_map[alias] is template
+
+
+def test_tone_is_not_a_supported_metric():
+    assert "Tone" not in _client().metric_map
+
+
+def _swallowed_request(*args, **kwargs):
+    # _check_rule_sync wraps the request in try/except and returns a
+    # synthetic failed tuple, so the batch completes without raising.
+    raise Exception("stubbed")
+
+
+def test_legacy_alias_emits_future_warning(monkeypatch):
+    client = _client()
+    monkeypatch.setattr(client.evaluator, "request", _swallowed_request)
+
+    with warnings.catch_warnings(record=True) as caught:
+        warnings.simplefilter("always")
+        client.protect(
+            inputs="hello",
+            protect_rules=[{"metric": "content_moderation"}],
+        )
+    assert any(
+        issubclass(w.category, FutureWarning)
+        and "content_moderation" in str(w.message)
+        for w in caught
+    ), "expected a FutureWarning for the deprecated alias"
+
+
+def test_canonical_metric_does_not_emit_warning(monkeypatch):
+    client = _client()
+    monkeypatch.setattr(client.evaluator, "request", _swallowed_request)
+
+    with warnings.catch_warnings(record=True) as caught:
+        warnings.simplefilter("always")
+        client.protect(
+            inputs="hello",
+            protect_rules=[{"metric": "toxicity"}],
+        )
+    assert not any(
+        issubclass(w.category, FutureWarning) for w in caught
+    ), "canonical metric must not raise a deprecation warning"

typescript/ai-evaluation/src/__tests__/protect.test.ts

@@ -44,7 +44,8 @@ describe('Protect', () => {
 
     describe('Input and Rule Validation', () => {
         it('should throw an error for invalid inputs', async () => {
-            await expect(protect.protect(null as any)).rejects.toThrow('inputs with value null is of type object, but expected from string or list of strings');
+            // TS side currently accepts a single string only (Python parity gap - Python accepts string | list).
+            await expect(protect.protect(null as any)).rejects.toThrow('inputs with value null is of type object, but expected from string');
         });
 
         it('should throw an error for invalid protectRules', async () => {
@@ -78,19 +79,64 @@ describe('Protect', () => {
                 requestSpy.mockResolvedValue({
                     eval_results: [{ data: ['Not Toxic'], failure: false, reason: '' }]
                 });
-                const result = await protect.protect(mockInput, [{ metric: 'Toxicity' }]);
+                const result = await protect.protect(mockInput, [{ metric: 'toxicity' }]);
                 expect(result.status).toBe('passed');
-                expect(result.completed_rules).toContain('Toxicity');
+                expect(result.completed_rules).toContain('toxicity');
             });
 
             it('should return "failed" when a rule is triggered', async () => {
                 requestSpy.mockResolvedValue({
                     eval_results: [{ data: ['Failed'], failure: true, reason: 'High toxicity score' }]
                 });
-                const result = await protect.protect(mockInput, [{ metric: 'Toxicity' }], 'Blocked', true);
+                const result = await protect.protect(mockInput, [{ metric: 'toxicity' }], 'Blocked', true);
                 expect(result.status).toBe('failed');
                 expect(result.reasons).toBe('High toxicity score');
             });
         });
     });
-}); 
+
+    describe('Canonical + legacy metric parity with Python', () => {
+        const canonical = [
+            'toxicity',
+            'bias_detection',
+            'prompt_injection',
+            'data_privacy_compliance',
+        ];
+        const legacyAliases = [
+            'Toxicity',
+            'Sexism',
+            'Prompt Injection',
+            'Data Privacy',
+            'content_moderation',
+            'security',
+        ];
+
+        beforeEach(() => {
+            requestSpy.mockResolvedValue({
+                eval_results: [{ data: ['Not Toxic'], failure: false, reason: '' }],
+            });
+        });
+
+        it.each(canonical)('accepts canonical metric %s', async (metric) => {
+            const result = await protect.protect('hi', [{ metric }]);
+            expect(result.status).toBe('passed');
+            expect(result.completed_rules).toContain(metric);
+        });
+
+        it.each(legacyAliases)('accepts legacy alias %s with a deprecation warning', async (metric) => {
+            const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});
+            const result = await protect.protect('hi', [{ metric }]);
+            expect(result.status).toBe('passed');
+            expect(warnSpy).toHaveBeenCalledWith(
+                expect.stringContaining(`Protect metric "${metric}" is deprecated`),
+            );
+            warnSpy.mockRestore();
+        });
+
+        it('rejects the previously-supported "Tone" metric', async () => {
+            await expect(
+                protect.protect('hi', [{ metric: 'Tone', contains: ['friendly'] }]),
+            ).rejects.toThrow(/metric in Rule at index 0/);
+        });
+    });
+});