feat: add language-aware system prompt routing for multi-model agents

lib/crewai/src/crewai/agent/core.py

@@ -9,6 +9,7 @@
 from datetime import datetime
 import inspect
 import json
+import logging
 import os
 from pathlib import Path
 import time
@@ -117,6 +118,9 @@
     AgentResponseProtocol = None  # type: ignore[assignment, misc]
 
 
+logger = logging.getLogger(__name__)
+
+
 if TYPE_CHECKING:
     from crewai_files import FileInput
 
@@ -1013,6 +1017,39 @@ def _build_execution_prompt(
             response_template=self.response_template,
         ).task_execution()
 
+        # Language-aware prompt optimization: translate system prompt to the
+        # language the target model handles best before sending to LLM.
+        # Only translates system-level prompts — user messages carry intent
+        # and must never be machine-translated.
+        if self.auto_translate_prompt and self.llm:
+            try:
+                from crewai.utilities.prompt_translator import (
+                    optimize_system_prompt,
+                )
+
+                model_name = getattr(self.llm, "model", "") or ""
+                if isinstance(prompt, SystemPromptResult):
+                    if prompt.system:
+                        prompt.system = optimize_system_prompt(
+                            prompt.system,
+                            model_name,
+                            self.prompt_glossary,
+                            llm_caller=self.llm,
+                        )
+                else:
+                    if prompt.prompt:
+                        prompt.prompt = optimize_system_prompt(
+                            prompt.prompt,
+                            model_name,
+                            self.prompt_glossary,
+                            llm_caller=self.llm,
+                        )
+            except Exception:
+                logger.debug(
+                    "Prompt translation failed; using original prompt.",
+                    exc_info=True,
+                )
+
         stop_words = [I18N_DEFAULT.slice("observation")]
         if self.response_template:
             stop_words.append(

lib/crewai/src/crewai/agents/agent_builder/base_agent.py

@@ -391,6 +391,22 @@ class BaseAgent(BaseModel, ABC, metaclass=AgentMeta):
         description="Agent Skills. Accepts paths for discovery, inline SKILL.md strings, pre-loaded Skill objects, or '@org/name' registry refs.",
         min_length=1,
     )
+    auto_translate_prompt: bool = Field(
+        default=True,
+        description=(
+            "Automatically translate the system prompt to the language best "
+            "suited for the target model before each LLM call. Set to False "
+            "to disable and keep the prompt in its original language."
+        ),
+    )
+    prompt_glossary: dict[str, str] | None = Field(
+        default=None,
+        description=(
+            "Glossary of terms that should not be translated when "
+            "auto_translate_prompt is enabled. Maps source-term to "
+            "target-term (e.g. {'API Key': 'API Key', 'crewAI': 'crewAI'})."
+        ),
+    )
     execution_context: ExecutionContext | None = Field(default=None)
     checkpoint_kickoff_event_id: str | None = Field(default=None)
 

lib/crewai/src/crewai/utilities/model_profiles.py

@@ -0,0 +1,255 @@
+"""Model profile registry for language-aware prompt routing.
+
+Each model family has different tokenizer efficiency characteristics across
+languages. This module maps model identifiers to profiles that describe
+their language preferences and tokenizer behavior, enabling automatic
+system prompt optimization.
+
+References:
+    - Multi-IF benchmark (arXiv:2410.15553): Non-Latin scripts exhibit
+      systematically higher instruction-following error rates.
+    - PromptQuorum (2026-05): Documents "English SP + native UP" pattern
+      for DeepSeek-family models.
+    - Presenc AI tokenizer benchmark (2026-05): Quantifies token cost
+      asymmetry across model families.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import re
+
+
+@dataclass(frozen=True)
+class ModelProfile:
+    """Language capability profile for a model family.
+
+    Attributes:
+        preferred_language: ISO 639-1 code for the model's optimal prompt language.
+        english_tokens_per_word: Average tokens per English word for this model's tokenizer.
+        non_english_chars_per_token: Average non-English characters per token
+            (higher = more efficient for that language).
+        bilingual_capability: Score 0.0-1.0 indicating how well the model
+            handles mixed-language prompts. Higher = better.
+        family: Model family identifier for logging/debugging.
+    """
+
+    preferred_language: str
+    english_tokens_per_word: float
+    non_english_chars_per_token: float
+    bilingual_capability: float
+    family: str
+
+
+# ---------------------------------------------------------------------------
+# Registry: maps model name patterns to profiles.
+# Lookup uses boundary-aware regex matching to avoid false positives.
+# Unknown models return None (conservative: no translation).
+# ---------------------------------------------------------------------------
+
+# Each entry is (compiled_regex, ModelProfile).
+# The regex must match at a word boundary or after a separator (/, -, _).
+_MODEL_PATTERNS: list[tuple[re.Pattern[str], ModelProfile]] = [
+    # --- OpenAI ---
+    (
+        re.compile(r"(?:^|[/_-])gpt-4o(?:[/_-]|$)"),
+        ModelProfile(
+            preferred_language="en",
+            english_tokens_per_word=1.1,
+            non_english_chars_per_token=0.6,
+            bilingual_capability=0.85,
+            family="openai",
+        ),
+    ),
+    (
+        re.compile(r"(?:^|[/_-])gpt-5(?:[/_-]|$)"),
+        ModelProfile(
+            preferred_language="en",
+            english_tokens_per_word=1.1,
+            non_english_chars_per_token=0.6,
+            bilingual_capability=0.9,
+            family="openai",
+        ),
+    ),
+    (
+        re.compile(r"(?:^|[/_-])o1(?:-|preview|$)"),
+        ModelProfile(
+            preferred_language="en",
+            english_tokens_per_word=1.1,
+            non_english_chars_per_token=0.6,
+            bilingual_capability=0.85,
+            family="openai",
+        ),
+    ),
+    (
+        re.compile(r"(?:^|[/_-])o3(?:-|mini|$)"),
+        ModelProfile(
+            preferred_language="en",
+            english_tokens_per_word=1.1,
+            non_english_chars_per_token=0.6,
+            bilingual_capability=0.88,
+            family="openai",
+        ),
+    ),
+    (
+        re.compile(r"(?:^|[/_-])o4(?:-|mini|$)"),
+        ModelProfile(
+            preferred_language="en",
+            english_tokens_per_word=1.1,
+            non_english_chars_per_token=0.6,
+            bilingual_capability=0.88,
+            family="openai",
+        ),
+    ),
+    # --- Anthropic ---
+    (
+        re.compile(r"(?:^|[/_-])claude"),
+        ModelProfile(
+            preferred_language="en",
+            english_tokens_per_word=1.05,
+            non_english_chars_per_token=0.85,
+            bilingual_capability=0.9,
+            family="anthropic",
+        ),
+    ),
+    # --- Meta LLaMA ---
+    (
+        re.compile(r"(?:^|[/_-])llama"),
+        ModelProfile(
+            preferred_language="en",
+            english_tokens_per_word=1.1,
+            non_english_chars_per_token=0.4,
+            bilingual_capability=0.3,
+            family="llama",
+        ),
+    ),
+    # --- Qwen (Chinese-native) ---
+    (
+        re.compile(r"(?:^|[/_-])qwen"),
+        ModelProfile(
+            preferred_language="zh",
+            english_tokens_per_word=1.05,
+            non_english_chars_per_token=0.95,
+            bilingual_capability=0.95,
+            family="qwen",
+        ),
+    ),
+    # --- DeepSeek ---
+    (
+        re.compile(r"(?:^|[/_-])deepseek"),
+        ModelProfile(
+            preferred_language="en",
+            english_tokens_per_word=1.05,
+            non_english_chars_per_token=0.9,
+            bilingual_capability=0.85,
+            family="deepseek",
+        ),
+    ),
+    # --- Google Gemini ---
+    (
+        re.compile(r"(?:^|[/_-])gemini"),
+        ModelProfile(
+            preferred_language="en",
+            english_tokens_per_word=1.1,
+            non_english_chars_per_token=0.8,
+            bilingual_capability=0.85,
+            family="google",
+        ),
+    ),
+    # --- Mistral ---
+    (
+        re.compile(r"(?:^|[/_-])mistral"),
+        ModelProfile(
+            preferred_language="en",
+            english_tokens_per_word=1.1,
+            non_english_chars_per_token=0.5,
+            bilingual_capability=0.6,
+            family="mistral",
+        ),
+    ),
+    # --- Mixtral ---
+    (
+        re.compile(r"(?:^|[/_-])mixtral"),
+        ModelProfile(
+            preferred_language="en",
+            english_tokens_per_word=1.1,
+            non_english_chars_per_token=0.5,
+            bilingual_capability=0.6,
+            family="mistral",
+        ),
+    ),
+    # --- Amazon Nova ---
+    (
+        re.compile(r"(?:^|[/_-])nova"),
+        ModelProfile(
+            preferred_language="en",
+            english_tokens_per_word=1.1,
+            non_english_chars_per_token=0.7,
+            bilingual_capability=0.8,
+            family="amazon",
+        ),
+    ),
+    # --- Cohere ---
+    (
+        re.compile(r"(?:^|[/_-])command"),
+        ModelProfile(
+            preferred_language="en",
+            english_tokens_per_word=1.1,
+            non_english_chars_per_token=0.5,
+            bilingual_capability=0.6,
+            family="cohere",
+        ),
+    ),
+    # --- AI21 ---
+    (
+        re.compile(r"(?:^|[/_-])jamba"),
+        ModelProfile(
+            preferred_language="en",
+            english_tokens_per_word=1.1,
+            non_english_chars_per_token=0.6,
+            bilingual_capability=0.7,
+            family="ai21",
+        ),
+    ),
+]
+
+
+def get_model_profile(model_name: str) -> ModelProfile | None:
+    """Look up a model's language profile by name.
+
+    Uses boundary-aware regex matching to avoid false positives.
+    ``"gpt-4o-2024-05-13"`` matches the ``"gpt-4o"`` entry, but
+    ``"proto1"`` or ``"bio1"`` will not match ``"o1"``.
+
+    Returns ``None`` for unknown models so callers can fall back to
+    conservative (no-translation) behaviour.
+
+    Args:
+        model_name: The model identifier string (e.g. ``"gpt-4o"``).
+
+    Returns:
+        The matching :class:`ModelProfile`, or ``None`` if not found.
+    """
+    if not model_name:
+        return None
+
+    model_lower = model_name.lower()
+
+    # Normalize: treat `.`, `/`, `-`, `_` as equivalent separators
+    normalized = re.sub(r"[./_-]", "-", model_lower)
+
+    # Find all matching patterns and pick the longest match
+    best_match: re.Match[str] | None = None
+    best_profile: ModelProfile | None = None
+
+    for pattern, profile in _MODEL_PATTERNS:
+        match = pattern.search(normalized)
+        if match is not None:
+            # Prefer the match with the longest matched span
+            if best_match is None or (match.end() - match.start()) > (
+                best_match.end() - best_match.start()
+            ):
+                best_match = match
+                best_profile = profile
+
+    return best_profile