feat: hides headings by default on encoding

Author: Verdenroz

src/agon/__init__.py

@@ -3,7 +3,7 @@
 A self-describing, token-efficient data interchange format optimized for LLMs.
 """
 
-from agon.core import AGON, EncodingResult, Format
+from agon.core import AGON, AGONEncoding, Format
 from agon.errors import (
     AGONColumnsError,
     AGONError,
@@ -14,10 +14,10 @@
 __all__ = [
     "AGON",
     "AGONColumnsError",
+    "AGONEncoding",
     "AGONError",
     "AGONStructError",
     "AGONTextError",
-    "EncodingResult",
     "Format",
 ]
 __version__ = "0.1.0"

src/agon/core.py

@@ -13,7 +13,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast
+from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast, overload
 
 if TYPE_CHECKING:
     from collections.abc import Callable  # pragma: no cover
@@ -25,14 +25,44 @@
 from agon.formats import AGONColumns, AGONFormat, AGONStruct, AGONText
 
 Format = Literal["auto", "json", "text", "columns", "struct"]
+ConcreteFormat = Literal["json", "text", "columns", "struct"]
 
 
 @dataclass(frozen=True)
-class EncodingResult:
-    """Result of AGON encoding with format metadata."""
+class AGONEncoding:
+    r"""Result of AGON encoding with format metadata.
+
+    Use directly in LLM prompts - str() returns the encoded text.
+
+    Example:
+        >>> result = AGON.encode(data)
+        >>> prompt = f"Analyze this data:\\n{result}"  # uses __str__
+        >>> len(result)  # character count
+        >>> AGON.decode(response, format=result.format)
+    """
 
     format: Format
     text: str
+    header: str = ""
+
+    def __str__(self) -> str:
+        """Return encoded text for use in prompts."""
+        return self.text
+
+    def __len__(self) -> int:
+        """Return character count of encoded text."""
+        return len(self.text)
+
+    def __repr__(self) -> str:
+        """Return debug representation."""
+        preview = self.text[:50] + "..." if len(self.text) > 50 else self.text
+        return f"AGONEncoding(format={self.format!r}, len={len(self.text)}, text={preview!r})"
+
+    def with_header(self) -> str:
+        """Return encoded text with header prepended (for auto-detect decoding)."""
+        if not self.header:
+            return self.text
+        return f"{self.header}\n\n{self.text}"
 
 
 class AGON:
@@ -54,12 +84,20 @@ class AGON:
         - Self-describing: no training or config required.
     """
 
-    # Format registries
-    _encoders: ClassVar[dict[str, Callable[[Any], str]]] = {
+    # Format headers (for decoding)
+    _headers: ClassVar[dict[ConcreteFormat, str]] = {
+        "json": "",
+        "text": "@AGON text",
+        "columns": "@AGON columns",
+        "struct": "@AGON struct",
+    }
+
+    # Format registries (encode without headers - headers added separately)
+    _encoders: ClassVar[dict[ConcreteFormat, Callable[[Any], str]]] = {
         "json": lambda data: orjson.dumps(data).decode(),
-        "text": AGONText.encode,
-        "columns": AGONColumns.encode,
-        "struct": AGONStruct.encode,
+        "text": lambda data: AGONText.encode(data, include_header=False),
+        "columns": lambda data: AGONColumns.encode(data, include_header=False),
+        "struct": lambda data: AGONStruct.encode(data, include_header=False),
     }
 
     _decoders: ClassVar[dict[str, Callable[[str], Any]]] = {
@@ -70,17 +108,17 @@ class AGON:
 
     @staticmethod
     def encode(
-        data: Any,
+        data: object,
         *,
         format: Format = "auto",
         force: bool = False,
         min_savings: float = 0.10,
         encoding: str = DEFAULT_ENCODING,
-    ) -> str:
+    ) -> AGONEncoding:
         """Encode data to the most token-efficient AGON format.
 
         Args:
-            data: Data to encode. Any JSON-serializable value.
+            data: Data to encode. Must be JSON-serializable.
             format: Format to use:
                 - "auto": Select best format based on token count (default)
                 - "json": Raw JSON
@@ -92,58 +130,30 @@ def encode(
             encoding: Tiktoken encoding for token counting (default: o200k_base).
 
         Returns:
-            Encoded string in the selected format.
+            EncodingResult containing:
+            - format: The format used
+            - text: Encoded data (send this to LLMs)
+            - header: Format header (for decoding with auto-detect)
 
         Example:
             >>> data = [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}]
-            >>> AGON.encode(data, format="text")
-        """
-        # Direct format dispatch
-        if encoder := AGON._encoders.get(format):
-            return encoder(data)
-
-        # format == "auto": select best
-        candidates = [
-            (fmt, encoder(data))
-            for fmt, encoder in AGON._encoders.items()
-            if force is False or fmt != "json"
-        ]
-
-        # Select smallest token count
-        token_counts = [count_tokens(text, encoding=encoding) for _, text in candidates]
-        best_idx = min(range(len(candidates)), key=lambda i: token_counts[i])
-        best_format, best_text = candidates[best_idx]
-
-        # Apply min_savings threshold
-        if not force and best_format != "json":
-            json_idx = next(i for i, (fmt, _) in enumerate(candidates) if fmt == "json")
-            json_tokens = token_counts[json_idx]
-            savings = 1.0 - (token_counts[best_idx] / max(1, json_tokens))
-            if savings < min_savings:
-                return candidates[json_idx][1]
-
-        return best_text
-
-    @staticmethod
-    def encode_with_format(
-        data: Any,
-        *,
-        format: Format = "auto",
-        force: bool = False,
-        min_savings: float = 0.10,
-        encoding: str = DEFAULT_ENCODING,
-    ) -> EncodingResult:
-        """Encode data and return result with format metadata.
-
-        Same as encode() but returns an EncodingResult with format info.
+            >>> result = AGON.encode(data)
+            >>> response = send_to_llm(f"Analyze: {result}")  # uses __str__
+            >>> AGON.decode(response, result)  # decode using same format
         """
         # Direct format dispatch
-        if encoder := AGON._encoders.get(format):
-            return EncodingResult(format, encoder(data))
+        if format != "auto":
+            text = AGON._encoders[format](data)
+            header = AGON._headers[format]
+            return AGONEncoding(format, text, header)
 
         # format == "auto"
         candidates = [
-            EncodingResult(cast("Format", fmt), encoder(data))
+            AGONEncoding(
+                cast("Format", fmt),
+                encoder(data),
+                AGON._headers.get(fmt, ""),
+            )
             for fmt, encoder in AGON._encoders.items()
             if force is False or fmt != "json"
         ]
@@ -162,31 +172,64 @@ def encode_with_format(
 
         return best
 
+    @overload
     @staticmethod
-    def decode(payload: str) -> Any:
-        """Decode an AGON-encoded payload.
+    def decode(payload: AGONEncoding) -> Any: ...
 
-        Automatically detects the format by prefix matching.
+    @overload
+    @staticmethod
+    def decode(payload: str, format: Format | None = None) -> Any: ...
+
+    @staticmethod
+    def decode(
+        payload: str | AGONEncoding,
+        format: Format | None = None,
+    ) -> Any:
+        """Decode an AGON-encoded payload.
 
         Args:
-            payload: Encoded string in any AGON format.
+            payload: What to decode. Can be:
+                - AGONEncoding: Decode using its text and format
+                - str: Encoded string (use format param or auto-detect)
+            format: Format to use (only for str payload). If None, auto-detects.
 
         Returns:
             Decoded Python value.
 
         Raises:
             AGONError: If the payload is invalid.
-        """
-        payload = payload.strip()
 
-        # Prefix-based decoder dispatch
-        for prefix, decoder in AGON._decoders.items():
-            if payload.startswith(prefix):
-                return decoder(payload)
+        Example:
+            >>> result = AGON.encode(data)
+            >>> AGON.decode(result)  # decode AGONEncoding directly
+        """
+        if isinstance(payload, AGONEncoding):
+            format, payload = payload.format, payload.text
+
+        text = payload.strip()
+
+        # Auto-detect from header prefix
+        if format is None or format == "auto":
+            for prefix, decoder in AGON._decoders.items():
+                if text.startswith(prefix):
+                    return decoder(text)
+            return AGON._decode_json(text)
+
+        # Dispatch by format
+        match format:
+            case "json":
+                return AGON._decode_json(text)
+            case "text" | "columns" | "struct":
+                header = AGON._headers[cast("ConcreteFormat", format)]
+                if not text.startswith(header):
+                    text = f"{header}\n\n{text}"
+                return AGON._decoders[header](text)
 
-        # Fallback: raw JSON
+    @staticmethod
+    def _decode_json(text: str) -> object:
+        """Decode JSON text."""
         try:
-            return orjson.loads(payload)
+            return orjson.loads(text)
         except orjson.JSONDecodeError as e:
             raise AGONError(f"Invalid JSON: {e}") from e
 

src/agon/formats/base.py

@@ -15,6 +15,18 @@ class AGONFormat(ABC):
         - hint() -> str
     """
 
+    @staticmethod
+    @abstractmethod
+    def encode(data: object, *, include_header: bool = False) -> str:
+        """Encode data to this format."""
+        ...
+
+    @staticmethod
+    @abstractmethod
+    def decode(payload: str) -> object:
+        """Decode a payload in this format."""
+        ...
+
     @staticmethod
     @abstractmethod
     def hint() -> str:

src/agon/formats/columns.py

@@ -58,7 +58,7 @@ def hint() -> str:
 
     @staticmethod
     def encode(
-        data: Any,
+        data: object,
         *,
         delimiter: str = DEFAULT_DELIMITER,
         include_header: bool = True,

src/agon/formats/struct.py

@@ -69,7 +69,7 @@ def hint() -> str:
 
     @staticmethod
     def encode(
-        data: Any,
+        data: object,
         *,
         include_header: bool = True,
         min_occurrences: int = 3,
@@ -101,7 +101,10 @@ def encode(
             lines.append(HEADER)
             lines.append("")
 
-            # Emit struct definitions
+        # Emit struct definitions even when headers are disabled.
+        # The header is used for auto-detect decoding, but LLM prompts need
+        # the struct templates to interpret instances like FR(v1, v2).
+        if struct_defs:
             for name, fields, optional, parents in struct_defs:
                 fields_str = ", ".join(f + "?" if f in optional else f for f in fields)
                 if parents:
@@ -110,8 +113,7 @@ def encode(
                 else:
                     lines.append(f"@{name}: {fields_str}")
 
-            if struct_defs:
-                lines.append("")
+            lines.append("")
 
         _encode_value(data, lines, depth=0, registry=registry)
 
@@ -200,7 +202,7 @@ def _register_struct(
 
 
 def _detect_shapes(
-    data: Any,
+    data: object,
     shapes: Counter[tuple[str, ...]] | None = None,
 ) -> Counter[tuple[str, ...]]:
     """Detect repeated object shapes in data."""

src/agon/formats/text.py

@@ -51,7 +51,7 @@ def hint() -> str:
 
     @staticmethod
     def encode(
-        data: Any,
+        data: object,
         *,
         delimiter: str = DEFAULT_DELIMITER,
         include_header: bool = True,

tests/test_benchmarks.py

@@ -117,12 +117,12 @@ def test_fixture_benchmark(fixture_path: Path) -> None:
         format_results[fmt] = (tokens, savings, encode_ms, decode_ms)
 
     # Test auto selection
-    result = AGON.encode_with_format(records, format="auto")
+    result = AGON.encode(records, format="auto")
     auto_tokens = count_tokens(result.text)
     auto_savings = (1 - auto_tokens / max(1, raw_tokens)) * 100
 
-    # Verify auto decode
-    decoded = AGON.decode(result.text)
+    # Verify auto decode (decode AGONEncoding directly)
+    decoded = AGON.decode(result)
     assert normalize_floats(decoded) == normalize_floats(records), "auto roundtrip failed"
 
     # Print results