diff --git a/packages/claude-agents/README.md b/packages/claude-agents/README.md index aaaafa12..026a6d5e 100644 --- a/packages/claude-agents/README.md +++ b/packages/claude-agents/README.md @@ -131,6 +131,46 @@ async def main(): asyncio.run(main()) ``` +## Agent Skills (experimental) + +> **Experimental.** Imported from `launchdarkly_ai_claude_agents.experimental.skills`. The name and behavior may change in a minor release, and each change is listed under **Experimental** in the changelog. + +`create_claude_agents_handler()` doesn't give the model a variation's skills. `create_claude_agents_skills_handler()` does. It takes the same options, plus `skills_temp_dir`. + +```python +import os +from launchdarkly_ai_server import config +from launchdarkly_ai_server.experimental.skills import FDv2SkillStore, set_skill_store +from launchdarkly_ai_claude_agents.experimental.skills import ( + create_claude_agents_skills_handler, +) + +store = FDv2SkillStore(os.environ["LD_SDK_KEY"]).start() +store.wait_for_skills(timeout=10) +set_skill_store(store) + +triage = config( + key="triage-agent", + handler=create_claude_agents_skills_handler(setting_sources=[]), +) +``` + +On every invoke and stream it: + +1. Writes the variation's `skills` into a new directory under `skills_temp_dir` (default: `tempfile.gettempdir()`), through the store you configured with `set_skill_store`. Pinned versions are honored. +2. Hands that directory to the Claude Agent SDK as a local plugin named `launchdarkly`, enables only those skills (`launchdarkly:`), and turns on the `Skill` tool. +3. Removes the directory when the run ends, whether it succeeded, failed, or a stream was abandoned. + +Each run has its own directory, so concurrent runs and different variations never share files, and nothing needs to watch for changes. Detaching a skill from a variation takes effect on the next run. + +`setting_sources` behaves as on the core factory and is always sent, `[]` unless you give sources. Without it, `claude-agent-sdk` would load the `user` and `project` settings sources whenever skills are enabled. + +**A run whose skills can't be written doesn't start.** The handler raises before the model is called if no store is configured, the store hasn't received its first payload, a skill fails integrity verification, or a pinned version isn't in the store. The error names each skill. A variation with no skills runs exactly as it would with `create_claude_agents_handler()`, and needs no store. + +A `Skill` call shows up in traces as an `execute_tool Skill` span. + +**Not covered yet:** `claude_agents()`, `claude_graph()` and `to_claude_agents()` use the core handler, so a variation run through them doesn't get its skills. + ## How It Works - Uses the system prompt and conversation history defined in your LaunchDarkly flag config. diff --git a/packages/claude-agents/agents.md b/packages/claude-agents/agents.md index 22d25000..684928a7 100644 --- a/packages/claude-agents/agents.md +++ b/packages/claude-agents/agents.md @@ -30,6 +30,7 @@ That means the LaunchDarkly flag variation must have `provider.name == "Anthropi | `src/launchdarkly_ai_claude_agents/graph.py` | `claude_graph()` convenience wrapper around `graph()` | | `src/launchdarkly_ai_claude_agents/native_graph.py` | `to_claude_agents()` native graph adapter | | `src/launchdarkly_ai_claude_agents/builtins.py` | Pre-constructed `NativeTool` sentinels for Claude built-in tools | +| `src/launchdarkly_ai_claude_agents/experimental/skills.py` | Experimental `create_claude_agents_skills_handler()`. Not imported by the package `__init__.py` | | `src/launchdarkly_ai_claude_agents/__init__.py` | Package exports | | `src/launchdarkly_ai_claude_agents/_version.py` | `PACKAGE_NAME` and `__version__`, sent with `$ld:ai:sdk:info` and `$ld:ai:sdk:usage` | @@ -168,6 +169,20 @@ On error: `span.record_exception(exc)`, status set to ERROR, span ended, error r id wins. An app that opens a fresh CLI session per turn and re-feeds history must pass its own conversation id, or each turn becomes its own conversation. +### 5. Agent Skills (experimental, `experimental/skills.py`) + +`create_claude_agents_skills_handler(...)` is importable only from `launchdarkly_ai_claude_agents.experimental.skills`, and the package root never imports it (a test checks this in a fresh interpreter). It reports `experimental.createClaudeAgentsSkillsHandler` and builds the handler through `_create_claude_agents_handler(..., _prepare=...)`. + +`_prepare` is a package-internal hook, a private keyword argument of the internal factory only, so core names nothing experimental. It runs at the start of every invoke and stream, inside the `invoke_agent` span, and returns `None` (the run is exactly the core handler's) or a `QueryPreparation`. That holds extra `query()` options, extra built-ins appended to `tools` (never to `allowed_tools`), and a `cleanup` that both paths await last in their `finally`. + +For a variation with skills, `_prepare_skills`: + +1. `tempfile.mkdtemp(prefix="ld-skills-", dir=skills_temp_dir)`, then `os.path.realpath`, then `os.mkdir(/launchdarkly)`. +2. `write_skills(skill_refs(config), /launchdarkly/skills, prune=False)`. A report that is not `ok` raises, naming each failed key, and `query()` is never called. +3. Returns `plugins=[{"type": "local", "path": /launchdarkly}]`, `skills=["launchdarkly:", ...]` and `extra_builtins=["Skill"]`. + +The plugin's name comes from its directory's base name, so no manifest is written. The skill names are qualified because a bare name would also match a host plugin's skill. `claude-agent-sdk`'s `SdkPluginConfig` has no `skipMcpDiscovery` key, so unlike TypeScript the plugin entry carries only `type` and `path`. The directory holds no `.mcp.json`, so there is nothing to discover. `setting_sources` is never `None` here, because the core handler always sends it. With `skills` set and `setting_sources=None`, `claude-agent-sdk` sends `--setting-sources=user,project`. The tool hooks are installed whenever `Skill` is in `tools`, so a `Skill` call gets an `execute_tool Skill` span. The directory is removed with a synchronous `shutil.rmtree`, so a cancellation during teardown cannot interrupt it. There's no shared root, no watcher and no prune: every run writes exactly its own variation's references, which is also how detaching a skill takes effect. + --- ## OTel Setup diff --git a/packages/claude-agents/api-report/launchdarkly_ai_claude_agents.experimental.skills.txt b/packages/claude-agents/api-report/launchdarkly_ai_claude_agents.experimental.skills.txt new file mode 100644 index 00000000..76e498e4 --- /dev/null +++ b/packages/claude-agents/api-report/launchdarkly_ai_claude_agents.experimental.skills.txt @@ -0,0 +1,2 @@ +# Public API of launchdarkly_ai_claude_agents.experimental.skills. Generated by `make api-report`; do not edit. +create_claude_agents_skills_handler(*, capture_content: bool | None = None, cwd: str | os.PathLike[str] | None = None, setting_sources: list[SettingSource] | None = None, skills_temp_dir: str | os.PathLike[str] | None = None) -> ProviderHandler diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/experimental/__init__.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/experimental/__init__.py new file mode 100644 index 00000000..e6457fd8 --- /dev/null +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/experimental/__init__.py @@ -0,0 +1,7 @@ +""" +Experimental features of ``launchdarkly-ai-claude-agents``. + +Each feature is a submodule, for example ``launchdarkly_ai_claude_agents.experimental.skills``. +Names imported from here may change in a minor release, so review the changelog when you +upgrade. None of them is exported from the package root. +""" diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/experimental/skills.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/experimental/skills.py new file mode 100644 index 00000000..5dba59e7 --- /dev/null +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/experimental/skills.py @@ -0,0 +1,176 @@ +""" +Agent Skills for Claude Agents (experimental). + +``create_claude_agents_skills_handler`` builds a Claude Agents handler that gives each run the +skills attached to its variation. This name may change in a minor release, so review the +changelog when you upgrade. +""" + +from __future__ import annotations + +import logging +import os +import shutil +import tempfile +from collections.abc import Sequence + +from claude_agent_sdk import SettingSource + +from launchdarkly_ai_server import AiConfigRep, ProviderHandler, report_usage +from launchdarkly_ai_server.experimental.skills import ( + ReconcileReport, + SkillReference, + skill_refs, + write_skills, +) + +from .._version import PACKAGE_NAME, __version__ +from ..handler import QueryPreparation, _create_claude_agents_handler + +__all__ = ["create_claude_agents_skills_handler"] + +logger = logging.getLogger(__name__) + +PLUGIN_NAME = "launchdarkly" +"""The local plugin's directory name, and so its name: the CLI names a plugin with no manifest +after its directory, and lists its skills as ``launchdarkly:``.""" + +_RUN_DIR_PREFIX = "ld-skills-" +"""Prefix of each per-invoke directory under *skills_temp_dir*.""" + + +def _materialization_error( + report: ReconcileReport, refs: Sequence[SkillReference] +) -> RuntimeError: + """Why a run did not start, naming each skill that could not be written. + + Built from the report's action errors, which carry no skill content. A run-level error + carries an empty key, so the requested keys are named as well. + """ + failures = [ + f"{action.key}: {action.error}" if action.key else f"{action.error}" + for action in report.errors + ] + requested = ", ".join(ref.key for ref in refs) + return RuntimeError( + "Claude agent run not started: the skills its variation references could not be " + f"written (requested: {requested}). {'; '.join(failures)}" + ) + + +def _remove_run_directory(path: str) -> None: + """Removes one run's directory. A failure is logged, never raised over the run's own outcome. + + Synchronous on purpose: with no ``await`` in it, a cancellation arriving during teardown + cannot interrupt it and leave the directory behind. + """ + try: + shutil.rmtree(path) + except FileNotFoundError: + pass + except Exception as exc: + logger.warning( + "[LaunchDarkly] could not remove the Claude agent skills directory %s: %s", + path, + exc, + ) + + +async def _prepare_skills( + config: AiConfigRep, parent: str | os.PathLike[str] | None +) -> QueryPreparation | None: + """Writes the variation's skills into a directory private to this run, and returns the + options that hand it to ``query()`` as a local plugin. ``None`` when the variation has no + skills, so that run is exactly the core handler's. + + Fails closed: if any skill cannot be written, the run does not start. The variation names + that skill, so running without it would run a different agent from the one configured. + """ + refs = skill_refs(config) + if not refs: + return None + + run_dir = os.path.realpath(tempfile.mkdtemp(prefix=_RUN_DIR_PREFIX, dir=parent)) + try: + plugin_root = os.path.join(run_dir, PLUGIN_NAME) + # ``write_skills`` creates only the missing leaf of its root. + os.mkdir(plugin_root) + # A fresh directory: there is nothing to prune, and nothing else writes here. + report = await write_skills( + refs, os.path.join(plugin_root, "skills"), prune=False + ) + if not report.ok: + raise _materialization_error(report, refs) + except BaseException: + _remove_run_directory(run_dir) + raise + + async def _cleanup() -> None: + _remove_run_directory(run_dir) + + return QueryPreparation( + options={ + "plugins": [{"type": "local", "path": plugin_root}], + # Qualified: a bare name would also match a host plugin's skill of the same name. + "skills": [f"{PLUGIN_NAME}:{ref.key}" for ref in refs], + }, + extra_builtins=["Skill"], + cleanup=_cleanup, + ) + + +def create_claude_agents_skills_handler( + *, + capture_content: bool | None = None, + cwd: str | os.PathLike[str] | None = None, + setting_sources: list[SettingSource] | None = None, + skills_temp_dir: str | os.PathLike[str] | None = None, +) -> ProviderHandler: + """A Claude Agents handler that gives each run the skills attached to its variation. + + Takes the options of ``create_claude_agents_handler``, which behave the same way here, plus + *skills_temp_dir*: the parent of the directory each run writes its skills into, by default + ``tempfile.gettempdir()``. + + On every invoke and stream, the variation's ``skills`` are written with ``write_skills``, + through the store configured with ``set_skill_store``, into a new directory under + *skills_temp_dir*. That directory is passed to ``query()`` as a local plugin, only those + skills are enabled, the ``Skill`` tool is turned on, and the directory is removed when the + run ends. A variation with no skills runs exactly as with ``create_claude_agents_handler``. + + ``setting_sources`` is always sent, ``[]`` unless given. ``claude-agent-sdk`` would + otherwise load the ``user`` and ``project`` settings sources whenever ``skills`` is set. + + If any of the variation's skills cannot be written, because no store is configured, the + store has not received its first payload, a skill fails verification, or a pinned version + is absent, the run raises before the model is called. + + .. code-block:: python + + import os + + from launchdarkly_ai_server import config + from launchdarkly_ai_server.experimental.skills import FDv2SkillStore, set_skill_store + from launchdarkly_ai_claude_agents.experimental.skills import ( + create_claude_agents_skills_handler, + ) + + store = FDv2SkillStore(os.environ["LD_SDK_KEY"]).start() + store.wait_for_skills(timeout=10) + set_skill_store(store) + + triage = config(key="triage-agent", handler=create_claude_agents_skills_handler()) + """ + report_usage( + "experimental.createClaudeAgentsSkillsHandler", PACKAGE_NAME, __version__ + ) + + async def _prepare(config: AiConfigRep) -> QueryPreparation | None: + return await _prepare_skills(config, skills_temp_dir) + + return _create_claude_agents_handler( + capture_content=capture_content, + cwd=cwd, + setting_sources=setting_sources, + _prepare=_prepare, + ) diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py index c2a14d77..7dd92792 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py @@ -17,7 +17,8 @@ import asyncio import json import os -from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping +from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Mapping +from dataclasses import dataclass from typing import Any from claude_agent_sdk import ( @@ -655,8 +656,14 @@ def _build_query_options( mcp_allowed_tools: list[str], tool_mcp: Any, hooks: dict[str, list[HookMatcher]] | None, + extra_builtins: list[str] | None = None, **extra: Any, ) -> ClaudeAgentOptions: + """*extra_builtins* are built-ins the handler itself turns on, after the config's native tools, + because a per-invoke preparation asked for them (``Skill``, for example). They go in ``tools`` + only, never ``allowed_tools``: a built-in the Agent SDK enables through its own option needs no + permission entry here. + """ all_allowed = [*mcp_allowed_tools, *native_tool_names] params = _options_parameters(config) kwargs: dict[str, Any] = { @@ -670,7 +677,7 @@ def _build_query_options( # the full built-in set, and its read-only tools need no permission. A config turns a # built-in on by attaching its sentinel. User-defined tools go through the MCP server and # are not governed by `tools`. - "tools": list(native_tool_names), + "tools": [*native_tool_names, *(extra_builtins or [])], **extra, } if system_prompt: @@ -683,6 +690,27 @@ def _build_query_options( # --------------------------------------------------------------------------- +@dataclass(frozen=True) +class QueryPreparation: + """What a per-invoke preparation adds to one run. + + Package-internal and generic: it names no feature, so code at a later stage can register + through it without this module importing or naming that code. + """ + + options: dict[str, Any] + """Extra ``query()`` options, set with the handler's own host options.""" + extra_builtins: list[str] + """Built-in tool names appended to the config's native tools.""" + cleanup: Callable[[], Awaitable[None]] + """Runs once when the run ends, however it ends. Must not raise.""" + + +PrepareQuery = Callable[[AiConfigRep], Awaitable[QueryPreparation | None]] +"""Called at the start of every invoke and stream, inside the run's span, so a raise fails the run +like any other error. ``None`` means nothing to add, and the run is exactly the core handler's.""" + + def _host_query_options( cwd: str | os.PathLike[str] | None, setting_sources: list[SettingSource] | None, @@ -736,8 +764,13 @@ def _create_claude_agents_handler( capture_content: bool | None = None, cwd: str | os.PathLike[str] | None = None, setting_sources: list[SettingSource] | None = None, + _prepare: PrepareQuery | None = None, ) -> ProviderHandler: - """Non-reporting :func:`create_claude_agents_handler`, used by this package's wrappers.""" + """Non-reporting :func:`create_claude_agents_handler`, used by this package's wrappers. + + *_prepare* is a package-internal hook, never an option of the public factory: see + :data:`PrepareQuery`. + """ content_override = capture_content # Resolved once, so a caller mutating its list afterwards does not change later runs. host_options = _host_query_options(cwd, setting_sources) @@ -783,6 +816,7 @@ async def _call_impl( # already reported an authoritative run-level total and must not overwrite it. root_usage_written = False gen: AsyncIterator[Any] | None = None + prepared: QueryPreparation | None = None try: # Inside the guard, because serialising the prompt raises on anything that is not # JSON-serialisable and a raise out here would leave the root open: never ended, never @@ -794,6 +828,8 @@ async def _call_impl( messages=opening.messages, tool_definitions=catalog.current, ) + prepared = await _prepare(config) if _prepare is not None else None + extra_builtins = prepared.extra_builtins if prepared is not None else [] tool_mcp = ( await build_tool_mcp(user_config_tools, th) if user_config_tools @@ -801,7 +837,7 @@ async def _call_impl( ) mcp_allowed_tools = [MCP_TOOL_PREFIX + n for n in user_config_tools] hooks = None - if native_tool_names or mcp_allowed_tools: + if native_tool_names or mcp_allowed_tools or extra_builtins: # The blocking path cannot be abandoned, so it takes no abandon hook, but it can # still be cancelled, so it keeps cancel_open_spans. hooks, tool_telemetry, _, cancel_tool_spans = build_tool_hooks( @@ -815,7 +851,13 @@ async def _call_impl( mcp_allowed_tools, tool_mcp, hooks, - **host_options, + extra_builtins, + # Merged first, so a preparation's key replaces a host option's rather than + # colliding with it. + **{ + **host_options, + **(prepared.options if prepared is not None else {}), + }, ) output = "" @@ -898,18 +940,26 @@ async def _call_impl( # the root is the only span carrying the feature_flag event and the launchdarkly.* # attributes, so the whole run would vanish from AI Config Monitoring rather than show # as incomplete. - inference.finish() - if cancel_tool_spans is not None: - cancel_tool_spans() - if ( - open_root_span is not None - and not root_usage_written - and inference.run_usage["reported"] - ): - # The turns that completed were billed, the same reason the failure path reports - # them. - finish_root_span(open_root_span, config, inference.run_usage["total"]) - end_unfinished_spans(open_root_span) + try: + inference.finish() + if cancel_tool_spans is not None: + cancel_tool_spans() + if ( + open_root_span is not None + and not root_usage_written + and inference.run_usage["reported"] + ): + # The turns that completed were billed, the same reason the failure path + # reports them. + finish_root_span( + open_root_span, config, inference.run_usage["total"] + ) + end_unfinished_spans(open_root_span) + finally: + # Last, after the span tree is closed, and in its own finally so a raise above + # still removes the preparation's temp dir. + if prepared is not None: + await prepared.cleanup() def _stream_impl( config: AiConfigRep, @@ -926,6 +976,7 @@ def _stream_impl( history, capture_content=effective_capture_content(content_override), host_options=host_options, + prepare=_prepare, ) return _create_handler( @@ -990,6 +1041,7 @@ async def _stream_gen( *, capture_content: bool = False, host_options: dict[str, Any] | None = None, + prepare: PrepareQuery | None = None, ) -> AsyncGenerator[dict[str, Any], None]: """Streams the run, emitting the same span tree as the blocking path. @@ -1027,6 +1079,7 @@ async def _stream_gen( pending: list[dict[str, Any]] = [] block_ids: dict[int, str] = {} saw_args: set[str] = set() + prepared: QueryPreparation | None = None try: # Inside the guard, because serialising the prompt raises on anything that is not # JSON-serialisable. A raise out here would leave the root open with the `finally` never @@ -1038,6 +1091,8 @@ async def _stream_gen( messages=opening.messages, tool_definitions=catalog.current, ) + prepared = await prepare(config) if prepare is not None else None + extra_builtins = prepared.extra_builtins if prepared is not None else [] tool_mcp = ( await build_tool_mcp(user_config_tools, tool_handlers) if user_config_tools @@ -1045,7 +1100,7 @@ async def _stream_gen( ) mcp_allowed_tools = [MCP_TOOL_PREFIX + n for n in user_config_tools] hooks = None - if native_tool_names or mcp_allowed_tools: + if native_tool_names or mcp_allowed_tools or extra_builtins: hooks, tool_telemetry, abandon_tool_spans, _ = build_tool_hooks( native_tool_map, parent, @@ -1061,7 +1116,11 @@ async def _stream_gen( mcp_allowed_tools, tool_mcp, hooks, - **(host_options or _host_query_options(None, None)), + extra_builtins, + **{ + **(host_options or _host_query_options(None, None)), + **(prepared.options if prepared is not None else {}), + }, include_partial_messages=True, ) @@ -1179,25 +1238,34 @@ async def _stream_gen( yield rich raise finally: - # A no-op on the success path, where the result message already closed the last response. - # On the error and abandonment paths this is the only thing that ends a chat span still - # open, and the exporter only receives ended spans. - inference.finish() - # A no-op on the success and failure paths; on abandonment it is the only chance to close - # the tree, including any tool span whose PostToolUse hook never fired, and the only chance - # to report what the responses that did arrive cost. - if id(span) not in ended: - # Abandonment, not failure: UNSET plus the abandoned marker, the same as the model - # span and the root get just below. - if abandon_tool_spans is not None: - abandon_tool_spans(ended, cancelled=cancelled) - if not root_usage_written and inference.run_usage["reported"]: - finish_root_span(span, config, inference.run_usage["total"]) - end_span_once(span, ended, abandoned=True, cancelled=cancelled) - # Same reasoning as the blocking path: a bare exit through this generator's boundary - # abandons the vendor's own generator if it is not closed explicitly. - if gen is not None: - await gen.aclose() # type: ignore[attr-defined] + try: + # A no-op on the success path, where the result message already closed the last + # response. On the error and abandonment paths this is the only thing that ends a chat + # span still open, and the exporter only receives ended spans. + inference.finish() + # A no-op on the success and failure paths; on abandonment it is the only chance to + # close the tree, including any tool span whose PostToolUse hook never fired, and the + # only chance to report what the responses that did arrive cost. + if id(span) not in ended: + # Abandonment, not failure: UNSET plus the abandoned marker, the same as the model + # span and the root get just below. + if abandon_tool_spans is not None: + abandon_tool_spans(ended, cancelled=cancelled) + if not root_usage_written and inference.run_usage["reported"]: + finish_root_span(span, config, inference.run_usage["total"]) + end_span_once(span, ended, abandoned=True, cancelled=cancelled) + finally: + # In their own finally, so a raise closing the spans still closes the vendor's + # generator and removes the preparation's temp dir. Same reasoning as the blocking + # path: a bare exit through this generator's boundary abandons the vendor's own + # generator if it is not closed explicitly. + try: + if gen is not None: + await gen.aclose() # type: ignore[attr-defined] + finally: + # Last, after the span tree is closed: the run is over however it ended. + if prepared is not None: + await prepared.cleanup() def claude_agents( diff --git a/packages/claude-agents/tests/test_sdk_usage.py b/packages/claude-agents/tests/test_sdk_usage.py index fd576f0f..dff6c77c 100644 --- a/packages/claude-agents/tests/test_sdk_usage.py +++ b/packages/claude-agents/tests/test_sdk_usage.py @@ -15,6 +15,9 @@ ) from launchdarkly_ai_claude_agents._version import PACKAGE_NAME from launchdarkly_ai_claude_agents._version import __version__ as PACKAGE_VERSION +from launchdarkly_ai_claude_agents.experimental.skills import ( + create_claude_agents_skills_handler, +) from launchdarkly_ai_server import SDK_INFO_CONTEXT, __version__, init_client, shutdown CONTEXT = {"kind": "user", "key": "user-1"} @@ -55,6 +58,11 @@ async def client() -> Any: ("claude-agents.createClaudeAgentsHandler", create_claude_agents_handler), ("claude-agents.claudeGraph", lambda: claude_graph("k")), ("claude-agents.toClaudeAgents", lambda: to_claude_agents(None)), + # Its own ID, and not also the core factory's. + ( + "experimental.createClaudeAgentsSkillsHandler", + create_claude_agents_skills_handler, + ), ], ) async def test_helper_sends_one_usage_event( diff --git a/packages/claude-agents/tests/test_skills.py b/packages/claude-agents/tests/test_skills.py new file mode 100644 index 00000000..2c88a035 --- /dev/null +++ b/packages/claude-agents/tests/test_skills.py @@ -0,0 +1,757 @@ +""" +Tests for TESTING.md ยง2.x Claude Agents Skills Integration (A.18). + +``query()`` is replaced with a fake async generator that records the options it was given and can +look at the filesystem while the run is in progress. The skill store is a real +``InMemorySkillStore``, the filesystem is a per-test ``tmp_path`` passed as ``skills_temp_dir``, +and spans go to a real ``TracerProvider``. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import os +import subprocess +import sys +from collections.abc import AsyncIterator, Awaitable, Callable +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from claude_agent_sdk import ResultMessage, StreamEvent +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +import launchdarkly_ai_claude_agents.handler as handler_mod +import launchdarkly_ai_server.lifecycle as lifecycle_module +from launchdarkly_ai_claude_agents import ClaudeRead, create_claude_agents_handler +from launchdarkly_ai_claude_agents.experimental.skills import ( + create_claude_agents_skills_handler, +) +from launchdarkly_ai_server import NATIVE_TOOL_KEY, config, init_client, shutdown +from launchdarkly_ai_server.experimental.skills import ( + InMemorySkillStore, + set_skill_store, +) + +# --------------------------------------------------------------------------- +# Spans: share whichever SDK provider is installed, or install one +# --------------------------------------------------------------------------- + +_exporter = InMemorySpanExporter() +_provider = trace.get_tracer_provider() +if not isinstance(_provider, TracerProvider): + _provider = TracerProvider() + trace.set_tracer_provider(_provider) +_provider.add_span_processor(SimpleSpanProcessor(_exporter)) + + +@pytest.fixture(autouse=True) +def _reset() -> Any: + """Clears the exporter and the configured skill store around each test.""" + _exporter.clear() + lifecycle_module._reset_for_testing() + yield + lifecycle_module._reset_for_testing() + + +_PATHS = ["invoke", "stream"] + +SKILL_BODY = "---\nname: {key}\n---\nVersion {version} of {key}.\n" + +BASE_CONFIG: dict[str, Any] = { + "model": {"name": "claude-opus-4-5"}, + "provider": {"name": "Anthropic"}, + "instructions": "You are helpful.", +} + + +def _config(*refs: tuple[str, int], **extra: Any) -> dict[str, Any]: + return { + **BASE_CONFIG, + "skills": [{"key": key, "version": version} for key, version in refs], + **extra, + } + + +def _body(key: str, version: int) -> str: + return SKILL_BODY.format(key=key, version=version) + + +def _raw(key: str, version: int, *, content_hash: str | None = None) -> dict[str, Any]: + body = _body(key, version) + return { + "key": key, + "version": version, + "content": body, + "contentHash": content_hash or hashlib.sha256(body.encode()).hexdigest(), + } + + +def _store(*objects: dict[str, Any]) -> InMemorySkillStore: + store = InMemorySkillStore() + for raw in objects: + store.put(raw) + set_skill_store(store) + return store + + +def _result(subtype: str = "success") -> ResultMessage: + return ResultMessage( + subtype=subtype, + duration_ms=1, + duration_api_ms=1, + is_error=subtype != "success", + num_turns=1, + session_id="sess-1", + usage={"input_tokens": 3, "output_tokens": 2}, + result="done", + ) + + +def _text_event(text: str) -> StreamEvent: + return StreamEvent( + uuid="u1", + session_id="sess-1", + event={ + "type": "content_block_delta", + "delta": {"type": "text_delta", "text": text}, + }, + ) + + +During = Callable[[dict[str, Any]], Awaitable[None]] + + +class _FakeQuery: + """Stands in for ``query()``: records each call's options, runs *during* while the run is in + progress, then replays *messages*.""" + + def __init__( + self, messages: list[Any] | None = None, during: During | None = None + ) -> None: + self.calls: list[dict[str, Any]] = [] + self.messages = messages if messages is not None else [_result()] + self.during = during + + @property + def options(self) -> Any: + assert len(self.calls) == 1 + return self.calls[0]["options"] + + async def __call__(self, **kwargs: Any) -> AsyncIterator[Any]: + self.calls.append(kwargs) + if self.during is not None: + await self.during(kwargs) + for message in self.messages: + yield message + + +async def _run( + handler: Any, + path: str, + cfg: dict[str, Any], + tool_handlers: dict[str, Any] | None = None, + user_input: str = "q", +) -> Any: + if path == "invoke": + return await handler(cfg, user_input, tool_handlers or {}) + events = [] + async for event in await handler.stream(cfg, user_input, tool_handlers or {}): + events.append(event) + return events + + +def _plugin_root(options: Any) -> Path: + return Path(options.plugins[0]["path"]) + + +def _run_dir(options: Any) -> Path: + return _plugin_root(options).parent + + +def _read_stub() -> MagicMock: + stub = MagicMock() + setattr(stub, NATIVE_TOOL_KEY, ClaudeRead) + return stub + + +# --------------------------------------------------------------------------- +# Routing, options and the no-skills path +# --------------------------------------------------------------------------- + + +class TestRoutingAndOptions: + def test_routes_exactly_as_the_core_handler_does(self) -> None: + assert create_claude_agents_skills_handler().provides_for == ( + create_claude_agents_handler().provides_for + ) + assert tuple(create_claude_agents_skills_handler().provides_for) == ( + "Anthropic", + "agent", + ) + + def test_is_absent_from_the_package_root(self) -> None: + """A fresh interpreter, so this module's own import of the entry point doesn't count.""" + probe = ( + "import sys, launchdarkly_ai_claude_agents as p;" + "assert 'launchdarkly_ai_claude_agents.experimental' not in sys.modules;" + "assert not hasattr(p, 'create_claude_agents_skills_handler');" + "assert 'create_claude_agents_skills_handler' not in p.__all__" + ) + subprocess.run([sys.executable, "-c", probe], check=True) + + @pytest.mark.parametrize("path", _PATHS) + async def test_forwards_the_host_options( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, path: str + ) -> None: + _store(_raw("a", 1)) + fake = _FakeQuery() + monkeypatch.setattr(handler_mod, "query", fake) + handler = create_claude_agents_skills_handler( + cwd="/srv/app", setting_sources=["project"], skills_temp_dir=tmp_path + ) + await _run(handler, path, _config(("a", 1))) + assert fake.options.cwd == "/srv/app" + assert fake.options.setting_sources == ["project"] + + @pytest.mark.parametrize("path", _PATHS) + async def test_sends_an_explicit_empty_setting_sources_with_skills( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, path: str + ) -> None: + """``claude-agent-sdk`` sends ``--setting-sources=user,project`` when ``skills`` is set and + ``setting_sources`` is ``None``, so the handler must never leave it ``None``.""" + _store(_raw("a", 1)) + fake = _FakeQuery() + monkeypatch.setattr(handler_mod, "query", fake) + handler = create_claude_agents_skills_handler(skills_temp_dir=tmp_path) + await _run(handler, path, _config(("a", 1))) + assert fake.options.skills == ["launchdarkly:a"] + assert fake.options.setting_sources == [] + + @pytest.mark.parametrize("path", _PATHS) + @pytest.mark.parametrize("skills", [None, []]) + async def test_a_config_with_no_skills_runs_as_the_core_handler_would( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + path: str, + skills: list[Any] | None, + ) -> None: + """No store is configured: a variation with no skills needs none.""" + cfg = dict(BASE_CONFIG) if skills is None else {**BASE_CONFIG, "skills": skills} + core, experimental = _FakeQuery(), _FakeQuery() + monkeypatch.setattr(handler_mod, "query", core) + await _run(create_claude_agents_handler(cwd="/srv/app"), path, cfg) + monkeypatch.setattr(handler_mod, "query", experimental) + await _run( + create_claude_agents_skills_handler( + cwd="/srv/app", skills_temp_dir=tmp_path + ), + path, + cfg, + ) + assert experimental.options == core.options + assert experimental.options.plugins == [] + assert experimental.options.skills is None + assert "Skill" not in experimental.options.tools + assert list(tmp_path.iterdir()) == [] + + +# --------------------------------------------------------------------------- +# Materialization: one private directory per invoke +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("path", _PATHS) +class TestMaterialization: + async def test_writes_each_skill_into_a_private_plugin_directory( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, path: str + ) -> None: + _store(_raw("a", 1), _raw("b", 1)) + seen: dict[str, Any] = {} + + async def during(kwargs: dict[str, Any]) -> None: + plugin_root = _plugin_root(kwargs["options"]) + seen["run_dir"] = plugin_root.parent + seen["a"] = (plugin_root / "skills" / "a" / "SKILL.md").read_text() + seen["b"] = (plugin_root / "skills" / "b" / "SKILL.md").read_text() + + fake = _FakeQuery(during=during) + monkeypatch.setattr(handler_mod, "query", fake) + handler = create_claude_agents_skills_handler(skills_temp_dir=tmp_path) + await _run(handler, path, _config(("a", 1), ("b", 1))) + + run_dir = seen["run_dir"] + assert run_dir.parent == Path(os.path.realpath(tmp_path)) + assert run_dir.name.startswith("ld-skills-") + assert str(run_dir) == os.path.realpath(run_dir) + assert _plugin_root(fake.options).name == "launchdarkly" + assert seen["a"] == _body("a", 1) + assert seen["b"] == _body("b", 1) + + async def test_honors_a_pinned_version( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, path: str + ) -> None: + _store(_raw("a", 1), _raw("a", 2), _raw("a", 3)) + seen: list[str] = [] + + async def during(kwargs: dict[str, Any]) -> None: + skill = _plugin_root(kwargs["options"]) / "skills" / "a" / "SKILL.md" + seen.append(skill.read_text()) + + monkeypatch.setattr(handler_mod, "query", _FakeQuery(during=during)) + handler = create_claude_agents_skills_handler(skills_temp_dir=tmp_path) + await _run(handler, path, _config(("a", 2))) + assert seen == [_body("a", 2)] + + async def test_removes_the_directory_after_a_successful_run( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, path: str + ) -> None: + _store(_raw("a", 1)) + fake = _FakeQuery() + monkeypatch.setattr(handler_mod, "query", fake) + handler = create_claude_agents_skills_handler(skills_temp_dir=tmp_path) + await _run(handler, path, _config(("a", 1))) + assert not _run_dir(fake.options).exists() + assert list(tmp_path.iterdir()) == [] + + async def test_removes_the_directory_when_query_raises( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, path: str + ) -> None: + _store(_raw("a", 1)) + + async def during(_kwargs: dict[str, Any]) -> None: + raise RuntimeError("the CLI went away") + + monkeypatch.setattr(handler_mod, "query", _FakeQuery(during=during)) + handler = create_claude_agents_skills_handler(skills_temp_dir=tmp_path) + with pytest.raises(RuntimeError, match="the CLI went away"): + await _run(handler, path, _config(("a", 1))) + assert list(tmp_path.iterdir()) == [] + + async def test_removes_the_directory_after_a_non_success_result( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, path: str + ) -> None: + _store(_raw("a", 1)) + monkeypatch.setattr( + handler_mod, "query", _FakeQuery([_result("error_max_turns")]) + ) + handler = create_claude_agents_skills_handler(skills_temp_dir=tmp_path) + with pytest.raises(RuntimeError, match="error_max_turns"): + await _run(handler, path, _config(("a", 1))) + assert list(tmp_path.iterdir()) == [] + + async def test_a_new_invoke_does_not_write_a_detached_skill( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, path: str + ) -> None: + """Detaching a skill is a config change: the next invoke's ``skill_refs`` no longer names + it, so it is not written.""" + _store(_raw("a", 1), _raw("b", 1)) + held: list[list[str]] = [] + + async def during(kwargs: dict[str, Any]) -> None: + skills_dir = _plugin_root(kwargs["options"]) / "skills" + held.append(sorted(p.name for p in skills_dir.iterdir() if p.is_dir())) + + monkeypatch.setattr(handler_mod, "query", _FakeQuery(during=during)) + handler = create_claude_agents_skills_handler(skills_temp_dir=tmp_path) + await _run(handler, path, _config(("a", 1), ("b", 1))) + await _run(handler, path, _config(("a", 1))) + assert held == [["a", "b"], ["a"]] + + +async def test_removes_the_directory_when_closing_the_blocking_spans_raises( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _store(_raw("a", 1)) + monkeypatch.setattr(handler_mod, "query", _FakeQuery()) + + def boom(*_spans: Any) -> None: + raise RuntimeError("span cleanup failed") + + monkeypatch.setattr(handler_mod, "end_unfinished_spans", boom) + handler = create_claude_agents_skills_handler(skills_temp_dir=tmp_path) + with pytest.raises(RuntimeError, match="span cleanup failed"): + await _run(handler, "invoke", _config(("a", 1))) + assert list(tmp_path.iterdir()) == [] + + +async def test_removes_the_directory_when_closing_the_stream_spans_raises( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _store(_raw("a", 1)) + monkeypatch.setattr(handler_mod, "query", _FakeQuery()) + end_span_once = handler_mod.end_span_once + + def boom(span: Any, ended: set[int], **kwargs: Any) -> None: + # Only the teardown call: the success path ends the root without `abandoned`. + if kwargs.get("abandoned"): + raise RuntimeError("span cleanup failed") + end_span_once(span, ended, **kwargs) + + monkeypatch.setattr(handler_mod, "end_span_once", boom) + handler = create_claude_agents_skills_handler(skills_temp_dir=tmp_path) + with pytest.raises(RuntimeError, match="span cleanup failed"): + await _run(handler, "stream", _config(("a", 1))) + assert list(tmp_path.iterdir()) == [] + + +async def test_removes_the_directory_when_a_stream_consumer_stops_early( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _store(_raw("a", 1)) + fake = _FakeQuery([_text_event("hel"), _text_event("lo"), _result()]) + monkeypatch.setattr(handler_mod, "query", fake) + handler = create_claude_agents_skills_handler(skills_temp_dir=tmp_path) + gen = await handler.stream(_config(("a", 1)), "q", {}) + async for _event in gen: + assert _run_dir(fake.options).exists() + break + await gen.aclose() + assert list(tmp_path.iterdir()) == [] + + +async def test_concurrent_invokes_do_not_share_a_directory( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _store(_raw("a", 1)) + started = {"first": asyncio.Event(), "second": asyncio.Event()} + release = {"first": asyncio.Event(), "second": asyncio.Event()} + dirs: dict[str, Path] = {} + + async def during(kwargs: dict[str, Any]) -> None: + name = kwargs["prompt"] + dirs[name] = _run_dir(kwargs["options"]) + started[name].set() + await release[name].wait() + + monkeypatch.setattr(handler_mod, "query", _FakeQuery(during=during)) + handler = create_claude_agents_skills_handler(skills_temp_dir=tmp_path) + cfg = _config(("a", 1)) + first = asyncio.create_task(handler(cfg, "first", {})) + second = asyncio.create_task(handler(cfg, "second", {})) + await started["first"].wait() + await started["second"].wait() + + assert dirs["first"] != dirs["second"] + assert dirs["first"].exists() and dirs["second"].exists() + + release["first"].set() + await first + assert not dirs["first"].exists() + assert (dirs["second"] / "launchdarkly" / "skills" / "a" / "SKILL.md").exists() + + release["second"].set() + await second + assert list(tmp_path.iterdir()) == [] + + +async def test_a_failed_removal_is_logged_and_does_not_replace_the_outcome( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + import launchdarkly_ai_claude_agents.experimental.skills as skills_mod + + _store(_raw("a", 1)) + monkeypatch.setattr(handler_mod, "query", _FakeQuery()) + + def refuse(_path: str) -> None: + raise PermissionError("read-only") + + monkeypatch.setattr(skills_mod.shutil, "rmtree", refuse) + handler = create_claude_agents_skills_handler(skills_temp_dir=tmp_path) + result = await handler(_config(("a", 1)), "q", {}) + assert result["output"] == "done" + assert any("could not remove" in record.message for record in caplog.records) + + +# --------------------------------------------------------------------------- +# Fail closed +# --------------------------------------------------------------------------- + + +class _NotYetInitialized(InMemorySkillStore): + def is_initialized(self) -> bool: + return False + + +def _no_store() -> None: + pass + + +def _uninitialized_store() -> None: + store = _NotYetInitialized() + store.put(_raw("a", 1)) + set_skill_store(store) + + +def _withheld_by_verification() -> None: + _store(_raw("a", 1, content_hash="0" * 64)) + + +def _pin_not_held() -> None: + _store(_raw("a", 1), _raw("a", 3)) + + +@pytest.mark.parametrize("path", _PATHS) +@pytest.mark.parametrize( + "arrange", + [_no_store, _uninitialized_store, _withheld_by_verification, _pin_not_held], + ids=["no store", "not initialized", "withheld", "absent pin"], +) +async def test_a_reconcile_that_is_not_ok_fails_before_query( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + path: str, + arrange: Callable[[], None], +) -> None: + arrange() + fake = _FakeQuery() + monkeypatch.setattr(handler_mod, "query", fake) + handler = create_claude_agents_skills_handler(skills_temp_dir=tmp_path) + with pytest.raises(RuntimeError, match="not started") as excinfo: + await _run(handler, path, _config(("a", 2))) + assert fake.calls == [] + # Each failed key, with its action's error text. + assert "a: skill " in str(excinfo.value) + # The directory is removed on this path too. + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.parametrize("path", _PATHS) +async def test_one_failed_skill_stops_the_run_and_is_named( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, path: str +) -> None: + _store(_raw("good", 1), _raw("bad", 1, content_hash="0" * 64)) + fake = _FakeQuery() + monkeypatch.setattr(handler_mod, "query", fake) + handler = create_claude_agents_skills_handler(skills_temp_dir=tmp_path) + with pytest.raises(RuntimeError) as excinfo: + await _run(handler, path, _config(("good", 1), ("bad", 1))) + message = str(excinfo.value) + # The run does not start with the skill that could be written. + assert fake.calls == [] + assert "bad: skill 'bad' failed integrity verification" in message + assert "good: " not in message + # No skill content, and no path. + assert _body("good", 1) not in message + assert _body("bad", 1) not in message + assert str(tmp_path) not in message + assert list(tmp_path.iterdir()) == [] + + +# --------------------------------------------------------------------------- +# What query() receives +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("path", _PATHS) +class TestQueryOptions: + async def test_one_local_plugin_with_no_manifest( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, path: str + ) -> None: + _store(_raw("a", 1)) + manifests: list[bool] = [] + + async def during(kwargs: dict[str, Any]) -> None: + root = _plugin_root(kwargs["options"]) + manifests.append((root / ".claude-plugin").exists()) + + fake = _FakeQuery(during=during) + monkeypatch.setattr(handler_mod, "query", fake) + handler = create_claude_agents_skills_handler(skills_temp_dir=tmp_path) + await _run(handler, path, _config(("a", 1))) + root = _plugin_root(fake.options) + assert fake.options.plugins == [{"type": "local", "path": str(root)}] + assert root.name == "launchdarkly" + assert manifests == [False] + + async def test_enables_only_the_variation_skills_by_qualified_name( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, path: str + ) -> None: + _store(_raw("b", 1), _raw("a", 1), _raw("unattached", 1)) + fake = _FakeQuery() + monkeypatch.setattr(handler_mod, "query", fake) + handler = create_claude_agents_skills_handler(skills_temp_dir=tmp_path) + await _run(handler, path, _config(("b", 1), ("a", 1))) + assert fake.options.skills == ["launchdarkly:b", "launchdarkly:a"] + + async def test_skill_joins_the_builtin_set_once_last( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, path: str + ) -> None: + _store(_raw("a", 1)) + fake = _FakeQuery() + monkeypatch.setattr(handler_mod, "query", fake) + handler = create_claude_agents_skills_handler(skills_temp_dir=tmp_path) + await _run(handler, path, _config(("a", 1))) + assert fake.options.tools == ["Skill"] + assert "Skill" not in fake.options.allowed_tools + + with_read = _FakeQuery() + monkeypatch.setattr(handler_mod, "query", with_read) + await _run(handler, path, _config(("a", 1)), {"read": _read_stub()}) + assert with_read.options.tools == ["Read", "Skill"] + assert "Skill" not in with_read.options.allowed_tools + + async def test_no_config_parameter_reaches_the_handler_owned_keys( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, path: str + ) -> None: + _store(_raw("a", 1)) + cfg = _config( + ("a", 1), + model={ + "name": "claude-opus-4-5", + "parameters": { + "plugins": [{"type": "local", "path": "/from-config"}], + "skills": "all", + "tools": ["Bash"], + "setting_sources": ["user"], + }, + }, + ) + fake = _FakeQuery() + monkeypatch.setattr(handler_mod, "query", fake) + handler = create_claude_agents_skills_handler(skills_temp_dir=tmp_path) + await _run(handler, path, cfg) + assert fake.options.plugins == [ + {"type": "local", "path": str(_plugin_root(fake.options))} + ] + assert fake.options.skills == ["launchdarkly:a"] + assert fake.options.tools == ["Skill"] + assert fake.options.setting_sources == [] + + async def test_a_skill_call_gets_an_execute_tool_span( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, path: str + ) -> None: + """The hooks are installed whenever ``Skill`` is in ``tools``, even with no other tool.""" + _store(_raw("a", 1)) + + async def during(kwargs: dict[str, Any]) -> None: + hooks = kwargs["options"].hooks + event = { + "tool_name": "Skill", + "tool_use_id": "tu-skill", + "tool_input": {"skill": "launchdarkly:a"}, + "session_id": "sess-1", + } + await hooks["PreToolUse"][0].hooks[0]( + {**event, "hook_event_name": "PreToolUse"}, "tu-skill", None + ) + await hooks["PostToolUse"][0].hooks[0]( + {**event, "hook_event_name": "PostToolUse", "tool_response": "ok"}, + "tu-skill", + None, + ) + + monkeypatch.setattr(handler_mod, "query", _FakeQuery(during=during)) + handler = create_claude_agents_skills_handler(skills_temp_dir=tmp_path) + await _run(handler, path, _config(("a", 1))) + + finished = list(_exporter.get_finished_spans()) + root = next(s for s in finished if s.name == "invoke_agent") + skill = next(s for s in finished if s.name == "execute_tool Skill") + assert skill.attributes["gen_ai.tool.name"] == "Skill" + assert skill.parent is not None + assert skill.parent.span_id == root.context.span_id + + +# --------------------------------------------------------------------------- +# Through config(): the normal tracking path +# --------------------------------------------------------------------------- + +CONTEXT = {"kind": "user", "key": "user-1"} + + +@pytest.fixture +async def ld_client() -> Any: + await shutdown() + client = MagicMock() + client.track = MagicMock() + client.flush = MagicMock() + client.close = MagicMock() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=client) + yield client + await shutdown() + + +def _variation(client: MagicMock, *refs: tuple[str, int]) -> None: + async def variation(*_args: Any) -> dict[str, Any]: + return { + **_config(*refs), + "_ldMeta": { + "enabled": True, + "variationKey": "v1", + "version": 1, + "mode": "agent", + }, + } + + client.variation = variation + + +def _tracked(client: MagicMock) -> list[str]: + return [entry.args[0] for entry in client.track.call_args_list] + + +async def test_a_fail_closed_throw_is_tracked_as_a_generation_error( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ld_client: MagicMock +) -> None: + _variation(ld_client, ("a", 1)) + fake = _FakeQuery() + monkeypatch.setattr(handler_mod, "query", fake) + handler = create_claude_agents_skills_handler(skills_temp_dir=tmp_path) + with pytest.raises(RuntimeError, match="not started"): + await config(key="triage", handler=handler).invoke("q", CONTEXT) + assert fake.calls == [] + assert "$ld:ai:generation:error" in _tracked(ld_client) + + +async def test_a_skill_call_reports_no_tool_call_event( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ld_client: MagicMock +) -> None: + """``Skill`` is not an AI Config tool, so its call gets a span and no ``$ld:ai:tool_call``.""" + _variation(ld_client, ("a", 1)) + _store(_raw("a", 1)) + + async def during(kwargs: dict[str, Any]) -> None: + hooks = kwargs["options"].hooks + await hooks["PreToolUse"][0].hooks[0]( + { + "hook_event_name": "PreToolUse", + "tool_name": "Skill", + "tool_use_id": "tu-skill", + "tool_input": {"skill": "launchdarkly:a"}, + }, + "tu-skill", + None, + ) + + monkeypatch.setattr(handler_mod, "query", _FakeQuery(during=during)) + handler = create_claude_agents_skills_handler(skills_temp_dir=tmp_path) + await config(key="triage", handler=handler).invoke("q", CONTEXT) + tracked = _tracked(ld_client) + assert "$ld:ai:generation:success" in tracked + assert "$ld:ai:tool_call" not in tracked + + +# --------------------------------------------------------------------------- +# Not covered: the core handler never materializes skills +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("path", _PATHS) +async def test_the_core_handler_ignores_a_variation_skills( + monkeypatch: pytest.MonkeyPatch, path: str +) -> None: + _store(_raw("a", 1)) + fake = _FakeQuery() + monkeypatch.setattr(handler_mod, "query", fake) + await _run(create_claude_agents_handler(), path, _config(("a", 1))) + assert fake.options.plugins == [] + assert fake.options.skills is None + assert fake.options.tools == [] diff --git a/tests/test_never_forwarded_parameters.py b/tests/test_never_forwarded_parameters.py index df3d6297..d91ae208 100644 --- a/tests/test_never_forwarded_parameters.py +++ b/tests/test_never_forwarded_parameters.py @@ -29,6 +29,8 @@ #: checked against ``discover_handler_modules`` so a new handler module must be added here. FORWARDED_LISTS: dict[str, set[str]] = { "launchdarkly_ai_claude_agents.handler": {"_CLAUDE_AGENT_OPTIONS_FORWARDED_KEYS"}, + # Delegates to the core handler above, which forwards the parameters. + "launchdarkly_ai_claude_agents.experimental.skills": set(), "launchdarkly_ai_claude_messages.handler": {"_MESSAGES_FORWARDED_KEYS"}, "launchdarkly_ai_openai_agents.handler": {"_MODEL_SETTINGS_FORWARDED_KEYS"}, "launchdarkly_ai_openai_messages.handler": {"_RESPONSES_FORWARDED_KEYS"},