From 22b6da6eaee6b71cbe90f1eb1068060be06160d2 Mon Sep 17 00:00:00 2001 From: zdenekmusil-gd Date: Fri, 29 May 2026 12:56:10 +0200 Subject: [PATCH] feat(sdk): add LLM provider test & list-models enablers Add typed CatalogOrganizationService methods backed by ActionsApi so tooling can test connectivity and list models for LLM providers without using the raw api-client: - test_llm_provider / test_llm_provider_by_id -> CatalogLlmProviderTestResult - list_llm_provider_models / list_llm_provider_models_by_id -> CatalogLlmProviderModelsResult - Provider CRUD is unchanged JIRA: GDAI-1811 risk: nonprod --- .../gooddata-sdk/src/gooddata_sdk/__init__.py | 3 + .../organization/entity_model/llm_provider.py | 45 +++++++ .../catalog/organization/service.py | 78 +++++++++++ .../catalog/test_catalog_llm_provider.py | 125 ++++++++++++++++++ 4 files changed, 251 insertions(+) create mode 100644 packages/gooddata-sdk/tests/catalog/test_catalog_llm_provider.py diff --git a/packages/gooddata-sdk/src/gooddata_sdk/__init__.py b/packages/gooddata-sdk/src/gooddata_sdk/__init__.py index a23d22e66..9f8e46819 100644 --- a/packages/gooddata-sdk/src/gooddata_sdk/__init__.py +++ b/packages/gooddata-sdk/src/gooddata_sdk/__init__.py @@ -123,8 +123,11 @@ CatalogLlmProvider, CatalogLlmProviderDocument, CatalogLlmProviderModel, + CatalogLlmProviderModelsResult, CatalogLlmProviderPatch, CatalogLlmProviderPatchDocument, + CatalogLlmProviderTestResult, + CatalogModelTestResult, CatalogOpenAiApiKeyAuth, CatalogOpenAiProviderConfig, ) diff --git a/packages/gooddata-sdk/src/gooddata_sdk/catalog/organization/entity_model/llm_provider.py b/packages/gooddata-sdk/src/gooddata_sdk/catalog/organization/entity_model/llm_provider.py index 51d089eb5..84298d4d7 100644 --- a/packages/gooddata-sdk/src/gooddata_sdk/catalog/organization/entity_model/llm_provider.py +++ b/packages/gooddata-sdk/src/gooddata_sdk/catalog/organization/entity_model/llm_provider.py @@ -16,8 +16,11 @@ from gooddata_api_client.model.json_api_llm_provider_in_document import JsonApiLlmProviderInDocument from gooddata_api_client.model.json_api_llm_provider_patch import JsonApiLlmProviderPatch from gooddata_api_client.model.json_api_llm_provider_patch_document import JsonApiLlmProviderPatchDocument +from gooddata_api_client.model.list_llm_provider_models_response import ListLlmProviderModelsResponse +from gooddata_api_client.model.model_test_result import ModelTestResult from gooddata_api_client.model.open_ai_provider_auth import OpenAiProviderAuth from gooddata_api_client.model.open_ai_provider_config import OpenAIProviderConfig +from gooddata_api_client.model.test_llm_provider_response import TestLlmProviderResponse from gooddata_sdk.catalog.base import Base from gooddata_sdk.utils import safeget @@ -337,3 +340,45 @@ class CatalogLlmProviderPatchAttributes(Base): @staticmethod def client_class() -> type[JsonApiLlmProviderInAttributes]: return JsonApiLlmProviderInAttributes + + +# --- Action result types --- + + +@define(kw_only=True) +class CatalogModelTestResult(Base): + """Result of testing a single model on an LLM provider.""" + + model_id: str + successful: bool + message: str | None = None + + @staticmethod + def client_class() -> type[ModelTestResult]: + return ModelTestResult + + +@define(kw_only=True) +class CatalogLlmProviderTestResult(Base): + """Result of testing connectivity to an LLM provider.""" + + provider_reachable: bool + provider_message: str | None = None + model_results: list[CatalogModelTestResult] | None = None + + @staticmethod + def client_class() -> type[TestLlmProviderResponse]: + return TestLlmProviderResponse + + +@define(kw_only=True) +class CatalogLlmProviderModelsResult(Base): + """Result of listing the models available for an LLM provider.""" + + success: bool + message: str | None = None + models: list[CatalogLlmProviderModel] | None = None + + @staticmethod + def client_class() -> type[ListLlmProviderModelsResponse]: + return ListLlmProviderModelsResponse diff --git a/packages/gooddata-sdk/src/gooddata_sdk/catalog/organization/service.py b/packages/gooddata-sdk/src/gooddata_sdk/catalog/organization/service.py index c05de1350..250f0a264 100644 --- a/packages/gooddata-sdk/src/gooddata_sdk/catalog/organization/service.py +++ b/packages/gooddata-sdk/src/gooddata_sdk/catalog/organization/service.py @@ -16,7 +16,10 @@ from gooddata_api_client.model.json_api_organization_patch import JsonApiOrganizationPatch from gooddata_api_client.model.json_api_organization_patch_document import JsonApiOrganizationPatchDocument from gooddata_api_client.model.json_api_organization_setting_in_document import JsonApiOrganizationSettingInDocument +from gooddata_api_client.model.list_llm_provider_models_request import ListLlmProviderModelsRequest from gooddata_api_client.model.switch_identity_provider_request import SwitchIdentityProviderRequest +from gooddata_api_client.model.test_llm_provider_by_id_request import TestLlmProviderByIdRequest +from gooddata_api_client.model.test_llm_provider_definition_request import TestLlmProviderDefinitionRequest from gooddata_sdk import CatalogDeclarativeExportTemplate, CatalogExportTemplate from gooddata_sdk.catalog.catalog_service_base import CatalogServiceBase @@ -25,9 +28,13 @@ from gooddata_sdk.catalog.organization.entity_model.jwk import CatalogJwk, CatalogJwkDocument from gooddata_sdk.catalog.organization.entity_model.llm_provider import ( CatalogLlmProvider, + CatalogLlmProviderConfig, CatalogLlmProviderDocument, + CatalogLlmProviderModel, + CatalogLlmProviderModelsResult, CatalogLlmProviderPatch, CatalogLlmProviderPatchDocument, + CatalogLlmProviderTestResult, ) from gooddata_sdk.catalog.organization.entity_model.setting import CatalogOrganizationSetting from gooddata_sdk.catalog.organization.layout.identity_provider import CatalogDeclarativeIdentityProvider @@ -628,6 +635,77 @@ def delete_llm_provider(self, id: str) -> None: """ self._entities_api.delete_entity_llm_providers(id, _check_return_type=False) + def test_llm_provider( + self, + provider_config: CatalogLlmProviderConfig, + models: list[CatalogLlmProviderModel] | None = None, + ) -> CatalogLlmProviderTestResult: + """Test connectivity to an unsaved LLM provider configuration. + + Args: + provider_config: Provider configuration to test (OpenAI, AWS Bedrock, or Azure Foundry). + models: Optional list of models to test against the provider. + + Returns: + CatalogLlmProviderTestResult: Reachability and per-model test results. + """ + payload: dict = {"providerConfig": provider_config.to_api().to_dict()} + if models is not None: + payload["models"] = [m.to_api().to_dict() for m in models] + request = TestLlmProviderDefinitionRequest.from_dict(payload) + response = self._actions_api.test_llm_provider(request) + return CatalogLlmProviderTestResult.from_api(response) + + def test_llm_provider_by_id( + self, + id: str, + models: list[CatalogLlmProviderModel] | None = None, + ) -> CatalogLlmProviderTestResult: + """Test connectivity to a saved LLM provider by its identifier. + + Args: + id: LLM provider identifier. + models: Optional list of models to test (overrides the provider's models). + + Returns: + CatalogLlmProviderTestResult: Reachability and per-model test results. + """ + kwargs: dict = {} + if models is not None: + kwargs["test_llm_provider_by_id_request"] = TestLlmProviderByIdRequest.from_dict( + {"models": [m.to_api().to_dict() for m in models]} + ) + response = self._actions_api.test_llm_provider_by_id(id, **kwargs) + return CatalogLlmProviderTestResult.from_api(response) + + def list_llm_provider_models( + self, + provider_config: CatalogLlmProviderConfig, + ) -> CatalogLlmProviderModelsResult: + """List the models available for an unsaved LLM provider configuration. + + Args: + provider_config: Provider configuration to query. + + Returns: + CatalogLlmProviderModelsResult: Available models and query status. + """ + request = ListLlmProviderModelsRequest.from_dict({"providerConfig": provider_config.to_api().to_dict()}) + response = self._actions_api.list_llm_provider_models(request) + return CatalogLlmProviderModelsResult.from_api(response) + + def list_llm_provider_models_by_id(self, id: str) -> CatalogLlmProviderModelsResult: + """List the models available for a saved LLM provider by its identifier. + + Args: + id: LLM provider identifier. + + Returns: + CatalogLlmProviderModelsResult: Available models and query status. + """ + response = self._actions_api.list_llm_provider_models_by_id(id) + return CatalogLlmProviderModelsResult.from_api(response) + # Layout APIs def get_declarative_notification_channels(self) -> list[CatalogDeclarativeNotificationChannel]: diff --git a/packages/gooddata-sdk/tests/catalog/test_catalog_llm_provider.py b/packages/gooddata-sdk/tests/catalog/test_catalog_llm_provider.py new file mode 100644 index 000000000..a969169b6 --- /dev/null +++ b/packages/gooddata-sdk/tests/catalog/test_catalog_llm_provider.py @@ -0,0 +1,125 @@ +# (C) 2026 GoodData Corporation +from __future__ import annotations + +from unittest.mock import MagicMock + +from gooddata_api_client.model.list_llm_provider_models_response import ListLlmProviderModelsResponse +from gooddata_api_client.model.llm_model import LlmModel +from gooddata_api_client.model.model_test_result import ModelTestResult +from gooddata_api_client.model.test_llm_provider_response import TestLlmProviderResponse +from gooddata_sdk import ( + CatalogLlmProviderModel, + CatalogLlmProviderModelsResult, + CatalogLlmProviderTestResult, + CatalogModelTestResult, + CatalogOpenAiApiKeyAuth, + CatalogOpenAiProviderConfig, +) +from gooddata_sdk.catalog.organization.service import CatalogOrganizationService + + +def test_provider_test_result_from_api(): + api_response = TestLlmProviderResponse( + provider_reachable=True, + provider_message="ok", + model_results=[ModelTestResult(model_id="gpt-4o", successful=True, message="ok")], + ) + + result = CatalogLlmProviderTestResult.from_api(api_response) + + assert result.provider_reachable is True + assert result.provider_message == "ok" + assert len(result.model_results) == 1 + assert isinstance(result.model_results[0], CatalogModelTestResult) + assert result.model_results[0].model_id == "gpt-4o" + assert result.model_results[0].successful is True + assert result.model_results[0].message == "ok" + + +def test_provider_models_result_from_api(): + api_response = ListLlmProviderModelsResponse( + success=True, + message="ok", + models=[LlmModel(id="gpt-4o", family="OPENAI")], + ) + + result = CatalogLlmProviderModelsResult.from_api(api_response) + + assert result.success is True + assert result.message == "ok" + assert len(result.models) == 1 + assert isinstance(result.models[0], CatalogLlmProviderModel) + assert result.models[0].id == "gpt-4o" + assert result.models[0].family == "OPENAI" + + +def _openai_config(): + return CatalogOpenAiProviderConfig( + auth=CatalogOpenAiApiKeyAuth(api_key="secret"), + base_url="https://api.openai.com/v1", + ) + + +def test_test_llm_provider_maps_response(): + service = CatalogOrganizationService.__new__(CatalogOrganizationService) + service._actions_api = MagicMock() + service._actions_api.test_llm_provider.return_value = TestLlmProviderResponse( + provider_reachable=False, + provider_message="bad key", + model_results=[], + ) + + result = service.test_llm_provider(_openai_config()) + + assert result.provider_reachable is False + assert result.provider_message == "bad key" + # the request body passed to the client carries the provider config discriminator + sent = service._actions_api.test_llm_provider.call_args.args[0] + assert sent["provider_config"]["type"] == "OPENAI" + + +def test_test_llm_provider_by_id_calls_client_with_id(): + service = CatalogOrganizationService.__new__(CatalogOrganizationService) + service._actions_api = MagicMock() + service._actions_api.test_llm_provider_by_id.return_value = TestLlmProviderResponse( + provider_reachable=True, + provider_message="ok", + model_results=[], + ) + + result = service.test_llm_provider_by_id("my-provider") + + assert result.provider_reachable is True + assert service._actions_api.test_llm_provider_by_id.call_args.args[0] == "my-provider" + + +def test_list_llm_provider_models_maps_response(): + service = CatalogOrganizationService.__new__(CatalogOrganizationService) + service._actions_api = MagicMock() + service._actions_api.list_llm_provider_models.return_value = ListLlmProviderModelsResponse( + success=True, + message="ok", + models=[LlmModel(id="gpt-4o", family="OPENAI")], + ) + + result = service.list_llm_provider_models(_openai_config()) + + assert result.success is True + assert result.models[0].id == "gpt-4o" + sent = service._actions_api.list_llm_provider_models.call_args.args[0] + assert sent["provider_config"]["type"] == "OPENAI" + + +def test_list_llm_provider_models_by_id_calls_client_with_id(): + service = CatalogOrganizationService.__new__(CatalogOrganizationService) + service._actions_api = MagicMock() + service._actions_api.list_llm_provider_models_by_id.return_value = ListLlmProviderModelsResponse( + success=True, + message="ok", + models=[], + ) + + result = service.list_llm_provider_models_by_id("my-provider") + + assert result.success is True + assert service._actions_api.list_llm_provider_models_by_id.call_args.args[0] == "my-provider"