Merge pull request #481 from dgageot/remote-mcp-servers-by-ref

Better support remote MCP servers references by their name in the MCP Catalog

Author: dgageot

examples/apify.yaml

@@ -0,0 +1,14 @@
+#!/usr/bin/env cagent run
+version: "2"
+
+agents:
+  root:
+    model: openai/gpt-4o
+    description: Agent knowledgeable in Apify.
+    instruction: |
+      You are an AI assistant with a deep understanding of Apify.
+      Your responses should be clear, concise, and focused on providing accurate information about
+      Apify concepts, best practices, and usage.
+    toolsets:
+      - type: mcp
+        ref: docker:apify

pkg/gateway/catalog.go

@@ -13,17 +13,33 @@ import (
 const DockerCatalogURL = "https://desktop.docker.com/mcp/catalog/v3/catalog.yaml"
 
 func RequiredEnvVars(ctx context.Context, serverName string) ([]Secret, error) {
+	server, err := ServerSpec(ctx, serverName)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO(dga): until the MCP Gateway supports oauth with cagent,
+	// we ignore every secret listed on `remote` servers and assume
+	// we can use oauth by connecting directly to the server's url.
+	if server.Type == "remote" {
+		return nil, nil
+	}
+
+	return server.Secrets, nil
+}
+
+func ServerSpec(ctx context.Context, serverName string) (Server, error) {
 	catalog, err := readCatalogOnce()
 	if err != nil {
-		return nil, fmt.Errorf("failed to fetch MCP catalog: %w", err)
+		return Server{}, fmt.Errorf("failed to fetch MCP catalog: %w", err)
 	}
 
 	server, ok := catalog[serverName]
 	if !ok {
-		return nil, fmt.Errorf("MCP server %q not found in MCP catalog", serverName)
+		return Server{}, fmt.Errorf("MCP server %q not found in MCP catalog", serverName)
 	}
 
-	return server.Secrets, nil
+	return server, nil
 }
 
 // Read the MCP Catalog only once and cache the result.

pkg/gateway/catalog_test.go

@@ -7,11 +7,34 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestRequiredEnvVars(t *testing.T) {
+func TestRequiredEnvVars_local(t *testing.T) {
 	secrets, err := RequiredEnvVars(t.Context(), "github-official")
 	require.NoError(t, err)
 
 	assert.Len(t, secrets, 1)
 	assert.Equal(t, "GITHUB_PERSONAL_ACCESS_TOKEN", secrets[0].Env)
 	assert.Equal(t, "github.personal_access_token", secrets[0].Name)
 }
+
+func TestRequiredEnvVars_remote(t *testing.T) {
+	secrets, err := RequiredEnvVars(t.Context(), "apify")
+	require.NoError(t, err)
+
+	assert.Empty(t, secrets)
+}
+
+func TestServerSpec_local(t *testing.T) {
+	server, err := ServerSpec(t.Context(), "fetch")
+	require.NoError(t, err)
+
+	assert.Equal(t, "server", server.Type)
+}
+
+func TestServerSpec_remote(t *testing.T) {
+	server, err := ServerSpec(t.Context(), "apify")
+	require.NoError(t, err)
+
+	assert.Equal(t, "remote", server.Type)
+	assert.Equal(t, "https://mcp.apify.com", server.Remote.URL)
+	assert.Equal(t, "streamable-http", server.Remote.TransportType)
+}

pkg/gateway/types.go

@@ -7,7 +7,14 @@ type topLevel struct {
 type Catalog map[string]Server
 
 type Server struct {
+	Type    string   `json:"type"`
 	Secrets []Secret `json:"secrets,omitempty"`
+	Remote  Remote   `json:"remote,omitempty"`
+}
+
+type Remote struct {
+	URL           string `json:"url"`
+	TransportType string `json:"transport_type"`
 }
 
 type Secret struct {

pkg/teamloader/teamloader.go

@@ -346,6 +346,16 @@ func createTool(ctx context.Context, toolset latest.Toolset, a *latest.AgentConf
 
 	case toolset.Type == "mcp" && toolset.Ref != "":
 		mcpServerName := gateway.ParseServerRef(toolset.Ref)
+		serverSpec, err := gateway.ServerSpec(ctx, mcpServerName)
+		if err != nil {
+			return nil, fmt.Errorf("fetching MCP server spec for %q: %w", mcpServerName, err)
+		}
+
+		// TODO(dga): until the MCP Gateway supports oauth with cagent, we fetch the remote url and directly connect to it.
+		if serverSpec.Type == "remote" {
+			return mcp.NewRemoteToolset(serverSpec.Remote.URL, serverSpec.Remote.TransportType, nil, toolset.Tools, runtimeConfig.RedirectURI)
+		}
+
 		return mcp.NewGatewayToolset(mcpServerName, toolset.Config, toolset.Tools, envProvider), nil
 
 	case toolset.Type == "mcp" && toolset.Command != "":

pkg/tools/mcp/remote.go

@@ -92,7 +92,7 @@ func (c *remoteMCPClient) Initialize(ctx context.Context, request *mcp.Initializ
 			Endpoint:   c.url,
 			HTTPClient: httpClient,
 		}
-	case "streamable":
+	case "streamable", "streamable-http":
 		transport = &mcp.StreamableClientTransport{
 			Endpoint:   c.url,
 			HTTPClient: httpClient,