fix(mcp): probe-based OAuth detection in test-connection

apps/sim/app/api/mcp/servers/test-connection/route.ts

@@ -12,8 +12,9 @@ import {
   validateMcpServerSsrf,
 } from '@/lib/mcp/domain-check'
 import { getParsedBody, withMcpAuth } from '@/lib/mcp/middleware'
+import { detectMcpAuthType } from '@/lib/mcp/oauth'
 import { resolveMcpConfigEnvVars } from '@/lib/mcp/resolve-config'
-import type { McpTransport } from '@/lib/mcp/types'
+import type { McpAuthType, McpTransport } from '@/lib/mcp/types'
 import { createMcpErrorResponse, createMcpSuccessResponse } from '@/lib/mcp/utils'
 
 const logger = createLogger('McpServerTestAPI')
@@ -31,6 +32,8 @@ function isUrlBasedTransport(transport: McpTransport): boolean {
 interface TestConnectionResult {
   success: boolean
   error?: string
+  authRequired?: boolean
+  authType?: McpAuthType
   serverInfo?: {
     name: string
     version: string
@@ -163,6 +166,18 @@ export const POST = withRouteHandler(
       }
 
       const result: TestConnectionResult = { success: false }
+
+      // Skip unauth connect when the server returns an RFC 9728 OAuth challenge.
+      if (testConfig.url) {
+        const detectedAuthType = await detectMcpAuthType(testConfig.url)
+        if (detectedAuthType === 'oauth') {
+          result.authRequired = true
+          result.authType = 'oauth'
+          return createMcpSuccessResponse(result, 200)
+        }
+        result.authType = detectedAuthType
+      }
+
       let client: McpClient | null = null
 
       try {

apps/sim/app/workspace/[workspaceId]/settings/components/mcp/components/mcp-server-form-modal/mcp-server-form-modal.tsx

@@ -17,7 +17,7 @@ import {
   Textarea,
 } from '@/components/emcn'
 import { cn } from '@/lib/core/utils/cn'
-import type { McpTransport } from '@/lib/mcp/types'
+import type { McpAuthType, McpTransport } from '@/lib/mcp/types'
 import {
   checkEnvVarTrigger,
   EnvVarDropdown,
@@ -52,6 +52,7 @@ export interface McpServerFormConfig {
   timeout: number
   oauthClientId?: string
   oauthClientSecret?: string
+  authType?: McpAuthType
 }
 
 export interface McpServerFormModalProps {
@@ -109,11 +110,12 @@ interface EnvVarDropdownConfig {
 }
 
 function getTestButtonLabel(
-  testResult: { success: boolean; error?: string } | null,
+  testResult: { success: boolean; error?: string; authRequired?: boolean } | null,
   isTestingConnection: boolean
 ): string {
   if (isTestingConnection) return 'Testing...'
   if (testResult?.success) return 'Connection success'
+  if (testResult?.authRequired) return 'Requires OAuth'
   if (testResult && !testResult.success) return 'No connection: retry'
   return 'Test Connection'
 }
@@ -517,19 +519,11 @@ export function McpServerFormModal({
         workspaceId,
       })
 
-      if (!connectionResult.success) {
-        const errorText = (connectionResult.error || '').toLowerCase()
-        const looksLikeAuthRequired =
-          /\b401\b/.test(errorText) ||
-          errorText.includes('unauthorized') ||
-          errorText.includes('oauth') ||
-          errorText.includes('authentication')
-        if (!looksLikeAuthRequired) {
-          setSubmitError(
-            connectionResult.error || 'Connection test failed. Please check the URL and try again.'
-          )
-          return
-        }
+      if (!connectionResult.success && !connectionResult.authRequired) {
+        setSubmitError(
+          connectionResult.error || 'Connection test failed. Please check the URL and try again.'
+        )
+        return
       }
 
       await onSubmit({
@@ -538,6 +532,7 @@ export function McpServerFormModal({
         url: formData.url!,
         headers,
         timeout: formData.timeout || 30000,
+        authType: connectionResult.authType,
         oauthClientId:
           mode === 'edit'
             ? oauthClientIdChanged
@@ -587,7 +582,7 @@ export function McpServerFormModal({
         workspaceId,
       })
 
-      if (!connectionResult.success) {
+      if (!connectionResult.success && !connectionResult.authRequired) {
         setSubmitError(
           connectionResult.error || 'Connection test failed. Please check the URL and try again.'
         )
@@ -600,6 +595,7 @@ export function McpServerFormModal({
         url: config.url,
         headers: config.headers,
         timeout: 30000,
+        authType: connectionResult.authType,
       })
 
       onOpenChange(false)

apps/sim/hooks/queries/mcp.ts

@@ -22,7 +22,13 @@ import {
 } from '@/lib/api/contracts/mcp'
 import { isLoopbackHostname } from '@/lib/core/utils/urls'
 import { sanitizeForHttp, sanitizeHeaders } from '@/lib/mcp/shared'
-import type { McpServerStatusConfig, McpTool, McpTransport, StoredMcpTool } from '@/lib/mcp/types'
+import type {
+  McpAuthType,
+  McpServerStatusConfig,
+  McpTool,
+  McpTransport,
+  StoredMcpTool,
+} from '@/lib/mcp/types'
 import { workflowMcpServerKeys } from '@/hooks/queries/workflow-mcp-servers'
 
 const logger = createLogger('McpQueries')
@@ -54,6 +60,7 @@ export interface McpServerInput {
   enabled: boolean
   oauthClientId?: string
   oauthClientSecret?: string
+  authType?: McpAuthType
 }
 
 async function fetchMcpServers(workspaceId: string, signal?: AbortSignal): Promise<McpServer[]> {

apps/sim/lib/api/contracts/mcp.ts

@@ -252,6 +252,8 @@ export const mcpServerTestResultSchema = z.object({
   success: z.boolean(),
   message: z.string().optional(),
   error: z.string().optional(),
+  authRequired: z.boolean().optional(),
+  authType: mcpAuthTypeSchema.optional(),
   serverInfo: z
     .object({
       name: z.string(),