|
| 1 | +name: MCP Setup Smoke Test |
| 2 | + |
| 3 | +# What this workflow proves |
| 4 | +# ───────────────────────── |
| 5 | +# The setup_devops_os_mcp.sh script is the primary on-boarding path for users |
| 6 | +# who want to connect DevOps-OS to Claude Code. This workflow proves the |
| 7 | +# full installation flow works end-to-end on a clean Ubuntu machine: |
| 8 | +# |
| 9 | +# 1. setup_devops_os_mcp.sh --local creates a Python venv at .venv/ |
| 10 | +# 2. All MCP server dependencies are installed (importable from the venv) |
| 11 | +# 3. The MCP server process starts successfully under the venv interpreter |
| 12 | +# 4. The server completes the MCP initialize handshake |
| 13 | +# 5. tools/list returns all 8 expected DevOps-OS tools |
| 14 | +# 6. The claude mcp add / claude mcp list commands are exercised via a |
| 15 | +# stub so the registration code path is validated even without a real |
| 16 | +# Claude CLI binary or API key |
| 17 | + |
| 18 | +on: |
| 19 | + push: |
| 20 | + branches: [main, "copilot/**"] |
| 21 | + paths: |
| 22 | + - "mcp_server/**" |
| 23 | + - ".github/workflows/mcp-setup-smoke.yml" |
| 24 | + pull_request: |
| 25 | + branches: [main] |
| 26 | + paths: |
| 27 | + - "mcp_server/**" |
| 28 | + - ".github/workflows/mcp-setup-smoke.yml" |
| 29 | + |
| 30 | +permissions: |
| 31 | + contents: read |
| 32 | + |
| 33 | +jobs: |
| 34 | + mcp-setup-smoke: |
| 35 | + name: MCP Setup Smoke Test |
| 36 | + runs-on: ubuntu-latest |
| 37 | + timeout-minutes: 20 |
| 38 | + |
| 39 | + steps: |
| 40 | + - uses: actions/checkout@v4 |
| 41 | + |
| 42 | + - name: Set up Python 3.11 |
| 43 | + uses: actions/setup-python@v5 |
| 44 | + with: |
| 45 | + python-version: "3.11" |
| 46 | + |
| 47 | + # ── Step 1: stub the 'claude' binary ──────────────────────────────────── |
| 48 | + # The setup script calls `claude mcp list` and `claude mcp add`. |
| 49 | + # We provide a minimal stub that: |
| 50 | + # • Returns a non-zero exit code for `mcp list` (so the script skips |
| 51 | + # the "remove existing entry" path) — mirrors a fresh install. |
| 52 | + # • Exits 0 for `mcp add` and logs the invocation so we can assert |
| 53 | + # the correct arguments were passed. |
| 54 | + # This validates the registration code path without a real Claude binary. |
| 55 | + - name: Install claude CLI stub |
| 56 | + run: | |
| 57 | + mkdir -p "$HOME/.local/bin" |
| 58 | + cat > "$HOME/.local/bin/claude" << 'EOF' |
| 59 | + #!/usr/bin/env bash |
| 60 | + # Claude CLI stub for CI smoke testing. |
| 61 | + # Logs every invocation to $HOME/claude-stub.log. |
| 62 | + echo "claude stub called: $*" >> "$HOME/claude-stub.log" |
| 63 | + if [[ "$1 $2" == "mcp list" ]]; then |
| 64 | + # Return 1 so the 'grep -q "^devops-os"' check fails ─ simulates |
| 65 | + # a clean install with no prior registration. |
| 66 | + exit 1 |
| 67 | + fi |
| 68 | + # All other sub-commands (mcp add, mcp remove) succeed silently. |
| 69 | + exit 0 |
| 70 | + EOF |
| 71 | + chmod +x "$HOME/.local/bin/claude" |
| 72 | + echo "$HOME/.local/bin" >> "$GITHUB_PATH" |
| 73 | +
|
| 74 | + # ── Step 2: run the setup script in local mode ────────────────────────── |
| 75 | + - name: Run setup_devops_os_mcp.sh --local |
| 76 | + run: bash mcp_server/setup_devops_os_mcp.sh --local |
| 77 | + |
| 78 | + # ── Step 3: verify .venv was created ──────────────────────────────────── |
| 79 | + - name: Verify .venv was created |
| 80 | + run: | |
| 81 | + echo "Checking for .venv directory..." |
| 82 | + test -d .venv || { echo "ERROR: .venv was not created by setup script"; exit 1; } |
| 83 | + test -f .venv/bin/python || { echo "ERROR: .venv/bin/python not found"; exit 1; } |
| 84 | + echo "✓ .venv exists: $(.venv/bin/python --version)" |
| 85 | +
|
| 86 | + # ── Step 4: verify MCP server dependencies are installed ──────────────── |
| 87 | + - name: Verify MCP server dependencies installed in venv |
| 88 | + run: | |
| 89 | + echo "Checking that 'mcp' package is importable from .venv..." |
| 90 | + .venv/bin/python -c " |
| 91 | +import importlib.metadata |
| 92 | +v = importlib.metadata.version('mcp') |
| 93 | +from mcp.server.fastmcp import FastMCP |
| 94 | +print('✓ mcp version:', v, '— FastMCP importable') |
| 95 | +" || { echo "ERROR: mcp package not installed in .venv"; exit 1; } |
| 96 | + |
| 97 | + echo "Checking that 'yaml' package is importable from .venv..." |
| 98 | + .venv/bin/python -c "import yaml; print('✓ pyyaml imported')" \ |
| 99 | + || { echo "ERROR: pyyaml not installed in .venv"; exit 1; } |
| 100 | + |
| 101 | + # ── Step 5: verify claude stub was called with the right arguments ─────── |
| 102 | + - name: Verify claude mcp add was invoked with correct arguments |
| 103 | + run: | |
| 104 | + echo "Contents of claude-stub.log:" |
| 105 | + cat "$HOME/claude-stub.log" |
| 106 | +
|
| 107 | + # Assert 'claude mcp add' was called |
| 108 | + grep -q "mcp add" "$HOME/claude-stub.log" \ |
| 109 | + || { echo "ERROR: 'claude mcp add' was never called by the setup script"; exit 1; } |
| 110 | +
|
| 111 | + # Assert --transport stdio was passed |
| 112 | + grep -q "stdio" "$HOME/claude-stub.log" \ |
| 113 | + || { echo "ERROR: '--transport stdio' was not passed to 'claude mcp add'"; exit 1; } |
| 114 | +
|
| 115 | + # Assert the server name 'devops-os' was registered |
| 116 | + grep -q "devops-os" "$HOME/claude-stub.log" \ |
| 117 | + || { echo "ERROR: 'devops-os' server name was not passed to 'claude mcp add'"; exit 1; } |
| 118 | +
|
| 119 | + # Assert mcp_server.server module was referenced |
| 120 | + grep -q "mcp_server.server" "$HOME/claude-stub.log" \ |
| 121 | + || { echo "ERROR: 'mcp_server.server' module not referenced in 'claude mcp add' call"; exit 1; } |
| 122 | +
|
| 123 | + echo "✓ claude mcp add was called with all expected arguments" |
| 124 | +
|
| 125 | + # ── Step 6: verify the MCP server starts and responds via JSON-RPC ─────── |
| 126 | + - name: Verify MCP server starts and responds to tools/list |
| 127 | + run: | |
| 128 | + .venv/bin/python - << 'PYEOF' |
| 129 | + import json, os, subprocess, sys |
| 130 | +
|
| 131 | + repo_root = os.getcwd() |
| 132 | + env = {**os.environ, "PYTHONPATH": repo_root} |
| 133 | + venv_python = os.path.join(repo_root, ".venv", "bin", "python") |
| 134 | +
|
| 135 | + print("Starting MCP server via venv python:", venv_python) |
| 136 | + proc = subprocess.Popen( |
| 137 | + [venv_python, "-m", "mcp_server.server"], |
| 138 | + stdin=subprocess.PIPE, |
| 139 | + stdout=subprocess.PIPE, |
| 140 | + stderr=subprocess.PIPE, |
| 141 | + text=True, |
| 142 | + env=env, |
| 143 | + cwd=repo_root, |
| 144 | + ) |
| 145 | +
|
| 146 | + def send(method, params=None, req_id=None): |
| 147 | + msg = {"jsonrpc": "2.0", "method": method} |
| 148 | + if req_id is not None: |
| 149 | + msg["id"] = req_id |
| 150 | + if params is not None: |
| 151 | + msg["params"] = params |
| 152 | + proc.stdin.write(json.dumps(msg) + "\n") |
| 153 | + proc.stdin.flush() |
| 154 | + if req_id is not None: |
| 155 | + return json.loads(proc.stdout.readline()) |
| 156 | +
|
| 157 | + # MCP initialize handshake |
| 158 | + init_resp = send("initialize", { |
| 159 | + "protocolVersion": "2024-11-05", |
| 160 | + "capabilities": {}, |
| 161 | + "clientInfo": {"name": "smoke-test", "version": "1.0"}, |
| 162 | + }, req_id=1) |
| 163 | +
|
| 164 | + if "error" in init_resp: |
| 165 | + proc.terminate() |
| 166 | + print("ERROR: initialize failed:", init_resp["error"]) |
| 167 | + sys.exit(1) |
| 168 | +
|
| 169 | + proto_ver = init_resp["result"]["protocolVersion"] |
| 170 | + server_name = init_resp["result"]["serverInfo"]["name"] |
| 171 | + print(f"✓ Initialize handshake OK — server: {server_name!r}, protocol: {proto_ver}") |
| 172 | +
|
| 173 | + # Send notifications/initialized (required before tool calls) |
| 174 | + send("notifications/initialized") |
| 175 | +
|
| 176 | + # tools/list |
| 177 | + list_resp = send("tools/list", {}, req_id=2) |
| 178 | + if "error" in list_resp: |
| 179 | + proc.terminate() |
| 180 | + print("ERROR: tools/list failed:", list_resp["error"]) |
| 181 | + sys.exit(1) |
| 182 | +
|
| 183 | + tools = list_resp["result"]["tools"] |
| 184 | + names = {t["name"] for t in tools} |
| 185 | + expected = { |
| 186 | + "generate_github_actions_workflow", |
| 187 | + "generate_gitlab_ci_pipeline", |
| 188 | + "generate_jenkins_pipeline", |
| 189 | + "generate_k8s_config", |
| 190 | + "generate_argocd_config", |
| 191 | + "generate_sre_configs", |
| 192 | + "scaffold_devcontainer", |
| 193 | + "generate_unittest_config", |
| 194 | + } |
| 195 | + missing = expected - names |
| 196 | + if missing: |
| 197 | + proc.terminate() |
| 198 | + print("ERROR: missing tools:", missing) |
| 199 | + sys.exit(1) |
| 200 | +
|
| 201 | + print(f"✓ tools/list returned {len(tools)} tools, all 8 DevOps-OS tools present") |
| 202 | + for t in sorted(tools, key=lambda x: x["name"]): |
| 203 | + print(f" • {t['name']}") |
| 204 | +
|
| 205 | + # Quick tools/call sanity check: generate a GHA workflow |
| 206 | + call_resp = send("tools/call", { |
| 207 | + "name": "generate_github_actions_workflow", |
| 208 | + "arguments": {"name": "smoke-test-app", "languages": "python"}, |
| 209 | + }, req_id=3) |
| 210 | +
|
| 211 | + if "error" in call_resp: |
| 212 | + proc.terminate() |
| 213 | + print("ERROR: tools/call failed:", call_resp["error"]) |
| 214 | + sys.exit(1) |
| 215 | +
|
| 216 | + content_text = call_resp["result"]["content"][0]["text"] |
| 217 | + assert "smoke-test-app" in content_text, "app name not in GHA output" |
| 218 | + assert "runs-on:" in content_text, "not a valid GHA YAML" |
| 219 | + print("✓ tools/call generate_github_actions_workflow returned valid GHA YAML") |
| 220 | +
|
| 221 | + proc.terminate() |
| 222 | + proc.wait(timeout=5) |
| 223 | + print("\nAll smoke checks passed ✓") |
| 224 | + PYEOF |
0 commit comments