Skip to content

Test Existing Examples #50

Test Existing Examples

Test Existing Examples #50

Workflow file for this run

name: Test Existing Examples
# Runs all examples on a schedule to catch regressions from SDK updates,
# API changes, or broken dependencies — independent of any PR activity.
on:
schedule:
- cron: '0 */6 * * *' # Every 6 hours
workflow_dispatch:
inputs:
reason:
description: 'Why are you triggering this manually?'
required: false
default: 'manual test'
concurrency:
group: test-existing
cancel-in-progress: false
permissions:
contents: write
pull-requests: write
issues: write
jobs:
# ── Node.js ────────────────────────────────────────────────────────────────
node:
runs-on: ubuntu-latest
outputs:
has_failures: ${{ steps.test.outputs.has_failures }}
failed_examples: ${{ steps.test.outputs.failed_examples }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: '20'
- name: Test Node.js examples
id: test
continue-on-error: true
env:
DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }}
TWILIO_ACCOUNT_SID: ${{ secrets.TWILIO_ACCOUNT_SID }}
TWILIO_AUTH_TOKEN: ${{ secrets.TWILIO_AUTH_TOKEN }}
TWILIO_PHONE_NUMBER: ${{ secrets.TWILIO_PHONE_NUMBER }}
LIVEKIT_URL: ${{ secrets.LIVEKIT_URL }}
LIVEKIT_API_KEY: ${{ secrets.LIVEKIT_API_KEY }}
LIVEKIT_API_SECRET: ${{ secrets.LIVEKIT_API_SECRET }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }}
DISCORD_CLIENT_ID: ${{ secrets.DISCORD_CLIENT_ID }}
VONAGE_APPLICATION_ID: ${{ secrets.VONAGE_APPLICATION_ID }}
VONAGE_PRIVATE_KEY: ${{ secrets.VONAGE_PRIVATE_KEY }}
DAILY_API_KEY: ${{ secrets.DAILY_API_KEY }}
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
SLACK_APP_TOKEN: ${{ secrets.SLACK_APP_TOKEN }}
run: |
FAILED=""
for dir in examples/*/; do
[ ! -f "${dir}package.json" ] && continue
echo "Testing: $dir"
pushd "$dir" > /dev/null
if [ -f ".env.example" ]; then
MISSING=""
while IFS= read -r line; do
[[ -z "${line// }" || "$line" == \#* ]] && continue
VAR_NAME="${line%%=*}"; VAR_NAME="${VAR_NAME// /}"
[ -z "$VAR_NAME" ] && continue
[ -z "${!VAR_NAME+x}" ] || [ -z "${!VAR_NAME}" ] && MISSING="$MISSING $VAR_NAME"
done < ".env.example"
if [ -n "$MISSING" ]; then
echo "⚠ Skipping $dir — missing secrets: $MISSING"
popd > /dev/null; continue
fi
fi
if [ -f "package-lock.json" ]; then npm ci --prefer-offline 2>&1; else npm install 2>&1; fi
npm test 2>&1 || FAILED="$FAILED $dir"
popd > /dev/null
done
[ -n "$FAILED" ] && echo "has_failures=true" >> $GITHUB_OUTPUT \
|| echo "has_failures=false" >> $GITHUB_OUTPUT
echo "failed_examples=${FAILED}" >> $GITHUB_OUTPUT
# ── Python ─────────────────────────────────────────────────────────────────
python:
runs-on: ubuntu-latest
outputs:
has_failures: ${{ steps.test.outputs.has_failures }}
failed_examples: ${{ steps.test.outputs.failed_examples }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.11'
- name: Test Python examples
id: test
continue-on-error: true
env:
DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }}
TWILIO_ACCOUNT_SID: ${{ secrets.TWILIO_ACCOUNT_SID }}
TWILIO_AUTH_TOKEN: ${{ secrets.TWILIO_AUTH_TOKEN }}
TWILIO_PHONE_NUMBER: ${{ secrets.TWILIO_PHONE_NUMBER }}
LIVEKIT_URL: ${{ secrets.LIVEKIT_URL }}
LIVEKIT_API_KEY: ${{ secrets.LIVEKIT_API_KEY }}
LIVEKIT_API_SECRET: ${{ secrets.LIVEKIT_API_SECRET }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }}
DISCORD_CLIENT_ID: ${{ secrets.DISCORD_CLIENT_ID }}
VONAGE_APPLICATION_ID: ${{ secrets.VONAGE_APPLICATION_ID }}
VONAGE_PRIVATE_KEY: ${{ secrets.VONAGE_PRIVATE_KEY }}
DAILY_API_KEY: ${{ secrets.DAILY_API_KEY }}
TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }}
run: |
FAILED=""
for dir in examples/*/; do
HAS_PY=false
[ -f "${dir}requirements.txt" ] && HAS_PY=true
[ -f "${dir}pyproject.toml" ] && HAS_PY=true
[ "$HAS_PY" = "false" ] && continue
echo "Testing: $dir"
pushd "$dir" > /dev/null
if [ -f ".env.example" ]; then
MISSING=""
while IFS= read -r line; do
[[ -z "${line// }" || "$line" == \#* ]] && continue
VAR_NAME="${line%%=*}"; VAR_NAME="${VAR_NAME// /}"
[ -z "$VAR_NAME" ] && continue
[ -z "${!VAR_NAME+x}" ] || [ -z "${!VAR_NAME}" ] && MISSING="$MISSING $VAR_NAME"
done < ".env.example"
if [ -n "$MISSING" ]; then
echo "⚠ Skipping $dir — missing secrets: $MISSING"
popd > /dev/null; continue
fi
fi
python -m pip install -q --upgrade pip
[ -f "requirements.txt" ] && pip install -q -r requirements.txt
[ -f "pyproject.toml" ] && pip install -q -e .
TEST_RAN=false
if find tests/ -name "test_*.py" 2>/dev/null | grep -q .; then
pip install -q pytest
python -m pytest tests/ -v 2>&1 && TEST_RAN=true || { FAILED="$FAILED $dir"; TEST_RAN=true; }
fi
if [ "$TEST_RAN" = "false" ] && ls tests/*.py 2>/dev/null | head -1 | grep -q .; then
python "$(ls tests/*.py | head -1)" 2>&1 || FAILED="$FAILED $dir"
fi
popd > /dev/null
done
[ -n "$FAILED" ] && echo "has_failures=true" >> $GITHUB_OUTPUT \
|| echo "has_failures=false" >> $GITHUB_OUTPUT
echo "failed_examples=${FAILED}" >> $GITHUB_OUTPUT
# ── Go ─────────────────────────────────────────────────────────────────────
go:
runs-on: ubuntu-latest
outputs:
has_failures: ${{ steps.test.outputs.has_failures }}
failed_examples: ${{ steps.test.outputs.failed_examples }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: '1.22'
- name: Test Go examples
id: test
continue-on-error: true
env:
DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }}
TWILIO_ACCOUNT_SID: ${{ secrets.TWILIO_ACCOUNT_SID }}
TWILIO_AUTH_TOKEN: ${{ secrets.TWILIO_AUTH_TOKEN }}
TWILIO_PHONE_NUMBER: ${{ secrets.TWILIO_PHONE_NUMBER }}
LIVEKIT_URL: ${{ secrets.LIVEKIT_URL }}
LIVEKIT_API_KEY: ${{ secrets.LIVEKIT_API_KEY }}
LIVEKIT_API_SECRET: ${{ secrets.LIVEKIT_API_SECRET }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }}
DISCORD_CLIENT_ID: ${{ secrets.DISCORD_CLIENT_ID }}
VONAGE_APPLICATION_ID: ${{ secrets.VONAGE_APPLICATION_ID }}
VONAGE_PRIVATE_KEY: ${{ secrets.VONAGE_PRIVATE_KEY }}
DAILY_API_KEY: ${{ secrets.DAILY_API_KEY }}
run: |
FAILED=""
for dir in examples/*/; do
[ ! -f "${dir}go.mod" ] && continue
echo "Testing: $dir"
pushd "$dir" > /dev/null
if [ -f ".env.example" ]; then
MISSING=""
while IFS= read -r line; do
[[ -z "${line// }" || "$line" == \#* ]] && continue
VAR_NAME="${line%%=*}"; VAR_NAME="${VAR_NAME// /}"
[ -z "$VAR_NAME" ] && continue
[ -z "${!VAR_NAME+x}" ] || [ -z "${!VAR_NAME}" ] && MISSING="$MISSING $VAR_NAME"
done < ".env.example"
if [ -n "$MISSING" ]; then
echo "⚠ Skipping $dir — missing secrets: $MISSING"
popd > /dev/null; continue
fi
fi
go mod download
go test ./... -v -timeout 60s 2>&1 || FAILED="$FAILED $dir"
popd > /dev/null
done
[ -n "$FAILED" ] && echo "has_failures=true" >> $GITHUB_OUTPUT \
|| echo "has_failures=false" >> $GITHUB_OUTPUT
echo "failed_examples=${FAILED}" >> $GITHUB_OUTPUT
# ── Java ───────────────────────────────────────────────────────────────────
java:
runs-on: ubuntu-latest
outputs:
has_failures: ${{ steps.test.outputs.has_failures }}
failed_examples: ${{ steps.test.outputs.failed_examples }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '21'
- name: Test Java examples
id: test
continue-on-error: true
env:
DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }}
TWILIO_ACCOUNT_SID: ${{ secrets.TWILIO_ACCOUNT_SID }}
TWILIO_AUTH_TOKEN: ${{ secrets.TWILIO_AUTH_TOKEN }}
TWILIO_PHONE_NUMBER: ${{ secrets.TWILIO_PHONE_NUMBER }}
LIVEKIT_URL: ${{ secrets.LIVEKIT_URL }}
LIVEKIT_API_KEY: ${{ secrets.LIVEKIT_API_KEY }}
LIVEKIT_API_SECRET: ${{ secrets.LIVEKIT_API_SECRET }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }}
DISCORD_CLIENT_ID: ${{ secrets.DISCORD_CLIENT_ID }}
VONAGE_APPLICATION_ID: ${{ secrets.VONAGE_APPLICATION_ID }}
VONAGE_PRIVATE_KEY: ${{ secrets.VONAGE_PRIVATE_KEY }}
DAILY_API_KEY: ${{ secrets.DAILY_API_KEY }}
run: |
FAILED=""
for dir in examples/*/; do
HAS_JAVA=false
[ -f "${dir}pom.xml" ] && HAS_JAVA=true
[ -f "${dir}build.gradle" ] && HAS_JAVA=true
[ "$HAS_JAVA" = "false" ] && continue
echo "Testing: $dir"
pushd "$dir" > /dev/null
if [ -f ".env.example" ]; then
MISSING=""
while IFS= read -r line; do
[[ -z "${line// }" || "$line" == \#* ]] && continue
VAR_NAME="${line%%=*}"; VAR_NAME="${VAR_NAME// /}"
[ -z "$VAR_NAME" ] && continue
[ -z "${!VAR_NAME+x}" ] || [ -z "${!VAR_NAME}" ] && MISSING="$MISSING $VAR_NAME"
done < ".env.example"
if [ -n "$MISSING" ]; then
echo "⚠ Skipping $dir — missing secrets: $MISSING"
popd > /dev/null; continue
fi
fi
if [ -f "pom.xml" ]; then
mvn test -q 2>&1 || FAILED="$FAILED $dir"
else
./gradlew test 2>&1 || FAILED="$FAILED $dir"
fi
popd > /dev/null
done
[ -n "$FAILED" ] && echo "has_failures=true" >> $GITHUB_OUTPUT \
|| echo "has_failures=false" >> $GITHUB_OUTPUT
echo "failed_examples=${FAILED}" >> $GITHUB_OUTPUT
# ── SDK version audit ───────────────────────────────────────────────────────
# Checks every merged example for outdated Deepgram SDK pins.
# Uses the public GitHub releases API — no auth required.
# Major-version gaps are flagged (e.g. pinned v4 when latest is v6).
sdk-audit:
runs-on: ubuntu-latest
outputs:
outdated_examples: ${{ steps.scan.outputs.outdated }}
has_outdated: ${{ steps.scan.outputs.has_outdated }}
steps:
- uses: actions/checkout@v4
- name: Fetch latest SDK versions
id: versions
run: |
latest_major() {
curl -sf "https://api.github.com/repos/deepgram/$1/releases/latest" \
| jq -r '.tag_name // "0"' | tr -d 'v' | cut -d. -f1
}
echo "python=$(latest_major deepgram-python-sdk)" >> $GITHUB_OUTPUT
echo "js=$(latest_major deepgram-js-sdk)" >> $GITHUB_OUTPUT
echo "go=$(latest_major deepgram-go-sdk)" >> $GITHUB_OUTPUT
echo "java=$(latest_major deepgram-java-sdk)" >> $GITHUB_OUTPUT
echo "rust=$(latest_major deepgram-rust-sdk)" >> $GITHUB_OUTPUT
echo "dotnet=$(latest_major deepgram-dotnet-sdk)" >> $GITHUB_OUTPUT
cat $GITHUB_OUTPUT
- name: Scan examples for outdated SDK pins
id: scan
run: |
PY_LATEST="${{ steps.versions.outputs.python }}"
JS_LATEST="${{ steps.versions.outputs.js }}"
GO_LATEST="${{ steps.versions.outputs.go }}"
JAVA_LATEST="${{ steps.versions.outputs.java }}"
RUST_LATEST="${{ steps.versions.outputs.rust }}"
DOTNET_LATEST="${{ steps.versions.outputs.dotnet }}"
OUTDATED=""
for dir in examples/*/; do
[ ! -d "$dir" ] && continue
# Python — deepgram-sdk==X or >=X
if [ -f "${dir}requirements.txt" ]; then
PINNED=$(grep -oE 'deepgram-sdk[^,\n]*' "${dir}requirements.txt" \
| grep -oE '[0-9]+' | head -1 || true)
if [ -n "$PINNED" ] && [ "$PINNED" -lt "$PY_LATEST" ] 2>/dev/null; then
echo "OUTDATED Python $dir: pinned major=$PINNED latest=$PY_LATEST"
OUTDATED="$OUTDATED $dir"
fi
fi
# JavaScript/TypeScript — @deepgram/sdk: "^X.y.z" or "X.y.z"
if [ -f "${dir}package.json" ]; then
PINNED=$(jq -r '
((.dependencies // {}) + (.devDependencies // {}))["@deepgram/sdk"]
// empty' "${dir}package.json" \
| grep -oE '[0-9]+' | head -1 || true)
if [ -n "$PINNED" ] && [ "$PINNED" -lt "$JS_LATEST" ] 2>/dev/null; then
echo "OUTDATED JS $dir: pinned major=$PINNED latest=$JS_LATEST"
OUTDATED="$OUTDATED $dir"
fi
fi
# Go — github.com/deepgram/deepgram-go-sdk/vX
if [ -f "${dir}go.mod" ]; then
PINNED=$(grep -oE 'deepgram-go-sdk/v[0-9]+' "${dir}go.mod" \
| grep -oE '[0-9]+' | head -1 || true)
if [ -n "$PINNED" ] && [ "$PINNED" -lt "$GO_LATEST" ] 2>/dev/null; then
echo "OUTDATED Go $dir: pinned major=v$PINNED latest=v$GO_LATEST"
OUTDATED="$OUTDATED $dir"
fi
fi
done
OUTDATED=$(echo "$OUTDATED" | xargs | tr ' ' '\n' | sort -u | xargs)
if [ -n "$OUTDATED" ]; then
echo "has_outdated=true" >> $GITHUB_OUTPUT
echo "outdated=$OUTDATED" >> $GITHUB_OUTPUT
echo "Examples with outdated SDKs: $OUTDATED"
else
echo "has_outdated=false" >> $GITHUB_OUTPUT
echo "outdated=" >> $GITHUB_OUTPUT
echo "All SDK pins are current"
fi
# ── Report failures + fix (one at a time) ──────────────────────────────────
# Pick the first failing or outdated example that has no open fix PR.
# Test failures take priority over SDK version upgrades.
report:
needs: [node, python, go, java, sdk-audit]
if: always()
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Configure git
run: |
git config user.name "examples-bot"
git config user.email "noreply@deepgram.com"
- name: Get date
id: date
run: echo "date=$(date -u +%Y-%m-%d)" >> $GITHUB_OUTPUT
- name: Fetch latest Deepgram SDK versions
id: sdk
run: |
latest() { curl -sf "https://api.github.com/repos/deepgram/$1/releases/latest" | jq -r '.tag_name // "unknown"'; }
echo "python=$(latest deepgram-python-sdk)" >> $GITHUB_OUTPUT
echo "js=$(latest deepgram-js-sdk)" >> $GITHUB_OUTPUT
echo "go=$(latest deepgram-go-sdk)" >> $GITHUB_OUTPUT
- name: Pick first unaddressed failure
id: collect
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# Test failures first (higher priority), then outdated SDK pins
ALL_FAILED=""
[ "${{ needs.node.outputs.has_failures }}" = "true" ] && \
ALL_FAILED="$ALL_FAILED ${{ needs.node.outputs.failed_examples }}"
[ "${{ needs.python.outputs.has_failures }}" = "true" ] && \
ALL_FAILED="$ALL_FAILED ${{ needs.python.outputs.failed_examples }}"
[ "${{ needs.go.outputs.has_failures }}" = "true" ] && \
ALL_FAILED="$ALL_FAILED ${{ needs.go.outputs.failed_examples }}"
[ "${{ needs.java.outputs.has_failures }}" = "true" ] && \
ALL_FAILED="$ALL_FAILED ${{ needs.java.outputs.failed_examples }}"
# Append outdated-SDK examples after test failures (lower priority)
[ "${{ needs.sdk-audit.outputs.has_outdated }}" = "true" ] && \
ALL_FAILED="$ALL_FAILED ${{ needs.sdk-audit.outputs.outdated_examples }}"
ALL_FAILED=$(echo "$ALL_FAILED" | xargs | tr ' ' '\n' | sort -u | xargs)
TARGET=""
TARGET_SLUG=""
for EXAMPLE in $ALL_FAILED; do
SLUG=$(basename "${EXAMPLE%/}")
# Skip if a fix PR is already open for this example
OPEN=$(gh pr list --repo ${{ github.repository }} --state open \
--search "$SLUG" --json number --jq 'length' 2>/dev/null || echo "0")
if [ "$OPEN" = "0" ]; then
TARGET="$EXAMPLE"
TARGET_SLUG="$SLUG"
echo "Selected: $EXAMPLE (no open fix PR)"
break
else
echo "Skipping $EXAMPLE — fix PR already open"
fi
done
if [ -n "$TARGET" ]; then
echo "has_target=true" >> $GITHUB_OUTPUT
echo "target=$TARGET" >> $GITHUB_OUTPUT
echo "slug=$TARGET_SLUG" >> $GITHUB_OUTPUT
else
echo "has_target=false" >> $GITHUB_OUTPUT
[ -n "$ALL_FAILED" ] && echo "All failures already have open fix PRs — nothing to do"
fi
- name: Open issue for this failure
if: steps.collect.outputs.has_target == 'true'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
SLUG="${{ steps.collect.outputs.slug }}"
RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
EXISTING=$(gh issue list --repo ${{ github.repository }} \
--label "queue:fix-example" --state open \
--search "[Regression] $SLUG" \
--json number --jq '.[0].number')
if [ -n "$EXISTING" ]; then
echo "Issue already open for $SLUG (#$EXISTING)"
else
gh issue create \
--repo ${{ github.repository }} \
--title "[Regression] $SLUG — tests failing" \
--label "queue:fix-example" \
--label "type:fix" \
--body "Regression in \`${{ steps.collect.outputs.target }}\`. Run: $RUN_URL"
echo "Created issue for $SLUG"
fi
- name: Run fix agent for this failure
if: steps.collect.outputs.has_target == 'true'
uses: anthropics/claude-code-action@beta
env:
KAPA_API_KEY: ${{ secrets.KAPA_API_KEY }}
KAPA_PROJECT_ID: 1908afc6-c134-4c6f-a684-ed7d8ce91759
with:
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
github_token: ${{ secrets.GITHUB_TOKEN }}
mode: agent
model: claude-opus-4-6
allowed_tools: "Bash,Read,Write,Edit,Glob,Grep,WebSearch,WebFetch"
direct_prompt: |
A merged example needs attention — either a test regression or an outdated SDK pin.
Example: ${{ steps.collect.outputs.target }}
1. Read the example's source, tests, and dependency files
2. Check if the issue is a test failure, an outdated SDK version, or both
3. Search Kapa for current correct SDK usage (instructions/kapa-search.md)
4. Fix the code and upgrade SDK pins to the required versions:
- Python: deepgram-sdk==${{ steps.sdk.outputs.python }}
- JavaScript: @deepgram/sdk@${{ steps.sdk.outputs.js }}
- Go: ${{ steps.sdk.outputs.go }}
5. Open or update a PR — one PR per example, additive:
- Check for an existing open PR whose branch starts with fix/${{ steps.collect.outputs.slug }}:
EXISTING=$(gh pr list --state open --search "fix ${{ steps.collect.outputs.slug }}" --json number,headRefName --jq '.[0]')
- If one exists: check out its branch, push the fix there (no new PR)
- If none exists: create branch fix/${{ steps.collect.outputs.slug }}-regression-${{ steps.date.outputs.date }},
commit, push, and open a new PR with label type:fix
- Title: [Fix] {NNN}-${{ steps.collect.outputs.slug }} — {brief description}
- Do NOT enable auto-merge — PRs wait for human review and merge
Today's date: ${{ steps.date.outputs.date }}
Repository: ${{ github.repository }}