Skip to content
Plugin Submission Orchestrator

add sliding kv_cache #424

Sign in to view logs

add sliding kv_cache

add sliding kv_cache #424

Workflow file for this run

.github/workflows/plugin_submission_orchestrator.yml at d7ffabf
name: Plugin Submission Orchestrator
# Combined workflow: Handles both prepare (metadata generation/updates) and validate (downstream validation, automerge, scoring)
# This workflow combines the prepare and validate workflows into a single unified workflow
on:
pull_request:
types: [opened, synchronize, labeled, reopened]
pull_request_target:
types: [closed]
branches: [main]
permissions:
contents: write
pull-requests: write
issues: write
checks: read
statuses: read
env:
DOMAIN: language
DOMAIN_ROOT: brainscore_language
PYTHON_VERSION: '3.11'
jobs:
# ============================================================================
# JOB 1: Detect and Classify Changes
# ============================================================================
detect_changes:
name: "1. Detect Changes"
if: |
(github.event_name == 'pull_request') ||
(github.event_name == 'pull_request_target' && github.event.pull_request.merged == true)
runs-on: ubuntu-latest
outputs:
has_plugins: ${{ steps.detect.outputs.has_plugins }}
plugin_type: ${{ steps.detect.outputs.plugin_type }}
plugin_dirs: ${{ steps.detect.outputs.plugin_dirs }}
has_new_models: ${{ steps.detect.outputs.has_new_models }}
metadata_only: ${{ steps.detect.outputs.metadata_only }}
needs_scoring: ${{ steps.detect.outputs.needs_scoring }}
needs_mapping: ${{ steps.detect.outputs.needs_mapping }}
needs_metadata_generation: ${{ steps.detect.outputs.needs_metadata_generation }}
is_automergeable: ${{ steps.detect.outputs.is_automergeable }}
plugin_info_json: ${{ steps.detect.outputs.plugin_info_json }}
is_web_submission: ${{ steps.detect.outputs.is_web_submission }}
is_fork_pr: ${{ steps.detect.outputs.is_fork_pr }}
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }}
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip'
- name: Install dependencies
run: |
python -m pip install --upgrade pip setuptools
python -m pip install ".[test]"
- name: Get changed files
id: changed_files
run: |
if [ "${{ github.event_name }}" == "pull_request_target" ] && [ "${{ github.event.pull_request.merged }}" == "true" ]; then
# Post-merge: compare merge commit to previous
git fetch origin refs/pull/${{ github.event.number }}/head
MERGE_COMMIT=$(git log --format='%H %P' --all | grep "$(git rev-parse FETCH_HEAD)$" | cut -f1 -d' ')
CHANGED_FILES=$(git diff --name-only origin/main~1 $MERGE_COMMIT | tr '\n' ' ' || echo "")
else
# Pre-merge: use GitHub API to get changed files (more reliable than git diff)
PR_NUMBER="${{ github.event.pull_request.number }}"
CHANGED_FILES=$(gh pr view ${PR_NUMBER} --json files --jq '.files[].path' | tr '\n' ' ' || echo "")
# Fallback to git diff if API fails or returns empty
if [ -z "$CHANGED_FILES" ] || [ "$CHANGED_FILES" = " " ]; then
echo "GitHub API returned no files, trying git diff fallback..."
BASE_REF="${{ github.event.pull_request.base.ref }}"
BASE_SHA="${{ github.event.pull_request.base.sha }}"
HEAD_SHA="${{ github.event.pull_request.head.sha }}"
git fetch origin ${BASE_REF} || true
git fetch origin ${HEAD_SHA} || true
if [ -n "$BASE_SHA" ] && [ -n "$HEAD_SHA" ]; then
CHANGED_FILES=$(git diff --name-only ${BASE_SHA}...${HEAD_SHA} | tr '\n' ' ' || echo "")
fi
if [ -z "$CHANGED_FILES" ] || [ "$CHANGED_FILES" = " " ]; then
CHANGED_FILES=$(git diff --name-only origin/${BASE_REF}...HEAD | tr '\n' ' ' || echo "")
fi
fi
fi
echo "CHANGED_FILES=${CHANGED_FILES}" >> $GITHUB_ENV
echo "Changed files: ${CHANGED_FILES}"
env:
GH_TOKEN: ${{ secrets.GH_MFERG_PAT }}
- name: Detect plugin changes
id: detect
run: |
CHANGED_FILES="${{ env.CHANGED_FILES }}"
if [ -z "$CHANGED_FILES" ]; then
echo "No changed files detected, using empty string"
CHANGED_FILES=""
fi
CHANGED_FILES_B64=$(echo -n "${CHANGED_FILES}" | base64 | tr -d '\n')
PLUGIN_INFO=$(python -W ignore -c "
from brainscore_core.plugin_management.parse_plugin_changes import get_scoring_info
import json
import sys
import base64
try:
changed_files_b64 = '${CHANGED_FILES_B64}'
changed_files = base64.b64decode(changed_files_b64).decode('utf-8') if changed_files_b64 else ''
get_scoring_info(changed_files, '${{ env.DOMAIN_ROOT }}')
except Exception as e:
error_msg = f'Error in get_scoring_info: {type(e).__name__}: {str(e)}'
print(error_msg, file=sys.stderr)
print('{}')
" 2>&1 | grep -v "^Error in get_scoring_info" | grep -v "^Warning:" | jq -c . 2>/dev/null | tail -n 1 || echo '{}')
if ! echo "$PLUGIN_INFO" | jq empty 2>/dev/null; then
echo "Warning: Invalid JSON from get_scoring_info, using empty object"
PLUGIN_INFO='{}'
fi
echo "PLUGIN_INFO=${PLUGIN_INFO}" >> $GITHUB_ENV
HAS_PLUGINS_RAW=$(echo "$PLUGIN_INFO" | jq -r 'if .modifies_plugins == true then "true" else "false" end' | head -n 1)
HAS_PLUGINS=$(echo "$HAS_PLUGINS_RAW" | tr '[:upper:]' '[:lower:]')
# Infer plugin_type from changed_plugins structure
HAS_MODELS=$(echo "$PLUGIN_INFO" | jq -r '.changed_plugins.models // [] | length' | head -n 1)
HAS_BENCHMARKS=$(echo "$PLUGIN_INFO" | jq -r '.changed_plugins.benchmarks // [] | length' | head -n 1)
PLUGIN_TYPE=""
if [ "$HAS_MODELS" -gt 0 ] 2>/dev/null && [ "$HAS_BENCHMARKS" -gt 0 ] 2>/dev/null; then
PLUGIN_TYPE="models,benchmarks"
elif [ "$HAS_MODELS" -gt 0 ] 2>/dev/null; then
PLUGIN_TYPE="models"
elif [ "$HAS_BENCHMARKS" -gt 0 ] 2>/dev/null; then
PLUGIN_TYPE="benchmarks"
fi
NEEDS_SCORING_RAW=$(echo "$PLUGIN_INFO" | jq -r '.run_score // "False"' | head -n 1)
NEEDS_SCORING=$(echo "$NEEDS_SCORING_RAW" | tr '[:upper:]' '[:lower:]')
# If plugin_type is benchmarks only, disable scoring and metadata generation
if [ "$PLUGIN_TYPE" = "benchmarks" ]; then
NEEDS_SCORING="false"
echo "Benchmarks-only PR detected - setting needs_scoring=false and needs_metadata_generation=false"
fi
IS_AUTOMERGEABLE_RAW=$(echo "$PLUGIN_INFO" | jq -r 'if .is_automergeable == true then "true" else "false" end' | head -n 1)
IS_AUTOMERGEABLE=$(echo "$IS_AUTOMERGEABLE_RAW" | tr '[:upper:]' '[:lower:]')
# Check if new_models string exists and is non-empty (it's a space-separated string, not an array)
# get_scoring_info sets new_models to a space-separated string of plugin IDs found in __init__.py files
NEW_MODELS=$(echo "$PLUGIN_INFO" | jq -r '.new_models // ""' | head -n 1)
# Debug: show what new_models contains
echo "DEBUG: new_models value from JSON: '${NEW_MODELS}'"
# Check if we have models in changed_plugins (this is more reliable than new_models)
MODELS_COUNT=$(echo "$PLUGIN_INFO" | jq -r '.changed_plugins.models // [] | length' | head -n 1)
echo "DEBUG: changed_plugins.models count: ${MODELS_COUNT}"
# Set has_new_models based on whether we have models in changed_plugins
# This is more reliable than relying on new_models which depends on get_plugin_ids finding registry entries
if [ -n "$MODELS_COUNT" ] && [ "$MODELS_COUNT" != "null" ] && [ "$MODELS_COUNT" != "0" ]; then
HAS_NEW_MODELS="true"
echo "DEBUG: Found ${MODELS_COUNT} model(s) in changed_plugins.models - setting has_new_models=true"
elif [ -n "$NEW_MODELS" ] && [ "$NEW_MODELS" != "null" ] && [ "$NEW_MODELS" != "" ]; then
HAS_NEW_MODELS="true"
echo "DEBUG: new_models contains: '${NEW_MODELS}' - setting has_new_models=true"
else
HAS_NEW_MODELS="false"
echo "DEBUG: No models found - setting has_new_models=false"
fi
MODELS=$(echo "$PLUGIN_INFO" | jq -r '.changed_plugins.models[]? // empty' | head -n 1)
BENCHMARKS=$(echo "$PLUGIN_INFO" | jq -r '.changed_plugins.benchmarks[]? // empty' | head -n 1)
PLUGIN_DIRS=""
if [ -n "$MODELS" ]; then
for model in $(echo "$PLUGIN_INFO" | jq -r '.changed_plugins.models[]? // empty'); do
if [ -n "$model" ]; then
if [ -z "$PLUGIN_DIRS" ]; then
PLUGIN_DIRS="${DOMAIN_ROOT}/models/${model}"
else
PLUGIN_DIRS="${PLUGIN_DIRS},${DOMAIN_ROOT}/models/${model}"
fi
fi
done
fi
if [ -n "$BENCHMARKS" ]; then
for benchmark in $(echo "$PLUGIN_INFO" | jq -r '.changed_plugins.benchmarks[]? // empty'); do
if [ -n "$benchmark" ]; then
if [ -z "$PLUGIN_DIRS" ]; then
PLUGIN_DIRS="${DOMAIN_ROOT}/benchmarks/${benchmark}"
else
PLUGIN_DIRS="${PLUGIN_DIRS},${DOMAIN_ROOT}/benchmarks/${benchmark}"
fi
fi
done
fi
NON_METADATA=$(echo "$CHANGED_FILES" | tr ' ' '\n' | grep -Ev "metadata\.ya?ml" || true)
METADATA_ONLY="false"
if [ -z "$NON_METADATA" ] && [ "$HAS_PLUGINS" = "true" ]; then
METADATA_ONLY="true"
fi
# If metadata_only is true, scoring should not be needed
if [ "$METADATA_ONLY" = "true" ]; then
NEEDS_SCORING="false"
echo "Metadata-only PR detected - setting needs_scoring=false"
fi
NEEDS_MAPPING="false"
if [ "$DOMAIN" != "language" ] && [ "$HAS_NEW_MODELS" = "true" ] && [ "$NEEDS_SCORING" = "true" ]; then
NEEDS_MAPPING="true"
fi
NEEDS_METADATA_GENERATION="false"
MISSING_METADATA_DIRS=()
# Skip metadata generation if plugin_type is benchmarks only
if [ "$PLUGIN_TYPE" != "benchmarks" ] && [ "$HAS_PLUGINS" = "true" ] && [ "$METADATA_ONLY" = "false" ]; then
IFS=',' read -ra DIRS <<< "$PLUGIN_DIRS"
for dir in "${DIRS[@]}"; do
if [ -n "$dir" ]; then
if [ ! -f "${dir}/metadata.yml" ] && [ ! -f "${dir}/metadata.yaml" ]; then
NEEDS_METADATA_GENERATION="true"
MISSING_METADATA_DIRS+=("${dir}")
echo "Plugin directory missing metadata: ${dir}"
fi
fi
done
fi
# Detect fork PRs
IS_FORK_PR="false"
if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
IS_FORK_PR="true"
fi
# Detect web submission from PR title (user:...) pattern
PR_TITLE="${{ github.event.pull_request.title }}"
IS_WEB_SUBMISSION="false"
if echo "$PR_TITLE" | grep -qE '\(user:[^)]+\)'; then
IS_WEB_SUBMISSION="true"
fi
echo "has_plugins=${HAS_PLUGINS:-false}" >> $GITHUB_OUTPUT
echo "plugin_type=${PLUGIN_TYPE:-}" >> $GITHUB_OUTPUT
echo "plugin_dirs=${PLUGIN_DIRS:-}" >> $GITHUB_OUTPUT
echo "has_new_models=${HAS_NEW_MODELS:-false}" >> $GITHUB_OUTPUT
echo "metadata_only=${METADATA_ONLY:-false}" >> $GITHUB_OUTPUT
echo "needs_scoring=${NEEDS_SCORING:-false}" >> $GITHUB_OUTPUT
echo "needs_mapping=${NEEDS_MAPPING:-false}" >> $GITHUB_OUTPUT
echo "needs_metadata_generation=${NEEDS_METADATA_GENERATION:-false}" >> $GITHUB_OUTPUT
echo "is_automergeable=${IS_AUTOMERGEABLE:-false}" >> $GITHUB_OUTPUT
echo "is_web_submission=${IS_WEB_SUBMISSION:-false}" >> $GITHUB_OUTPUT
echo "is_fork_pr=${IS_FORK_PR:-false}" >> $GITHUB_OUTPUT
if [ -n "$PLUGIN_INFO" ]; then
PLUGIN_INFO_B64=$(echo "$PLUGIN_INFO" | base64 | tr -d '\n')
echo "plugin_info_json=${PLUGIN_INFO_B64}" >> $GITHUB_OUTPUT
else
echo "plugin_info_json=" >> $GITHUB_OUTPUT
fi
echo "Detection results:"
echo " Has plugins: ${HAS_PLUGINS}"
echo " Plugin type: ${PLUGIN_TYPE}"
echo " Plugin dirs: ${PLUGIN_DIRS}"
echo " Has new models: ${HAS_NEW_MODELS}"
echo " Metadata only: ${METADATA_ONLY}"
echo " Needs scoring: ${NEEDS_SCORING}"
echo " Needs mapping: ${NEEDS_MAPPING}"
echo " Needs metadata generation: ${NEEDS_METADATA_GENERATION}"
echo " Is web submission: ${IS_WEB_SUBMISSION}"
echo " Is fork PR: ${IS_FORK_PR}"
if [ "${#MISSING_METADATA_DIRS[@]}" -gt 0 ]; then
echo " Plugins missing metadata: ${MISSING_METADATA_DIRS[*]}"
fi
echo " Changed files: ${CHANGED_FILES}"
# ============================================================================
# JOB 2: Validate PR (Minimal validation to proceed with mutation)
# ============================================================================
validate_pr:
name: "2. Validate PR"
needs: detect_changes
if: |
github.event_name == 'pull_request' &&
needs.detect_changes.outputs.has_plugins == 'true' &&
needs.detect_changes.outputs.metadata_only != 'true' &&
needs.detect_changes.outputs.is_web_submission == 'true'
runs-on: ubuntu-latest
outputs:
is_automergeable: ${{ steps.validate.outputs.is_automergeable }}
all_tests_pass: ${{ steps.validate.outputs.all_tests_pass }}
pr_number: ${{ steps.get_pr_info.outputs.pr_number }}
test_results_json: ${{ steps.validate.outputs.test_results_json }}
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.ref }}
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip'
- name: Install dependencies
run: |
python -m pip install --upgrade pip setuptools
python -m pip install ".[test]"
- name: Get PR number and latest head SHA
id: get_pr_info
run: |
echo "pr_number=${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT
# Get the latest commit SHA from the checked-out branch (after any mutations)
PR_HEAD_SHA=$(git rev-parse HEAD)
echo "pr_head_sha=${PR_HEAD_SHA}" >> $GITHUB_OUTPUT
echo "PR head SHA: ${PR_HEAD_SHA}"
- name: Validate PR
id: validate
run: |
RESULT=$(python brainscore_language/submission/actions_helpers.py validate_pr \
--pr-number ${{ steps.get_pr_info.outputs.pr_number }} \
--pr-head ${{ steps.get_pr_info.outputs.pr_head_sha }})
IS_AUTOMERGEABLE=$(echo "$RESULT" | jq -r '.is_automergeable // false' | head -n 1)
ALL_TESTS_PASS=$(echo "$RESULT" | jq -r '.all_tests_pass // false' | head -n 1)
TEST_RESULTS_JSON=$(echo "$RESULT" | jq -c '.test_results // {}' | head -n 1)
echo "is_automergeable=${IS_AUTOMERGEABLE}" >> $GITHUB_OUTPUT
echo "all_tests_pass=${ALL_TESTS_PASS}" >> $GITHUB_OUTPUT
TEST_RESULTS_B64=$(echo "$TEST_RESULTS_JSON" | base64 | tr -d '\n')
echo "test_results_json=${TEST_RESULTS_B64}" >> $GITHUB_OUTPUT
echo "Validation results:"
echo " Is automergeable: ${IS_AUTOMERGEABLE}"
echo " All tests pass: ${ALL_TESTS_PASS}"
- name: Add submission_prepared label if metadata already exists
if: |
needs.detect_changes.outputs.needs_metadata_generation == 'false' &&
steps.validate.outputs.all_tests_pass == 'true'
env:
GH_TOKEN: ${{ secrets.GH_MFERG_PAT }}
run: |
PR_NUM="${{ steps.get_pr_info.outputs.pr_number }}"
LABELS_JSON=$(gh pr view ${PR_NUM} --json labels)
# Check if submission_prepared already exists
if echo "$LABELS_JSON" | jq -e '.labels[] | select(.name == "submission_prepared")' >/dev/null; then
echo "submission_prepared label already exists on PR, skipping"
exit 0
fi
gh pr edit ${PR_NUM} --add-label "submission_prepared"
echo "Added 'submission_prepared' label to PR (metadata already exists and tests passed)"
- name: Check for submission_prepared label
id: check_submission_prepared
env:
GH_TOKEN: ${{ secrets.GH_MFERG_PAT }}
run: |
LABELS_JSON=$(gh pr view ${{ steps.get_pr_info.outputs.pr_number }} --json labels)
if echo "$LABELS_JSON" | jq -e '.labels[] | select(.name == "submission_prepared")' >/dev/null; then
echo "has_submission_prepared=true" >> $GITHUB_OUTPUT
else
echo "has_submission_prepared=false" >> $GITHUB_OUTPUT
fi
# ============================================================================
# JOB 3: Handle Metadata-Only PRs (Add label and terminate)
# ============================================================================
handle_metadata_only:
name: "3. Handle Metadata-Only PR"
needs: [detect_changes, validate_pr]
if: |
always() &&
needs.detect_changes.result == 'success' &&
github.event_name == 'pull_request' &&
needs.detect_changes.outputs.is_fork_pr != 'true' &&
needs.detect_changes.outputs.metadata_only == 'true'
runs-on: ubuntu-latest
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.ref }}
- name: Add labels
env:
GH_TOKEN: ${{ secrets.GH_MFERG_PAT }}
run: |
PR_NUM="${{ github.event.pull_request.number }}"
LABELS_JSON=$(gh pr view ${PR_NUM} --json labels)
# Check if label already exists
if echo "$LABELS_JSON" | jq -e '.labels[] | select(.name == "only_update_metadata")' >/dev/null; then
echo "only_update_metadata label already exists on PR - exiting this run"
echo "A new orchestrator run will be triggered by the label event"
exit 0
fi
# Add the label (this will trigger a new orchestrator run)
gh pr edit ${PR_NUM} --add-label "only_update_metadata" || echo "Failed to add only_update_metadata label (may already exist on PR)"
echo "Added 'only_update_metadata' label to PR (metadata-only PR - skipping submission_prepared/validated labels)"
echo "Exiting this orchestrator run - a new run will be triggered by the label addition"
exit 0
# ============================================================================
# JOB 4: Generate Mutations and Commit (Metadata Generation + Layer Mapping)
# All steps run in the same job so staged files persist across steps
# ============================================================================
generate_and_commit_mutations:
name: "4. Generate Mutations and Commit"
needs: [detect_changes, validate_pr]
if: |
always() &&
needs.detect_changes.result == 'success' &&
github.event_name == 'pull_request' &&
needs.detect_changes.outputs.is_fork_pr != 'true' &&
(
needs.detect_changes.outputs.needs_metadata_generation == 'true' ||
needs.detect_changes.outputs.needs_mapping == 'true'
) &&
(
needs.validate_pr.outputs.all_tests_pass == 'true' ||
needs.validate_pr.result == 'skipped'
)
runs-on: ubuntu-latest
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.ref }}
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip'
- name: Install dependencies
run: |
python -m pip install --upgrade pip setuptools
python -m pip install ".[test]"
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v3
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: "us-east-1"
- name: Configure Git
run: |
git config --local user.email "github-actions[bot]@users.noreply.github.com"
git config --local user.name "github-actions[bot]"
# Step 4: Generate Metadata (stages files)
- name: Generate metadata for plugins
id: generate_metadata
if: needs.detect_changes.outputs.needs_metadata_generation == 'true'
env:
BSC_DATABASESECRET: ${{ secrets.BSC_DATABASESECRET }}
run: |
IFS=',' read -ra PLUGIN_DIRS <<< "${{ needs.detect_changes.outputs.plugin_dirs }}"
GENERATED_MODELS=()
GENERATED_BENCHMARKS=()
for dir in "${PLUGIN_DIRS[@]}"; do
if [ -n "$dir" ]; then
if [[ "$dir" == *"/models/"* ]]; then
PLUGIN_TYPE="models"
elif [[ "$dir" == *"/benchmarks/"* ]]; then
PLUGIN_TYPE="benchmarks"
else
echo "Could not determine plugin type for ${dir}, skipping"
continue
fi
PLUGIN_NAME=$(basename "$dir")
if [ ! -f "${dir}/metadata.yml" ] && [ ! -f "${dir}/metadata.yaml" ]; then
echo "Generating metadata for: ${dir} (type: ${PLUGIN_TYPE})"
if python brainscore_language/submission/hardcoded_metadata.py "${dir}" "${PLUGIN_TYPE}"; then
if [ -f "${dir}/metadata.yml" ]; then
git add "${dir}/metadata.yml"
echo "Staged ${dir}/metadata.yml"
echo ""
echo "Generated metadata for ${PLUGIN_NAME}:"
echo "----------------------------------------"
cat "${dir}/metadata.yml"
echo "----------------------------------------"
echo ""
elif [ -f "${dir}/metadata.yaml" ]; then
git add "${dir}/metadata.yaml"
echo "Staged ${dir}/metadata.yaml"
echo ""
echo "Generated metadata for ${PLUGIN_NAME}:"
echo "----------------------------------------"
cat "${dir}/metadata.yaml"
echo "----------------------------------------"
echo ""
fi
if [ "$PLUGIN_TYPE" == "models" ]; then
GENERATED_MODELS+=("${PLUGIN_NAME}")
else
GENERATED_BENCHMARKS+=("${PLUGIN_NAME}")
fi
else
echo "Metadata generation failed for ${dir}, continuing..."
fi
else
echo "Metadata already exists for: ${dir}, skipping generation"
fi
fi
done
# Store generated plugin names for commit message
if [ ${#GENERATED_MODELS[@]} -gt 0 ]; then
GENERATED_MODELS_STR=$(IFS=','; echo "${GENERATED_MODELS[*]}")
echo "GENERATED_MODELS=${GENERATED_MODELS_STR}" >> $GITHUB_ENV
echo "generated_models=${GENERATED_MODELS_STR}" >> $GITHUB_OUTPUT
fi
if [ ${#GENERATED_BENCHMARKS[@]} -gt 0 ]; then
GENERATED_BENCHMARKS_STR=$(IFS=','; echo "${GENERATED_BENCHMARKS[*]}")
echo "GENERATED_BENCHMARKS=${GENERATED_BENCHMARKS_STR}" >> $GITHUB_ENV
echo "generated_benchmarks=${GENERATED_BENCHMARKS_STR}" >> $GITHUB_OUTPUT
fi
# Step 5: Layer Mapping (stages files)
- name: Trigger layer mapping
id: layer_mapping
if: |
needs.detect_changes.outputs.needs_mapping == 'true' &&
env.DOMAIN != 'language'
env:
JENKINS_USER: ${{ secrets.JENKINS_MAPPING_USER }}
JENKINS_USER_API: ${{ secrets.JENKINS_MAPPING_USER_API }}
JENKINS_TOKEN: ${{ secrets.JENKINS_MAPPING_TOKEN }}
JENKINS_TRIGGER: ${{ secrets.JENKINS_MAPPING_URL }}
run: |
PLUGIN_INFO_B64='${{ needs.detect_changes.outputs.plugin_info_json }}'
PLUGIN_INFO=$(echo "$PLUGIN_INFO_B64" | base64 -d)
NEW_MODELS=$(echo "$PLUGIN_INFO" | jq -r '.new_models // []')
python brainscore_language/submission/actions_helpers.py trigger_layer_mapping \
--new-models "$NEW_MODELS" \
--pr-number ${{ github.event.pull_request.number }} \
--source-repo ${{ github.event.pull_request.head.repo.clone_url }} \
--source-branch ${{ github.event.pull_request.head.ref }}
# Stage any layer mapping files that were generated
if [ -n "$(git status --porcelain)" ]; then
git add -A
echo "Staged layer mapping files"
fi
# Step 6: Commit and Push (commits all staged files from steps 4 and 5)
- name: Commit and push all mutations
env:
GH_TOKEN: ${{ secrets.GH_MFERG_PAT }}
run: |
# Commit all staged changes (metadata from step 4 + layer mapping from step 5)
# Staged files persist across steps within the same job
if ! git diff --cached --quiet; then
echo "Committing all mutations (metadata generation and/or layer mapping)..."
# Build commit message
COMMIT_PARTS=()
if [ "${{ steps.generate_metadata.outputs.generated_models }}" ] || [ "${{ steps.generate_metadata.outputs.generated_benchmarks }}" ]; then
GENERATED_MODELS="${{ steps.generate_metadata.outputs.generated_models }}"
GENERATED_BENCHMARKS="${{ steps.generate_metadata.outputs.generated_benchmarks }}"
if [ -n "$GENERATED_MODELS" ] && [ -n "$GENERATED_BENCHMARKS" ]; then
COMMIT_PARTS+=("metadata generation: models ($GENERATED_MODELS), benchmarks ($GENERATED_BENCHMARKS)")
elif [ -n "$GENERATED_MODELS" ]; then
COMMIT_PARTS+=("metadata generation: models ($GENERATED_MODELS)")
elif [ -n "$GENERATED_BENCHMARKS" ]; then
COMMIT_PARTS+=("metadata generation: benchmarks ($GENERATED_BENCHMARKS)")
else
COMMIT_PARTS+=("metadata generation")
fi
fi
if [ "${{ steps.layer_mapping.outcome }}" == "success" ]; then
COMMIT_PARTS+=("layer mapping")
fi
COMMIT_MSG="Auto-generate: $(IFS=', '; echo "${COMMIT_PARTS[*]}")"
git commit -m "$COMMIT_MSG" || echo "Commit failed (may be no changes)"
# Use PAT for push (required to trigger workflows)
echo "Using PAT for push and PR operations"
PUSH_TOKEN="$GH_TOKEN"
# Set remote URL with token (GitHub Actions automatically masks secrets in logs)
git remote set-url origin https://x-access-token:${PUSH_TOKEN}@github.com/${{ github.repository }}.git
# Push and capture exit code without exposing token in error output
if git push origin ${{ github.event.pull_request.head.ref }} >/dev/null 2>&1; then
echo "Successfully pushed mutations to PR branch"
echo "Adding submission_prepared label to PR"
PR_NUM="${{ github.event.pull_request.number }}"
gh pr edit ${PR_NUM} --add-label "submission_prepared" || echo "Failed to add label (may already exist on PR)"
else
echo "Push failed - this may prevent workflow rerun. Check if PAT is configured correctly."
exit 1
fi
else
echo "No staged changes to commit"
# If no changes were staged, check if we should add submission_prepared label (metadata might already exist)
PR_NUM="${{ github.event.pull_request.number }}"
echo "No changes to commit, but adding submission_prepared label"
gh pr edit ${PR_NUM} --add-label "submission_prepared" || echo "Failed to add label (may already exist on PR)"
fi
# ============================================================================
# JOB 5: Auto-merge (Conditional - only if validated and approved)
# ============================================================================
automerge:
name: "5. Auto-merge"
needs: [detect_changes, validate_pr, generate_and_commit_mutations]
if: |
always() &&
github.event_name == 'pull_request' &&
needs.detect_changes.outputs.is_fork_pr != 'true' &&
needs.detect_changes.outputs.is_web_submission == 'true' &&
needs.detect_changes.outputs.plugin_type != 'benchmarks' &&
(
needs.detect_changes.outputs.metadata_only == 'true' ||
(needs.detect_changes.outputs.metadata_only != 'true' &&
needs.validate_pr.outputs.is_automergeable == 'true' &&
needs.validate_pr.outputs.all_tests_pass == 'true')
) &&
(
needs.generate_and_commit_mutations.result == 'success' ||
needs.generate_and_commit_mutations.result == 'skipped' ||
(needs.detect_changes.outputs.needs_metadata_generation != 'true' && needs.detect_changes.outputs.needs_mapping != 'true')
)
runs-on: ubuntu-latest
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.ref }}
- name: Get PR info
id: get_pr_info
env:
GH_TOKEN: ${{ secrets.GH_MFERG_PAT }}
run: |
if [ "${{ needs.detect_changes.outputs.metadata_only }}" == "true" ]; then
PR_NUM="${{ github.event.pull_request.number }}"
PR_HEAD_SHA=$(gh pr view ${PR_NUM} --json headRefOid -q '.headRefOid')
echo "pr_number=${PR_NUM}" >> $GITHUB_OUTPUT
echo "pr_head_sha=${PR_HEAD_SHA}" >> $GITHUB_OUTPUT
else
PR_NUM="${{ needs.validate_pr.outputs.pr_number }}"
echo "pr_number=${PR_NUM}" >> $GITHUB_OUTPUT
fi
- name: Check if web submission
id: check_web_submission
env:
GH_TOKEN: ${{ secrets.GH_MFERG_PAT }}
run: |
PR_NUM="${{ steps.get_pr_info.outputs.pr_number }}"
PR_TITLE=$(gh pr view ${PR_NUM} --json title -q '.title')
if echo "$PR_TITLE" | grep -qE '\(user:[^)]+\)'; then
echo "is_web_submission=true" >> $GITHUB_OUTPUT
echo "Web submission detected - automerge enabled"
else
echo "is_web_submission=false" >> $GITHUB_OUTPUT
echo "Non-web submission - automerge disabled"
fi
- name: Set up Python (for metadata-only test check)
if: needs.detect_changes.outputs.metadata_only == 'true'
uses: actions/setup-python@v4
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip'
- name: Install dependencies (for metadata-only test check)
if: needs.detect_changes.outputs.metadata_only == 'true'
run: |
python -m pip install --upgrade pip setuptools
python -m pip install ".[test]"
- name: Check test status for metadata-only PRs
if: needs.detect_changes.outputs.metadata_only == 'true'
id: check_metadata_only_tests
env:
GH_TOKEN: ${{ secrets.GH_MFERG_PAT }}
run: |
# Call validate_pr and capture JSON output
VALIDATION_RESULT=$(python brainscore_language/submission/actions_helpers.py validate_pr \
--pr-number "${{ steps.get_pr_info.outputs.pr_number }}" \
--pr-head "${{ steps.get_pr_info.outputs.pr_head_sha }}" \
--token "${{ secrets.GH_MFERG_PAT }}" \
--poll-interval 30 \
--max-wait-time 7200 2>&1 | grep -E '^\s*\{' | tail -1)
# Parse JSON to extract all_tests_pass
ALL_TESTS_PASS=$(echo "$VALIDATION_RESULT" | python -c "import sys, json; data = json.load(sys.stdin); print('true' if data.get('all_tests_pass') else 'false')" 2>/dev/null || echo "false")
echo "all_tests_pass=${ALL_TESTS_PASS}" >> $GITHUB_OUTPUT
echo "Validation result: ${VALIDATION_RESULT}"
echo "All tests pass: ${ALL_TESTS_PASS}"
- name: Check for submission_prepared label (plugin PRs only)
if: needs.detect_changes.outputs.metadata_only != 'true'
id: check_submission_prepared_automerge
env:
GH_TOKEN: ${{ secrets.GH_MFERG_PAT }}
run: |
PR_NUM="${{ steps.get_pr_info.outputs.pr_number }}"
LABELS_JSON=$(gh pr view ${PR_NUM} --json labels)
if echo "$LABELS_JSON" | jq -e '.labels[] | select(.name == "submission_prepared")' >/dev/null; then
echo "has_submission_prepared=true" >> $GITHUB_OUTPUT
echo "Found submission_prepared label"
else
echo "has_submission_prepared=false" >> $GITHUB_OUTPUT
echo "submission_prepared label not found"
fi
- name: Auto Approve
if: |
steps.check_web_submission.outputs.is_web_submission == 'true' &&
(
(needs.detect_changes.outputs.metadata_only == 'true' && steps.check_metadata_only_tests.outputs.all_tests_pass == 'true') ||
(needs.detect_changes.outputs.metadata_only != 'true' &&
steps.check_submission_prepared_automerge.outputs.has_submission_prepared == 'true' &&
needs.validate_pr.outputs.all_tests_pass == 'true')
)
uses: hmarr/auto-approve-action@v4
with:
pull-request-number: ${{ steps.get_pr_info.outputs.pr_number }}
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Auto Merge (metadata-only PRs)
if: |
steps.check_web_submission.outputs.is_web_submission == 'true' &&
needs.detect_changes.outputs.metadata_only == 'true' &&
steps.check_metadata_only_tests.outputs.all_tests_pass == 'true'
uses: plm9606/automerge_actions@1.2.2
with:
github-token: ${{ secrets.GH_MFERG_PAT }}
label-name: "only_update_metadata"
merge-method: "squash"
auto-delete: "true"
- name: Auto Merge (plugin PRs)
if: |
steps.check_web_submission.outputs.is_web_submission == 'true' &&
needs.detect_changes.outputs.metadata_only != 'true' &&
steps.check_submission_prepared_automerge.outputs.has_submission_prepared == 'true' &&
needs.validate_pr.outputs.all_tests_pass == 'true'
uses: plm9606/automerge_actions@1.2.2
with:
github-token: ${{ secrets.GH_MFERG_PAT }}
label-name: "submission_prepared"
merge-method: "squash"
auto-delete: "true"
# ============================================================================
# JOB 6: Post-Merge Kickoff (Runs after PR is merged)
# ============================================================================
post_merge_scoring:
name: "6. Post-Merge Kickoff"
needs: detect_changes
if: |
github.event_name == 'pull_request_target' &&
github.event.pull_request.merged == true &&
needs.detect_changes.result == 'success' &&
(
(needs.detect_changes.outputs.needs_scoring == 'true' && needs.detect_changes.outputs.metadata_only == 'false') ||
(needs.detect_changes.outputs.metadata_only == 'true')
)
runs-on: ubuntu-latest
env:
BSC_DATABASESECRET: ${{ secrets.BSC_DATABASESECRET }}
steps:
- name: Check out repository code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: 'pip'
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-1
- name: Install dependencies
run: |
python -m pip install --upgrade pip setuptools
python -m pip install "." PyYAML
- name: Check if web submission and extract user_id
if: needs.detect_changes.outputs.metadata_only != 'true'
id: check_web_submission
run: |
PR_TITLE="${{ github.event.pull_request.title }}"
if echo "$PR_TITLE" | grep -qE '\(user:[^)]+\)'; then
echo "is_web_submission=true" >> $GITHUB_OUTPUT
USER_ID=$(echo "$PR_TITLE" | sed -E 's/.*\(user:([^)]+)\).*/\1/')
echo "user_id=${USER_ID}" >> $GITHUB_OUTPUT
echo "Extracted user_id from PR title (masked)"
else
echo "is_web_submission=false" >> $GITHUB_OUTPUT
echo "user_id=2" >> $GITHUB_OUTPUT
fi
- name: Find PR author email for non-web submissions
if: |
needs.detect_changes.outputs.metadata_only != 'true' &&
steps.check_web_submission.outputs.is_web_submission == 'false'
uses: evvanErb/get-github-email-by-username-action@v2.0
id: getemail
continue-on-error: true
with:
github-username: ${{ github.event.pull_request.user.login }}
token: ${{ secrets.GITHUB_TOKEN }}
- name: Extract email for non-web submissions
if: |
needs.detect_changes.outputs.metadata_only != 'true' &&
steps.check_web_submission.outputs.is_web_submission == 'false'
id: extract_email_non_web
run: |
EMAIL="${{ steps.getemail.outputs.email }}"
if [ -z "$EMAIL" ] || [ "$EMAIL" = "null" ] || [ "$EMAIL" = "None" ]; then
EMAIL="mferg@mit.edu"
echo "Could not find email for user ${{ github.event.pull_request.user.login }}, using default email" >&2
fi
echo "::add-mask::$EMAIL"
ENCRYPTED_EMAIL=$(echo -n "$EMAIL" | openssl enc -aes-256-cbc -a -A -salt -pass pass:${{ secrets.EMAIL_ENCRYPTION_KEY }})
echo "email=$ENCRYPTED_EMAIL" >> $GITHUB_OUTPUT
- name: Check if PR title provided (web submissions)
if: |
needs.detect_changes.outputs.metadata_only != 'true' &&
steps.check_web_submission.outputs.is_web_submission == 'true'
id: get_pr_title
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
PR_TITLE="${{ github.event.pull_request.title }}"
if [ -z "$PR_TITLE" ]; then
echo "Fetching PR title because it wasn't provided"
PR_TITLE=$(gh pr view ${{ github.event.pull_request.number }} --repo ${{ github.repository }} --json title -q .title)
fi
echo "PR_TITLE=$PR_TITLE" >> $GITHUB_ENV
- name: Extract email for web submissions (find email from uid)
if: |
needs.detect_changes.outputs.metadata_only != 'true' &&
steps.check_web_submission.outputs.is_web_submission == 'true'
id: extract_email_web
env:
BSC_DATABASESECRET: ${{ secrets.BSC_DATABASESECRET }}
run: |
BS_UID=$(echo "$PR_TITLE" | sed -E 's/.*\(user:([^)]+)\).*/\1/')
EMAIL=$(python -c "from brainscore_core.submission.database import email_from_uid; from brainscore_core.submission.endpoints import UserManager; import os; user_manager=UserManager(db_secret=os.environ['BSC_DATABASESECRET']); print(email_from_uid(int('$BS_UID')))" 2>/dev/null)
if [ -z "$EMAIL" ] || [ "$EMAIL" = "None" ]; then
EMAIL="mferg@mit.edu"
echo "Could not find email in database for user $BS_UID, using default email" >&2
echo "::add-mask::$EMAIL"
fi
echo "::add-mask::$EMAIL"
ENCRYPTED_EMAIL=$(echo -n "$EMAIL" | openssl enc -aes-256-cbc -a -A -salt -pass pass:${{ secrets.EMAIL_ENCRYPTION_KEY }})
echo "email=$ENCRYPTED_EMAIL" >> $GITHUB_OUTPUT
- name: Set email output and decrypt
if: needs.detect_changes.outputs.metadata_only != 'true'
id: extract_email
run: |
if [ "${{ steps.check_web_submission.outputs.is_web_submission }}" == "true" ]; then
ENCRYPTED_EMAIL="${{ steps.extract_email_web.outputs.email }}"
else
ENCRYPTED_EMAIL="${{ steps.extract_email_non_web.outputs.email }}"
fi
EMAIL=$(echo -n "$ENCRYPTED_EMAIL" | openssl enc -d -aes-256-cbc -a -A -salt -pass pass:${{ secrets.EMAIL_ENCRYPTION_KEY }})
echo "::add-mask::$EMAIL"
echo "email=$EMAIL" >> $GITHUB_OUTPUT
- name: Build plugin info
if: needs.detect_changes.outputs.metadata_only != 'true'
id: build_info
run: |
PLUGIN_INFO_B64='${{ needs.detect_changes.outputs.plugin_info_json }}'
PLUGIN_INFO_DECODED=$(echo "$PLUGIN_INFO_B64" | base64 -d)
USER_ID="${{ steps.check_web_submission.outputs.user_id }}"
PLUGIN_INFO=$(echo "$PLUGIN_INFO_DECODED" | jq -c \
--arg domain "language" \
--arg email "${{ steps.extract_email.outputs.email }}" \
--arg plugin_dirs "${{ needs.detect_changes.outputs.plugin_dirs }}" \
--arg plugin_type "${{ needs.detect_changes.outputs.plugin_type }}" \
--arg user_id "$USER_ID" \
--arg bsc_db_secret "${{ secrets.BSC_DATABASESECRET }}" \
'. + {
domain: $domain,
email: $email,
plugin_dirs: $plugin_dirs,
plugin_type: $plugin_type,
competition: "None",
model_type: "artificialsubject",
public: true,
user_id: (if $user_id != "" then $user_id else "2" end),
BSC_DATABASESECRET: $bsc_db_secret
}')
# Store PLUGIN_INFO in GITHUB_ENV for next step (contains email - will be masked by GitHub Actions)
echo "PLUGIN_INFO=${PLUGIN_INFO}" >> $GITHUB_ENV
echo "plugin_info=${PLUGIN_INFO}" >> $GITHUB_OUTPUT
# Mask email and other sensitive fields in log output
PLUGIN_INFO_MASKED=$(echo "$PLUGIN_INFO" | jq -c 'if .email then .email = "***MASKED***" else . end | if .BSC_DATABASESECRET then .BSC_DATABASESECRET = "***MASKED***" else . end' 2>/dev/null || echo "{}")
echo "Plugin info (sensitive data masked): ${PLUGIN_INFO_MASKED}"
- name: Trigger Jenkins scoring (plugin PRs only)
if: needs.detect_changes.outputs.metadata_only != 'true'
env:
JENKINS_USER: ${{ secrets.JENKINS_USER }}
JENKINS_TOKEN: ${{ secrets.JENKINS_TOKEN }}
JENKINS_TRIGGER: ${{ secrets.JENKINS_TRIGGER }}
PLUGIN_INFO: ${{ steps.build_info.outputs.plugin_info }}
run: |
# PLUGIN_INFO contains email - GitHub Actions automatically masks secrets in logs
# Suppress stdout/stderr to prevent any potential exposure of PLUGIN_INFO content
python -c 'from brainscore_language.submission.endpoints import call_jenkins_language; import os; call_jenkins_language(os.environ["PLUGIN_INFO"])' >/dev/null 2>&1 || {
echo "Jenkins scoring trigger failed (check Jenkins logs for details)"
exit 1
}
echo "Jenkins scoring job triggered successfully"
- name: Read metadata and layer mapping files (metadata-only PRs)
if: needs.detect_changes.outputs.metadata_only == 'true'
id: read_metadata_metadata_only
run: |
PLUGIN_DIRS="${{ needs.detect_changes.outputs.plugin_dirs }}"
# Write Python script to read metadata
cat > /tmp/read_metadata.py << 'PYTHON_SCRIPT'
import yaml
import json
import os
import sys
plugin_dirs_str = os.environ.get('PLUGIN_DIRS', '')
metadata_dict = {}
if plugin_dirs_str:
plugin_dirs = [d.strip() for d in plugin_dirs_str.split(',') if d.strip()]
for plugin_dir in plugin_dirs:
if not plugin_dir:
continue
plugin_name = os.path.basename(plugin_dir.rstrip('/'))
metadata_file = None
# Check for metadata.yml or metadata.yaml
if os.path.isfile(os.path.join(plugin_dir, 'metadata.yml')):
metadata_file = os.path.join(plugin_dir, 'metadata.yml')
elif os.path.isfile(os.path.join(plugin_dir, 'metadata.yaml')):
metadata_file = os.path.join(plugin_dir, 'metadata.yaml')
if metadata_file:
try:
with open(metadata_file, 'r') as f:
metadata_content = yaml.safe_load(f)
if metadata_content:
metadata_dict[plugin_name] = metadata_content
except Exception as e:
print(f'Error reading {metadata_file}: {e}', file=sys.stderr)
# Build the final structure
result = {
'metadata': metadata_dict,
'layer_mapping': None # Language domain doesn't have layer mapping
}
print(json.dumps(result))
PYTHON_SCRIPT
# Execute the script
export PLUGIN_DIRS
METADATA_AND_LAYER_MAP=$(python /tmp/read_metadata.py 2>/dev/null || echo '{"metadata": {}, "layer_mapping": null}')
# Base64 encode for safe storage
METADATA_AND_LAYER_MAP_B64=$(echo "$METADATA_AND_LAYER_MAP" | base64 | tr -d '\n')
echo "metadata_and_layer_map_b64=${METADATA_AND_LAYER_MAP_B64}" >> $GITHUB_OUTPUT
echo "Read metadata for plugins: $PLUGIN_DIRS"
- name: Trigger update_existing_metadata Jenkins job (metadata-only PRs)
if: needs.detect_changes.outputs.metadata_only == 'true'
env:
JENKINS_USER: ${{ secrets.JENKINS_USER }}
JENKINS_TOKEN: ${{ secrets.JENKINS_TOKEN }}
JENKINS_TRIGGER: ${{ secrets.JENKINS_TRIGGER }}
run: |
# Decode metadata_and_layer_map
METADATA_AND_LAYER_MAP_B64='${{ steps.read_metadata_metadata_only.outputs.metadata_and_layer_map_b64 }}'
METADATA_AND_LAYER_MAP=$(echo "$METADATA_AND_LAYER_MAP_B64" | base64 -d)
# Base64 encode metadata_and_layer_map for passing to Python script
METADATA_B64=$(echo "$METADATA_AND_LAYER_MAP" | base64 | tr -d '\n')
python brainscore_language/submission/actions_helpers.py trigger_update_existing_metadata \
--plugin-dirs "${{ needs.detect_changes.outputs.plugin_dirs }}" \
--plugin-type "${{ needs.detect_changes.outputs.plugin_type }}" \
--domain "${{ env.DOMAIN }}" \
--metadata-and-layer-map-b64 "$METADATA_B64"
echo "Jenkins update_existing_metadata job triggered successfully"
# ============================================================================
# JOB 7: Failure Notification (Runs if any job fails)
# ============================================================================
notify_on_failure:
name: "7. Notify on Failure"
needs: [detect_changes, validate_pr, handle_metadata_only, generate_and_commit_mutations, automerge, post_merge_scoring]
if: |
always() &&
github.event_name == 'pull_request' &&
needs.detect_changes.outputs.is_fork_pr != 'true' &&
(
needs.detect_changes.result == 'failure' ||
needs.validate_pr.result == 'failure' ||
(needs.validate_pr.result == 'success' && needs.validate_pr.outputs.all_tests_pass == 'false') ||
needs.handle_metadata_only.result == 'failure' ||
needs.generate_and_commit_mutations.result == 'failure' ||
needs.automerge.result == 'failure' ||
needs.post_merge_scoring.result == 'failure'
)
runs-on: ubuntu-latest
steps:
- name: Check out repository code
if: github.event_name == 'pull_request'
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-1
- name: Install dependencies
run: |
python -m pip install --upgrade pip setuptools
python -m pip install "."
- name: Determine which job failed and extract failed tests
id: failed_job
run: |
if [ "${{ needs.detect_changes.result }}" == "failure" ]; then
FAILED_JOB="Detect Changes"
FAILED_TESTS=""
elif [ "${{ needs.validate_pr.result }}" == "failure" ] || [ "${{ needs.validate_pr.outputs.all_tests_pass }}" == "false" ]; then
FAILED_JOB="Validate PR"
# Extract failed tests from test_results
TEST_RESULTS_B64="${{ needs.validate_pr.outputs.test_results_json }}"
if [ -n "$TEST_RESULTS_B64" ] && [ "$TEST_RESULTS_B64" != "null" ]; then
TEST_RESULTS=$(echo "$TEST_RESULTS_B64" | base64 -d)
FAILED_TESTS=$(echo "$TEST_RESULTS" | jq -r 'to_entries | map(select(.value == "failure")) | map(.key) | join(", ")' 2>/dev/null || echo "")
else
FAILED_TESTS=""
fi
elif [ "${{ needs.handle_metadata_only.result }}" == "failure" ]; then
FAILED_JOB="Handle Metadata-Only PR"
FAILED_TESTS=""
elif [ "${{ needs.generate_and_commit_mutations.result }}" == "failure" ]; then
FAILED_JOB="Generate Mutations and Commit"
FAILED_TESTS=""
elif [ "${{ needs.automerge.result }}" == "failure" ]; then
FAILED_JOB="Auto-merge"
FAILED_TESTS=""
elif [ "${{ needs.post_merge_scoring.result }}" == "failure" ]; then
FAILED_JOB="Post-Merge Kickoff"
FAILED_TESTS=""
else
FAILED_JOB="Unknown"
FAILED_TESTS=""
fi
# Build failure reason with test details
if [ -n "$FAILED_TESTS" ] && [ "$FAILED_TESTS" != "" ]; then
FAILURE_REASON="${FAILED_JOB} - Tests failed: ${FAILED_TESTS}"
else
FAILURE_REASON="${FAILED_JOB}"
fi
echo "failed_job=${FAILURE_REASON}" >> $GITHUB_OUTPUT
echo "Failed job: ${FAILURE_REASON}"
- name: Check if web submission
id: check_web_submission
run: |
PR_TITLE="${{ github.event.pull_request.title }}"
if echo "$PR_TITLE" | grep -qE '\(user:[^)]+\)'; then
echo "is_web_submission=true" >> $GITHUB_OUTPUT
else
echo "is_web_submission=false" >> $GITHUB_OUTPUT
fi
- name: Find PR author email for non-web submissions
if: steps.check_web_submission.outputs.is_web_submission == 'false'
uses: evvanErb/get-github-email-by-username-action@v2.0
id: getemail
with:
github-username: ${{ github.event.pull_request.user.login }}
token: ${{ secrets.GITHUB_TOKEN }}
- name: Extract email for non-web submissions
if: steps.check_web_submission.outputs.is_web_submission == 'false'
id: extract_email_non_web
run: |
EMAIL="${{ steps.getemail.outputs.email }}"
if [ -z "$EMAIL" ]; then
EMAIL="mferg@mit.edu"
echo "Could not find email for user ${{ github.event.pull_request.user.login }}, using default email" >&2
echo "::add-mask::$EMAIL"
fi
echo "::add-mask::$EMAIL"
ENCRYPTED_EMAIL=$(echo -n "$EMAIL" | openssl enc -aes-256-cbc -a -A -salt -pass pass:${{ secrets.EMAIL_ENCRYPTION_KEY }})
echo "email=$ENCRYPTED_EMAIL" >> $GITHUB_OUTPUT
echo "email_found=true" >> $GITHUB_OUTPUT
- name: Check if PR title provided (web submissions)
if: steps.check_web_submission.outputs.is_web_submission == 'true'
id: get_pr_title
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
PR_TITLE="${{ github.event.pull_request.title }}"
if [ -z "$PR_TITLE" ]; then
echo "Fetching PR title because it wasn't provided"
PR_TITLE=$(gh pr view ${{ github.event.pull_request.number }} --repo ${{ github.repository }} --json title -q .title)
fi
echo "PR_TITLE=$PR_TITLE" >> $GITHUB_ENV
- name: Extract email for web submissions (find email from uid)
if: steps.check_web_submission.outputs.is_web_submission == 'true'
id: extract_email_web
env:
BSC_DATABASESECRET: ${{ secrets.BSC_DATABASESECRET }}
run: |
BS_UID=$(echo "$PR_TITLE" | sed -E 's/.*\(user:([^)]+)\).*/\1/')
EMAIL=$(python -c "from brainscore_core.submission.database import email_from_uid; from brainscore_core.submission.endpoints import UserManager; import os; user_manager=UserManager(db_secret=os.environ['BSC_DATABASESECRET']); print(email_from_uid(int('$BS_UID')))" 2>/dev/null)
if [ -z "$EMAIL" ] || [ "$EMAIL" = "None" ]; then
EMAIL="mferg@mit.edu"
echo "Could not find email in database for user $BS_UID, using default email" >&2
echo "::add-mask::$EMAIL"
fi
echo "::add-mask::$EMAIL"
ENCRYPTED_EMAIL=$(echo -n "$EMAIL" | openssl enc -aes-256-cbc -a -A -salt -pass pass:${{ secrets.EMAIL_ENCRYPTION_KEY }})
echo "email=$ENCRYPTED_EMAIL" >> $GITHUB_OUTPUT
echo "email_found=true" >> $GITHUB_OUTPUT
- name: Set email output and decrypt
id: get_email
run: |
if [ "${{ steps.check_web_submission.outputs.is_web_submission }}" == "true" ]; then
ENCRYPTED_EMAIL="${{ steps.extract_email_web.outputs.email }}"
EMAIL_FOUND="${{ steps.extract_email_web.outputs.email_found }}"
else
ENCRYPTED_EMAIL="${{ steps.extract_email_non_web.outputs.email }}"
EMAIL_FOUND="${{ steps.extract_email_non_web.outputs.email_found }}"
fi
EMAIL=$(echo -n "$ENCRYPTED_EMAIL" | openssl enc -d -aes-256-cbc -a -A -salt -pass pass:${{ secrets.EMAIL_ENCRYPTION_KEY }})
echo "::add-mask::$EMAIL"
echo "email=$EMAIL" >> $GITHUB_OUTPUT
echo "email_found=$EMAIL_FOUND" >> $GITHUB_OUTPUT
- name: Notify user
if: steps.get_email.outputs.email_found == 'true'
env:
GMAIL_USERNAME: ${{ secrets.GMAIL_USERNAME }}
GMAIL_PASSWORD: ${{ secrets.GMAIL_PASSWORD }}
run: |
PR_NUM="${{ github.event.pull_request.number }}"
python brainscore_language/submission/actions_helpers.py send_failure_email \
"${{ steps.get_email.outputs.email }}" \
"$PR_NUM" \
"${{ steps.failed_job.outputs.failed_job }}" \
"$GMAIL_USERNAME" \
"$GMAIL_PASSWORD"
- name: Log email extraction failure
if: steps.get_email.outputs.email_found == 'false'
run: |
echo "Could not send failure notification email - email not found for user ${{ github.event.pull_request.user.login }}"
echo "Failure reason: ${{ steps.failed_job.outputs.failed_job }}"
PR_NUM="${{ github.event.pull_request.number }}"
echo "PR: https://github.com/${{ github.repository }}/pull/${PR_NUM}"