Skip to content

Commit 53a15bc

Browse files
Vamshi-MicrosoftVamshi-Microsoft
committed
Enhance quota check process for OpenAI and Azure Search by refining region validation and error handling
1 parent 7867d17 commit 53a15bc

1 file changed

Lines changed: 107 additions & 63 deletions

File tree

‎.github/workflows/job-deploy.yml‎

Lines changed: 107 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -290,85 +290,129 @@ jobs:
290290
az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }}
291291
az account set --subscription ${{ secrets.AZURE_SUBSCRIPTION_ID }}
292292
293-
- name: Run Quota Check
293+
- name: Run OpenAI and Azure Search Quota Check
294294
id: quota-check
295-
run: |
296-
export AZURE_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }}
297-
export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}
298-
export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }}
299-
export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}"
300-
export GPT_MIN_CAPACITY="150"
301-
export TEXT_EMBEDDING_MIN_CAPACITY="30"
302-
export AZURE_REGIONS="${{ vars.AZURE_REGIONS }}"
303-
304-
chmod +x scripts/checkquota.sh
305-
if ! scripts/checkquota.sh; then
306-
if grep -q "No region with sufficient quota found" scripts/checkquota.sh; then
307-
echo "QUOTA_FAILED=true" >> $GITHUB_OUTPUT
308-
echo "Quota check failed - will notify via separate notification job"
309-
fi
310-
exit 1
311-
fi
312-
313-
- name: Set Deployment Region
314-
id: set_region
315295
shell: bash
316296
run: |
317-
echo "Selected Region from Quota Check: $VALID_REGION"
318-
echo "AZURE_ENV_OPENAI_LOCATION=$VALID_REGION" >> $GITHUB_ENV
319-
echo "AZURE_ENV_OPENAI_LOCATION=$VALID_REGION" >> $GITHUB_OUTPUT
320-
echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV
321-
echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_OUTPUT
322-
323-
- name: Check Azure Search Service Quota
324-
id: search-quota-check
325-
shell: bash
326-
run: |
327-
echo "🔍 Checking Azure Search service quota in region: ${{ env.AZURE_LOCATION }}"
297+
SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}"
298+
GPT_MIN_CAPACITY="${{ env.GPT_MIN_CAPACITY }}"
299+
TEXT_EMBEDDING_MIN_CAPACITY="${{ env.TEXT_EMBEDDING_MIN_CAPACITY }}"
300+
IFS=', ' read -ra REGIONS <<< "${{ vars.AZURE_REGIONS }}"
301+
302+
declare -A MIN_CAPACITY=(
303+
["OpenAI.GlobalStandard.gpt4.1"]=$GPT_MIN_CAPACITY
304+
["OpenAI.GlobalStandard.text-embedding-ada-002"]=$TEXT_EMBEDDING_MIN_CAPACITY
305+
)
306+
307+
VALID_REGION=""
308+
309+
for REGION in "${REGIONS[@]}"; do
310+
echo "========================================"
311+
echo "🔍 Checking region: $REGION"
312+
echo "========================================"
313+
314+
# --- Step 1: Check OpenAI quota ---
315+
echo "📊 Checking OpenAI quota..."
316+
QUOTA_INFO=$(az cognitiveservices usage list --location "$REGION" --output json 2>/dev/null)
317+
if [ -z "$QUOTA_INFO" ]; then
318+
echo "⚠️ WARNING: Failed to retrieve OpenAI quota for region $REGION. Skipping."
319+
continue
320+
fi
328321
329-
# Get current usage and limits for Azure Search in the selected region using REST API
330-
SEARCH_USAGE=$(az rest --method get \
331-
--url "https://management.azure.com/subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/providers/Microsoft.Search/locations/${{ env.AZURE_LOCATION }}/usages?api-version=2024-03-01-preview" \
332-
-o json 2>/dev/null || echo '{"value":[]}')
322+
OPENAI_OK=true
323+
for MODEL in "${!MIN_CAPACITY[@]}"; do
324+
MODEL_INFO=$(echo "$QUOTA_INFO" | awk -v model="\"value\": \"$MODEL\"" '
325+
BEGIN { RS="},"; FS="," }
326+
$0 ~ model { print $0 }
327+
')
328+
329+
if [ -z "$MODEL_INFO" ]; then
330+
echo "⚠️ WARNING: No quota information found for model: $MODEL in $REGION. Skipping region."
331+
OPENAI_OK=false
332+
break
333+
fi
334+
335+
CURRENT_VALUE=$(echo "$MODEL_INFO" | awk -F': ' '/"currentValue"/ {print $2}' | tr -d ',' | tr -d ' ')
336+
LIMIT=$(echo "$MODEL_INFO" | awk -F': ' '/"limit"/ {print $2}' | tr -d ',' | tr -d ' ')
337+
CURRENT_VALUE=${CURRENT_VALUE:-0}
338+
LIMIT=${LIMIT:-0}
339+
CURRENT_VALUE=$(echo "$CURRENT_VALUE" | cut -d'.' -f1)
340+
LIMIT=$(echo "$LIMIT" | cut -d'.' -f1)
341+
AVAILABLE=$((LIMIT - CURRENT_VALUE))
342+
343+
echo " Model: $MODEL | Used: $CURRENT_VALUE | Limit: $LIMIT | Available: $AVAILABLE"
344+
345+
if [ "$AVAILABLE" -lt "${MIN_CAPACITY[$MODEL]}" ]; then
346+
echo " ❌ $MODEL in $REGION has insufficient OpenAI quota."
347+
OPENAI_OK=false
348+
break
349+
fi
350+
done
351+
352+
if [ "$OPENAI_OK" != "true" ]; then
353+
echo "⏭️ Region $REGION failed OpenAI quota check. Trying next region..."
354+
continue
355+
fi
356+
echo "✅ OpenAI quota check passed for $REGION."
357+
358+
# --- Step 2: Check Azure Search quota ---
359+
echo "📊 Checking Azure Search quota..."
360+
SEARCH_USAGE=$(az rest --method get \
361+
--url "https://management.azure.com/subscriptions/${SUBSCRIPTION_ID}/providers/Microsoft.Search/locations/${REGION}/usages?api-version=2024-03-01-preview" \
362+
-o json 2>/dev/null || echo '{"value":[]}')
363+
364+
if [ "$(echo "$SEARCH_USAGE" | jq '.value | length')" -eq 0 ]; then
365+
echo "⚠️ Could not retrieve Azure Search quota for $REGION. Accepting region (search check skipped)."
366+
VALID_REGION="$REGION"
367+
break
368+
fi
333369
334-
if [ "$(echo "$SEARCH_USAGE" | jq '.value | length')" -eq 0 ]; then
335-
echo "⚠️ Could not retrieve Azure Search quota information. Proceeding with deployment..."
336-
echo "SEARCH_QUOTA_CHECK=skipped" >> $GITHUB_OUTPUT
337-
else
338-
# Check Standard tier quota
339370
STANDARD_USAGE=$(echo "$SEARCH_USAGE" | jq -r '.value[] | select(.name.value == "standard") | .currentValue // 0')
340371
STANDARD_LIMIT=$(echo "$SEARCH_USAGE" | jq -r '.value[] | select(.name.value == "standard") | .limit // 0')
341372
STANDARD_AVAILABLE=$((STANDARD_LIMIT - STANDARD_USAGE))
342373
343-
echo "Azure Search Standard Tier Quota for ${{ env.AZURE_LOCATION }}:"
344-
echo " Current Usage: $STANDARD_USAGE"
345-
echo " Limit: $STANDARD_LIMIT"
346-
echo " Available: $STANDARD_AVAILABLE"
374+
echo " Azure Search Standard Tier | Used: $STANDARD_USAGE | Limit: $STANDARD_LIMIT | Available: $STANDARD_AVAILABLE"
347375
348376
if [ "$STANDARD_AVAILABLE" -lt 1 ]; then
349-
echo ""
350-
echo "❌ ERROR: Insufficient Azure Search Standard tier quota in region: ${{ env.AZURE_LOCATION }}"
351-
echo "Please request a quota increase or select a different region."
352-
353-
# Add error to GitHub Summary
354-
echo "## ❌ Azure Search Quota Check Failed" >> $GITHUB_STEP_SUMMARY
355-
echo "" >> $GITHUB_STEP_SUMMARY
356-
echo "**Region:** \`${{ env.AZURE_LOCATION }}\`" >> $GITHUB_STEP_SUMMARY
357-
echo "" >> $GITHUB_STEP_SUMMARY
358-
echo "| Metric | Value |" >> $GITHUB_STEP_SUMMARY
359-
echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY
360-
echo "| **Tier** | Standard |" >> $GITHUB_STEP_SUMMARY
361-
echo "| **Current Usage** | $STANDARD_USAGE |" >> $GITHUB_STEP_SUMMARY
362-
echo "| **Limit** | $STANDARD_LIMIT |" >> $GITHUB_STEP_SUMMARY
363-
echo "| **Available** | $STANDARD_AVAILABLE |" >> $GITHUB_STEP_SUMMARY
364-
exit 1
377+
echo " ❌ Insufficient Azure Search Standard tier quota in $REGION."
378+
echo "⏭️ Trying next region..."
379+
continue
365380
fi
366381
382+
echo "✅ Azure Search quota check passed for $REGION."
383+
384+
VALID_REGION="$REGION"
385+
break
386+
done
387+
388+
if [ -z "$VALID_REGION" ]; then
367389
echo ""
368-
echo "✅ Azure Search Standard tier quota check passed."
369-
echo "SEARCH_QUOTA_CHECK=passed" >> $GITHUB_OUTPUT
390+
echo "❌ No region with sufficient OpenAI AND Azure Search quota found across all regions: ${REGIONS[*]}"
391+
392+
echo "## ❌ Quota Check Failed" >> $GITHUB_STEP_SUMMARY
393+
echo "" >> $GITHUB_STEP_SUMMARY
394+
echo "No region had sufficient quota for **both** OpenAI models and Azure Search Standard tier." >> $GITHUB_STEP_SUMMARY
395+
echo "" >> $GITHUB_STEP_SUMMARY
396+
echo "**Regions checked:** ${REGIONS[*]}" >> $GITHUB_STEP_SUMMARY
397+
398+
echo "QUOTA_FAILED=true" >> $GITHUB_OUTPUT
399+
exit 1
370400
fi
371401
402+
echo ""
403+
echo "✅ Final Selected Region: $VALID_REGION (passed both OpenAI and Azure Search quota checks)"
404+
echo "VALID_REGION=$VALID_REGION" >> $GITHUB_ENV
405+
406+
- name: Set Deployment Region
407+
id: set_region
408+
shell: bash
409+
run: |
410+
echo "Selected Region: $VALID_REGION"
411+
echo "AZURE_ENV_OPENAI_LOCATION=$VALID_REGION" >> $GITHUB_ENV
412+
echo "AZURE_ENV_OPENAI_LOCATION=$VALID_REGION" >> $GITHUB_OUTPUT
413+
echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV
414+
echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_OUTPUT
415+
372416
- name: Add Resource Group Name to Environment
373417
env:
374418
INPUT_RESOURCE_GROUP_NAME: ${{ inputs.resource_group_name }}

0 commit comments

Comments
 (0)