Skip to content

Commit 85b7914

Browse files
author
Shreyas-Microsoft
committed
add quota check files
1 parent ecccbfb commit 85b7914

File tree

5 files changed

+514
-0
lines changed

5 files changed

+514
-0
lines changed

‎azure.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,19 @@ deployment:
2121
AzureAiServiceLocation: ${{ parameters.AzureAiServiceLocation }}
2222
Prefix: ${{ parameters.Prefix }}
2323
baseUrl: ${{ parameters.baseUrl }}
24+
hooks:
25+
preprovision:
26+
posix:
27+
shell: sh
28+
run: >
29+
chmod u+r+x ./scripts/validate_model_deployment_quota.sh; chmod u+r+x ./scripts/validate_model_quota.sh; ./scripts/validate_model_deployment_quota.sh --SubscriptionId "$AZURE_SUBSCRIPTION_ID" --Location "${AZURE_AISERVICE_LOCATION:-japaneast}" --ModelsParameter "aiModelDeployments"
30+
interactive: false
31+
continueOnError: false
32+
33+
windows:
34+
shell: pwsh
35+
run: >
36+
$location = if ($env:AZURE_AISERVICE_LOCATION) { $env:AZURE_AISERVICE_LOCATION } else { "japaneast" };
37+
./scripts/validate_model_deployment_quota.ps1 -SubscriptionId $env:AZURE_SUBSCRIPTION_ID -Location $location -ModelsParameter "aiModelDeployments"
38+
interactive: false
39+
continueOnError: false

‎scripts/validate_model_deployment_quota.ps1

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
param (
2+
[string]$SubscriptionId,
3+
[string]$Location,
4+
[string]$ModelsParameter
5+
)
6+
7+
# Read from environment variables (do not pass in azure.yaml)
8+
$AiServiceName = $env:AZURE_AISERVICE_NAME
9+
$ResourceGroup = $env:AZURE_RESOURCE_GROUP
10+
11+
# Validate required parameters
12+
$MissingParams = @()
13+
14+
if (-not $SubscriptionId) {
15+
$MissingParams += "subscription"
16+
}
17+
if (-not $Location) {
18+
$MissingParams += "location"
19+
}
20+
if (-not $ModelsParameter) {
21+
$MissingParams += "models-parameter"
22+
}
23+
24+
if ($MissingParams.Count -gt 0) {
25+
Write-Error "❌ ERROR: Missing required parameters: $($MissingParams -join ', ')"
26+
Write-Host "Usage: .\validate_model_deployment_quotas.ps1 -SubscriptionId <SUBSCRIPTION_ID> -Location <LOCATION> -ModelsParameter <MODELS_PARAMETER>"
27+
exit 1
28+
}
29+
30+
# Load main.parameters.json
31+
$JsonContent = Get-Content -Path "./infra/main.parameters.json" -Raw | ConvertFrom-Json
32+
if (-not $JsonContent) {
33+
Write-Error "❌ ERROR: Failed to parse main.parameters.json. Ensure the JSON file is valid."
34+
exit 1
35+
}
36+
37+
$aiModelDeployments = $JsonContent.parameters.$ModelsParameter.value
38+
if (-not $aiModelDeployments -or -not ($aiModelDeployments -is [System.Collections.IEnumerable])) {
39+
Write-Error "❌ ERROR: The specified property '$ModelsParameter' does not exist or is not an array."
40+
exit 1
41+
}
42+
43+
# Check if AI resource + all deployments already exist
44+
if ($AiServiceName -and $ResourceGroup) {
45+
$existing = az cognitiveservices account show `
46+
--name $AiServiceName `
47+
--resource-group $ResourceGroup `
48+
--query "name" --output tsv 2>$null
49+
50+
if ($existing) {
51+
$deployedModels = az cognitiveservices account deployment list `
52+
--name $AiServiceName `
53+
--resource-group $ResourceGroup `
54+
--query "[].name" --output tsv 2>$null
55+
56+
$requiredDeployments = @()
57+
foreach ($deployment in $aiModelDeployments) {
58+
$requiredDeployments += $deployment.name
59+
}
60+
61+
$missingDeployments = @()
62+
foreach ($required in $requiredDeployments) {
63+
if ($deployedModels -notcontains $required) {
64+
$missingDeployments += $required
65+
}
66+
}
67+
68+
if ($missingDeployments.Count -eq 0) {
69+
Write-Host "ℹ️ Azure AI service '$AiServiceName' exists and all required model deployments are provisioned."
70+
Write-Host "⏭️ Skipping quota validation."
71+
exit 0
72+
} else {
73+
Write-Host "🔍 AI service exists, but the following model deployments are missing: $($missingDeployments -join ', ')"
74+
Write-Host "➡️ Proceeding with quota validation for missing models..."
75+
}
76+
}
77+
}
78+
79+
# Start quota validation
80+
az account set --subscription $SubscriptionId
81+
Write-Host "🎯 Active Subscription: $(az account show --query '[name, id]' --output tsv)"
82+
83+
$QuotaAvailable = $true
84+
85+
foreach ($deployment in $aiModelDeployments) {
86+
$name = if ($env:AZURE_ENV_MODEL_NAME) { $env:AZURE_ENV_MODEL_NAME } else { $deployment.name }
87+
$model = if ($env:AZURE_ENV_MODEL_NAME) { $env:AZURE_ENV_MODEL_NAME } else { $deployment.model.name }
88+
$type = if ($env:AZURE_ENV_MODEL_DEPLOYMENT_TYPE) { $env:AZURE_ENV_MODEL_DEPLOYMENT_TYPE } else { $deployment.sku.name }
89+
$capacity = if ($env:AZURE_ENV_MODEL_CAPACITY) { $env:AZURE_ENV_MODEL_CAPACITY } else { $deployment.sku.capacity }
90+
91+
Write-Host "`n🔍 Validating model deployment: $name ..."
92+
& .\scripts\validate_model_quota.ps1 -Location $Location -Model $model -Capacity $capacity -DeploymentType $type
93+
$exitCode = $LASTEXITCODE
94+
95+
if ($exitCode -ne 0) {
96+
if ($exitCode -eq 2) {
97+
exit 1 # already printed, graceful
98+
}
99+
Write-Error "❌ ERROR: Quota validation failed for model deployment: $name"
100+
$QuotaAvailable = $false
101+
}
102+
}
103+
104+
if (-not $QuotaAvailable) {
105+
Write-Error "❌ ERROR: One or more model deployments failed validation."
106+
exit 1
107+
} else {
108+
Write-Host "✅ All model deployments passed quota validation successfully."
109+
exit 0
110+
}

‎scripts/validate_model_deployment_quota.sh

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
#!/bin/bash
2+
3+
SUBSCRIPTION_ID=""
4+
LOCATION=""
5+
MODELS_PARAMETER=""
6+
7+
while [[ $# -gt 0 ]]; do
8+
case "$1" in
9+
--subscription)
10+
SUBSCRIPTION_ID="$2"
11+
shift 2
12+
;;
13+
--location)
14+
LOCATION="$2"
15+
shift 2
16+
;;
17+
--models-parameter)
18+
MODELS_PARAMETER="$2"
19+
shift 2
20+
;;
21+
*)
22+
echo "❌ ERROR: Unknown option: $1"
23+
exit 1
24+
;;
25+
esac
26+
done
27+
28+
# Validate required parameters
29+
MISSING_PARAMS=()
30+
[[ -z "$SUBSCRIPTION_ID" ]] && MISSING_PARAMS+=("subscription")
31+
[[ -z "$LOCATION" ]] && MISSING_PARAMS+=("location")
32+
[[ -z "$MODELS_PARAMETER" ]] && MISSING_PARAMS+=("models-parameter")
33+
34+
if [[ ${#MISSING_PARAMS[@]} -ne 0 ]]; then
35+
echo "❌ ERROR: Missing required parameters: ${MISSING_PARAMS[*]}"
36+
echo "Usage: $0 --subscription <SUBSCRIPTION_ID> --location <LOCATION> --models-parameter <MODELS_PARAMETER>"
37+
exit 1
38+
fi
39+
40+
# Read from environment
41+
AISERVICE_NAME="${AZURE_AISERVICE_NAME}"
42+
RESOURCE_GROUP="${AZURE_RESOURCE_GROUP}"
43+
44+
# Check service and deployment existence
45+
if [[ -n "$AISERVICE_NAME" && -n "$RESOURCE_GROUP" ]]; then
46+
existing=$(az cognitiveservices account show --name "$AISERVICE_NAME" --resource-group "$RESOURCE_GROUP" --query "name" --output tsv 2>/dev/null)
47+
if [[ -n "$existing" ]]; then
48+
echo "ℹ️ Found Azure AI service: $AISERVICE_NAME"
49+
50+
existing_deployments=$(az cognitiveservices account deployment list --name "$AISERVICE_NAME" --resource-group "$RESOURCE_GROUP" --query "[].name" --output tsv 2>/dev/null)
51+
52+
# Extract required model names
53+
required_models=$(jq -r ".parameters.$MODELS_PARAMETER.value[].name" ./infra/main.parameters.json 2>/dev/null)
54+
55+
if [[ -z "$required_models" ]]; then
56+
echo "❌ ERROR: Failed to extract required model names from main.parameters.json"
57+
exit 1
58+
fi
59+
60+
all_present=true
61+
for model in $required_models; do
62+
if ! grep -q -w "$model" <<< "$existing_deployments"; then
63+
all_present=false
64+
break
65+
fi
66+
done
67+
68+
if [[ "$all_present" == "true" ]]; then
69+
echo "✅ All required model deployments already exist in AI service '$AISERVICE_NAME'."
70+
echo "⏭️ Skipping quota validation."
71+
exit 0
72+
else
73+
echo "🔍 AI service exists but some model deployments are missing — proceeding with quota validation."
74+
fi
75+
fi
76+
fi
77+
78+
# If we reach here, continue with normal quota checks
79+
aiModelDeployments=$(jq -c ".parameters.$MODELS_PARAMETER.value[]" ./infra/main.parameters.json)
80+
if [[ $? -ne 0 ]]; then
81+
echo "❌ ERROR: Failed to parse main.parameters.json. Ensure jq is installed and the JSON is valid."
82+
exit 1
83+
fi
84+
85+
az account set --subscription "$SUBSCRIPTION_ID"
86+
echo "🎯 Active Subscription: $(az account show --query '[name, id]' --output tsv)"
87+
88+
quotaAvailable=true
89+
90+
while IFS= read -r deployment; do
91+
name=${AZURE_ENV_MODEL_NAME:-$(echo "$deployment" | jq -r '.name')}
92+
model=${AZURE_ENV_MODEL_NAME:-$(echo "$deployment" | jq -r '.model.name')}
93+
type=${AZURE_ENV_MODEL_DEPLOYMENT_TYPE:-$(echo "$deployment" | jq -r '.sku.name')}
94+
capacity=${AZURE_ENV_MODEL_CAPACITY:-$(echo "$deployment" | jq -r '.sku.capacity')}
95+
96+
echo "🔍 Validating model deployment: $name ..."
97+
./scripts/validate_model_quota.sh --location "$LOCATION" --model "$model" --capacity "$capacity" --deployment-type "$type"
98+
99+
exit_code=$?
100+
if [[ $exit_code -ne 0 ]]; then
101+
if [[ $exit_code -eq 2 ]]; then
102+
# Quota validation handled inside script — stop immediately
103+
exit 1
104+
fi
105+
echo "❌ ERROR: Quota validation failed for model deployment: $name"
106+
quotaAvailable=false
107+
fi
108+
done <<< "$(echo "$aiModelDeployments")"
109+
110+
if [[ "$quotaAvailable" = false ]]; then
111+
echo "❌ ERROR: One or more model deployments failed validation."
112+
exit 1
113+
else
114+
echo "✅ All model deployments passed quota validation successfully."
115+
exit 0
116+
fi

‎scripts/validate_model_quota.ps1

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
param (
2+
[string]$Location,
3+
[string]$Model,
4+
[string]$DeploymentType = "Standard",
5+
[int]$Capacity
6+
)
7+
8+
# Validate parameters
9+
$MissingParams = @()
10+
if (-not $Location) { $MissingParams += "location" }
11+
if (-not $Model) { $MissingParams += "model" }
12+
if (-not $Capacity) { $MissingParams += "capacity" }
13+
if (-not $DeploymentType) { $MissingParams += "deployment-type" }
14+
15+
if ($MissingParams.Count -gt 0) {
16+
Write-Error "❌ ERROR: Missing required parameters: $($MissingParams -join ', ')"
17+
Write-Host "Usage: .\validate_model_quota.ps1 -Location <LOCATION> -Model <MODEL> -Capacity <CAPACITY> [-DeploymentType <DEPLOYMENT_TYPE>]"
18+
exit 1
19+
}
20+
21+
if ($DeploymentType -ne "Standard" -and $DeploymentType -ne "GlobalStandard") {
22+
Write-Error "❌ ERROR: Invalid deployment type: $DeploymentType. Allowed values are 'Standard' or 'GlobalStandard'."
23+
exit 1
24+
}
25+
26+
$ModelType = "OpenAI.$DeploymentType.$Model"
27+
$PreferredRegions = @('australiaeast', 'eastus', 'eastus2', 'francecentral', 'japaneast', 'norwayeast', 'southindia', 'swedencentral', 'uksouth', 'westus', 'westus3')
28+
$AllResults = @()
29+
30+
function Check-Quota {
31+
param (
32+
[string]$Region
33+
)
34+
35+
try {
36+
$ModelInfoRaw = az cognitiveservices usage list --location $Region --query "[?name.value=='$ModelType']" --output json
37+
$ModelInfo = $ModelInfoRaw | ConvertFrom-Json
38+
if (-not $ModelInfo) { return }
39+
40+
$CurrentValue = ($ModelInfo | Where-Object { $_.name.value -eq $ModelType }).currentValue
41+
$Limit = ($ModelInfo | Where-Object { $_.name.value -eq $ModelType }).limit
42+
43+
$CurrentValue = [int]($CurrentValue -replace '\.0+$', '')
44+
$Limit = [int]($Limit -replace '\.0+$', '')
45+
$Available = $Limit - $CurrentValue
46+
47+
return [PSCustomObject]@{
48+
Region = $Region
49+
Model = $ModelType
50+
Limit = $Limit
51+
Used = $CurrentValue
52+
Available = $Available
53+
}
54+
} catch {
55+
return
56+
}
57+
}
58+
59+
# First, check the user-specified region
60+
Write-Host "`n🔍 Checking quota in the requested region '$Location'..."
61+
$PrimaryResult = Check-Quota -Region $Location
62+
63+
if ($PrimaryResult) {
64+
$AllResults += $PrimaryResult
65+
if ($PrimaryResult.Available -ge $Capacity) {
66+
Write-Host "`n✅ Sufficient quota found in original region '$Location'."
67+
exit 0
68+
} else {
69+
Write-Host "`n⚠️ Insufficient quota in '$Location' (Available: $($PrimaryResult.Available), Required: $Capacity). Checking fallback regions..."
70+
}
71+
} else {
72+
Write-Host "`n⚠️ Could not retrieve quota info for region '$Location'. Checking fallback regions..."
73+
}
74+
75+
# Remove primary region from fallback list
76+
$FallbackRegions = $PreferredRegions | Where-Object { $_ -ne $Location }
77+
78+
foreach ($region in $FallbackRegions) {
79+
$result = Check-Quota -Region $region
80+
if ($result) {
81+
$AllResults += $result
82+
}
83+
}
84+
85+
# Display Results Table
86+
Write-Host "`n-------------------------------------------------------------------------------------------------------------"
87+
Write-Host "| No. | Region | Model Name | Limit | Used | Available |"
88+
Write-Host "-------------------------------------------------------------------------------------------------------------"
89+
90+
$count = 1
91+
foreach ($entry in $AllResults) {
92+
$modelShort = $entry.Model.Substring($entry.Model.LastIndexOf(".") + 1)
93+
Write-Host ("| {0,-4} | {1,-16} | {2,-35} | {3,-7} | {4,-7} | {5,-9} |" -f $count, $entry.Region, $entry.Model, $entry.Limit, $entry.Used, $entry.Available)
94+
$count++
95+
}
96+
Write-Host "-------------------------------------------------------------------------------------------------------------"
97+
98+
# Suggest fallback regions
99+
$EligibleFallbacks = $AllResults | Where-Object { $_.Region -ne $Location -and $_.Available -ge $Capacity }
100+
101+
if ($EligibleFallbacks.Count -gt 0) {
102+
Write-Host "`n❌ Deployment cannot proceed in '$Location'."
103+
Write-Host "➡️ You can retry using one of the following regions with sufficient quota:`n"
104+
foreach ($region in $EligibleFallbacks) {
105+
Write-Host " • $($region.Region) (Available: $($region.Available))"
106+
}
107+
108+
Write-Host "`n🔧 To proceed, run:"
109+
Write-Host " azd env set AZURE_AISERVICE_LOCATION '<region>'"
110+
Write-Host "📌 To confirm it's set correctly, run:"
111+
Write-Host " azd env get-value AZURE_AISERVICE_LOCATION"
112+
Write-Host "▶️ Once confirmed, re-run azd up to deploy the model in the new region."
113+
exit 2
114+
}
115+
116+
Write-Error "`n❌ ERROR: No available quota found in any region."
117+
exit 1

0 commit comments

Comments
 (0)