Merge branch 'main' into psl-re-use-log-analytic-update

Author: Thanusree-Microsoft

.github/workflows/deploy.yml

@@ -36,6 +36,43 @@ jobs:
         run: |
           az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }}
 
+      - name: Run Quota Check
+        id: quota-check
+        run: |
+          export AZURE_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }}
+          export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}
+          export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }}
+          export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}"
+          export GPT_MIN_CAPACITY="${{ env.GPT_MIN_CAPACITY }}"
+          export AZURE_REGIONS="${{ vars.AZURE_REGIONS }}"
+          chmod +x scripts/checkquota.sh
+          if ! scripts/checkquota.sh; then
+            # If quota check fails due to insufficient quota, set the flag
+            if grep -q "No region with sufficient quota found" scripts/checkquota.sh; then
+              echo "QUOTA_FAILED=true" >> $GITHUB_ENV
+            fi
+            exit 1  # Fail the pipeline if any other failure occurs
+          fi
+
+      - name: Send Notification on Quota Failure
+        if: env.QUOTA_FAILED == 'true'
+        run: |
+          RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          EMAIL_BODY=$(cat <<EOF
+          {
+            "body": "<p>Dear Team,</p><p>The quota check has failed, and the pipeline cannot proceed.</p><p><strong>Build URL:</strong> <a href=\"${RUN_URL}\">${RUN_URL}</a></p><p>Please take necessary action.</p><p>Best regards,<br>Your Automation Team</p>"
+          }
+          EOF
+          )
+
+          curl -X POST "${{ secrets.LOGIC_APP_URL }}" \
+            -H "Content-Type: application/json" \
+            -d "$EMAIL_BODY" || echo "Failed to send notification"
+
+      - name: Fail Pipeline if Quota Check Fails
+        if: env.QUOTA_FAILED == 'true'
+        run: exit 1
+
       - name: Install Bicep CLI
         run: az bicep install
 
@@ -94,11 +131,11 @@ jobs:
             --resource-group ${{ env.RESOURCE_GROUP_NAME }} \
             --template-file infra/main.bicep \
             --parameters \
-                Prefix="${{ env.SOLUTION_PREFIX }}" \
-                AzureAiServiceLocation="eastus" \
+                solutionName="${{ env.SOLUTION_PREFIX }}" \
+                aiDeploymentsLocation="eastus" \
+                useWafAlignedArchitecture=false \
                 capacity=${{ env.GPT_MIN_CAPACITY }} \
-                imageVersion="${IMAGE_TAG}"\
-            --debug
+                imageVersion="${IMAGE_TAG}"
 
       - name: Get Deployment Output and extract Values
         id: get_output
@@ -111,24 +148,6 @@ jobs:
           echo "WEBAPP_URL=$WEBAPP_URL" >> $GITHUB_OUTPUT
           echo "Deployment output: $BICEP_OUTPUT"
 
-      - name: Send Notification on Failure
-        if: failure()
-        run: |
-          RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-      
-          # Construct the email body
-          EMAIL_BODY=$(cat <<EOF
-          {
-            "body": "<p>Dear Team,</p><p>We would like to inform you that the Modernize-your-code-solution-accelerator Automation process has encountered an issue and has failed to complete successfully.</p><p><strong>Build URL:</strong> ${RUN_URL}<br> ${OUTPUT}</p><p>Please investigate the matter at your earliest convenience.</p><p>Best regards,<br>Your Automation Team</p>"
-          }
-          EOF
-          )
-      
-          # Send the notification
-          curl -X POST "${{ secrets.LOGIC_APP_URL }}" \
-            -H "Content-Type: application/json" \
-            -d "$EMAIL_BODY" || echo "Failed to send notification"
-
       - name: Logout from Azure
         if: always()
         run: |
@@ -171,7 +190,6 @@ jobs:
       
           if [ -z "$log_analytics_workspace_name" ]; then
             echo "No Log Analytics workspace found in resource group ${{ env.RESOURCE_GROUP_NAME }}."
-            exit 1
           else
             echo "LOG_ANALYTICS_WORKSPACE_NAME=${log_analytics_workspace_name}" >> $GITHUB_ENV
             echo "Log Analytics workspace name: ${log_analytics_workspace_name}" 
@@ -224,22 +242,6 @@ jobs:
             echo "KEYVAULTS=$keyvault_array" >> $GITHUB_ENV
           fi
 
-      - name: Purge log analytics workspace
-        if: always()
-        id: log_analytics_workspace
-        run: |
-
-          set -e
-          # Purge Log Analytics Workspace
-          echo "Purging the Log Analytics Workspace..."
-          if ! az monitor log-analytics workspace delete --force --resource-group ${{ env.RESOURCE_GROUP_NAME }} --workspace-name ${{ env.LOG_ANALYTICS_WORKSPACE_NAME }} --yes --verbose; then
-            echo "Failed to purge Log Analytics workspace: ${{ env.LOG_ANALYTICS_WORKSPACE_NAME }}"
-          else
-            echo "Purged the Log Analytics workspace: ${{ env.LOG_ANALYTICS_WORKSPACE_NAME }}"
-          fi
-
-          echo "Log analytics workspace resource purging completed successfully"    
-      
       - name: Delete Bicep Deployment
         if: always()
         run: |
@@ -257,6 +259,23 @@ jobs:
             echo "Resource group does not exists."
           fi
 
+      - name: Purge log analytics workspace
+        if: always()
+        id: log_analytics_workspace
+        run: |
+
+          set -e
+          # Purge Log Analytics Workspace
+          echo "Purging the Log Analytics Workspace..."
+          if ! az monitor log-analytics workspace delete --force --resource-group ${{ env.RESOURCE_GROUP_NAME }} --workspace-name ${{ env.LOG_ANALYTICS_WORKSPACE_NAME }} --yes --verbose; then
+            echo "Failed to purge Log Analytics workspace: ${{ env.LOG_ANALYTICS_WORKSPACE_NAME }}"
+          else
+            echo "Purged the Log Analytics workspace: ${{ env.LOG_ANALYTICS_WORKSPACE_NAME }}"
+          fi
+
+          echo "Log analytics workspace resource purging completed successfully"    
+      
+
       - name: Wait for resource deletion to complete
         if: always()
         run: |
@@ -363,6 +382,22 @@ jobs:
           done
           echo "Resource purging completed successfully"
 
+      - name: Send Notification on Failure
+        if: failure() || needs.deploy.result == 'failure'
+        run: |
+          RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
+          EMAIL_BODY=$(cat <<EOF
+          {
+            "body": "<p>Dear Team,</p><p>We would like to inform you that the DocGen Deployment Automation process has encountered an issue and has failed to complete successfully.</p><p><strong>Build URL:</strong> <a href=\"${RUN_URL}\">${RUN_URL}</a><br></p><p>Please investigate the matter at your earliest convenience.</p><p>Best regards,<br>Your Automation Team</p>"
+          }
+          EOF
+          )
+
+          curl -X POST "${{ secrets.LOGIC_APP_URL }}" \
+            -H "Content-Type: application/json" \
+            -d "$EMAIL_BODY" || echo "Failed to send notification"
+
       - name: Logout from Azure
         if: always()
         run: |

README.md

@@ -8,7 +8,7 @@ The Modernize your code solution accelerator allows users to specify a group of
 <br/>
 
 <div align="center">
-  
+
 [**SOLUTION OVERVIEW**](#solution-overview) \| [**QUICK DEPLOY**](#quick-deploy) \| [**BUSINESS SCENARIO**](#business-scenario) \| [**SUPPORTING DOCUMENTATION**](#supporting-documentation)
 
 </div>
@@ -24,7 +24,10 @@ The solution leverages Azure AI Foundry, Azure OpenAI Service, Azure Container A
 |![image](./docs/images/read_me/solArchitecture.png)|
 |---|
 
+This architecture will be deployed with the 'sandbox' setting of our deployment process. Optionally you can deploy [Well-Architected Framework (WAF) aligned](https://learn.microsoft.com/en-us/azure/well-architected/) architecture, described in [WAF-Aligned Solution Architecture](./docs/ArchitectureWAF.md), with the WAF-Aligned deployment option described in [Deployment Guide](./docs/DeploymentGuide.md).
+
 ### Agentic architecture
+
 |![image](./docs/images/read_me/agentArchitecture.png)|
 |---|
 
@@ -51,16 +54,16 @@ If you'd like to customize the solution accelerator, here are some common areas
   <summary>Click to learn more about the key features this solution enables</summary>
 
   - **Code language modernization** <br/>
-  Modernizing outdated code ensures compatibility with current technologies, reduces reliance on legacy expertise, and keeps businesses competitive.
+    Modernizing outdated code ensures compatibility with current technologies, reduces reliance on legacy expertise, and keeps businesses competitive.
 
   - **Summary and review of new code** <br/>
-  Generating summaries and translating code files keeps humans in the loop, enhances their understanding, and facilitates timely interventions, ensuring the files are ready to export.
+    Generating summaries and translating code files keeps humans in the loop, enhances their understanding, and facilitates timely interventions, ensuring the files are ready to export.
 
   - **Business logic analysis** <br/>
-  Leveraging AI to decipher business logic from legacy code helps minimizes the risk of human error.
+    Leveraging AI to decipher business logic from legacy code helps minimizes the risk of human error.
 
   - **Efficient code transformation** <br/>
-  Streamlining the process of analyzing, converting, and iterative error testing reduces time and effort required to modernize the systems.
+    Streamlining the process of analyzing, converting, and iterative error testing reduces time and effort required to modernize the systems.
 
 </details>
 
@@ -77,7 +80,7 @@ Follow the quick deploy steps on the deployment guide to deploy this solution to
 
 | [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/microsoft/Modernize-your-Code-Solution-Accelerator) | [![Open in Dev Containers](https://img.shields.io/static/v1?style=for-the-badge&label=Dev%20Containers&message=Open&color=blue&logo=visualstudiocode)](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/microsoft/Modernize-your-Code-Solution-Accelerator) |
 |---|---|
- 
+
 <br/>
 
 > ⚠️ **Important: Check Azure OpenAI Quota Availability**
@@ -141,19 +144,19 @@ The sample data used in this repository is synthetic and generated using Azure O
   <summary>Click to learn more about what value this solution provides</summary>
 
   - **Accelerated Migration** <br/>
-  Automate the translation of SQL queries, significantly reducing migration time and effort.
+    Automate the translation of SQL queries, significantly reducing migration time and effort.
 
   - **Error Reduction** <br/>
-  Multi-agent validation ensures accurate translations and maintains data integrity.
+    Multi-agent validation ensures accurate translations and maintains data integrity.
 
   - **Knowledge Preservation** <br/>
-  Captures and preserves business logic during the modernization process.
+    Captures and preserves business logic during the modernization process.
 
   - **Cost Efficiency** <br/>
-  Reduces reliance on specialized legacy system expertise and manual translation efforts.
+    Reduces reliance on specialized legacy system expertise and manual translation efforts.
 
   - **Standardization** <br/>
-  Ensures consistent query translation across the organization.
+    Ensures consistent query translation across the organization.
 
 </details>
 

azure.yaml

@@ -1,6 +1,3 @@
-environment:
-  name: modernize-your-code-solution-accelerator
-  location: eastus
 name: modernize-your-code-solution-accelerator
 metadata:
     template: [email protected]
@@ -21,19 +18,3 @@ deployment:
     AzureAiServiceLocation: ${{ parameters.AzureAiServiceLocation }}
     Prefix: ${{ parameters.Prefix }}
     baseUrl: ${{ parameters.baseUrl }}
-hooks:
-  preprovision:
-    posix:
-      shell: sh
-      run: >
-        chmod u+r+x ./scripts/validate_model_deployment_quota.sh; chmod u+r+x ./scripts/validate_model_quota.sh; ./scripts/validate_model_deployment_quota.sh --SubscriptionId "$AZURE_SUBSCRIPTION_ID" --Location "${AZURE_AISERVICE_LOCATION:-japaneast}" --ModelsParameter "aiModelDeployments"
-      interactive: false
-      continueOnError: false
-
-    windows:
-      shell: pwsh
-      run: >
-        $location = if ($env:AZURE_AISERVICE_LOCATION) { $env:AZURE_AISERVICE_LOCATION } else { "japaneast" };
-        ./scripts/validate_model_deployment_quota.ps1 -SubscriptionId $env:AZURE_SUBSCRIPTION_ID -Location $location -ModelsParameter "aiModelDeployments"
-      interactive: false
-      continueOnError: false

docs/ArchitectureWAF.md

@@ -0,0 +1,59 @@
+# Azure WAF-Aligned Architecture
+
+This architecture implements [Azure Well-Architected Framework (WAF)](https://learn.microsoft.com/en-us/azure/well-architected/) principles for enterprise-grade deployments, deployed with the WAF-Aligned deployment option: 
+
+![WAF-Aligned Architecture Diagram](../docs/images/read_me/solArchitectureWAF.png)
+
+## WAF Pillars Implementation
+
+###  Security
+- **Zero Trust Network:** Private VNet with private endpoints for all PaaS services
+- **Identity & Access:** Managed identities with RBAC and least-privilege access
+- **Secure Admin Access:** Azure Bastion + Jumpbox for internal administration
+- **Secrets Management:** Azure Key Vault integration
+
+###  Operational Excellence  
+- **Observability:** Centralized logging via Log Analytics Workspace
+- **Application Monitoring:** Application Insights for telemetry and diagnostics
+- **Infrastructure as Code:** Bicep templates with parameterized configurations
+
+### Performance Efficiency
+- **Auto-scaling:** Container Apps with configurable scaling policies
+- **Regional Proximity:** Resources deployed in optimal Azure regions
+
+###  Cost Optimization
+- **Right-sizing:** Parameterized SKUs and capacity settings
+- **Resource Sharing:** Shared networking and monitoring infrastructure
+
+###  Reliability
+- **High Availability:** Multi-zone deployment options
+- **Data Redundancy:** Configurable geo-replication for critical data stores
+- **Private Connectivity:** Eliminates internet dependencies
+
+## Core Architecture Components
+
+| Component | Purpose | WAF Alignment |
+|-----------|---------|---------------|
+| **Virtual Network** | Network isolation boundary | Security, Reliability |
+| **Private Endpoints** | Secure PaaS connectivity (AI Services, Storage, Cosmos DB, Key Vault) | Security |
+| **Private DNS Zones** | Internal name resolution | Security, Reliability |
+| **Azure Bastion + Jumpbox** | Secure administrative access | Security |
+| **Container Apps** | Application hosting with VNet integration | Performance, Reliability |
+| **Log Analytics + App Insights** | Centralized monitoring and diagnostics | Operational Excellence |
+
+## Deployment Configuration
+- **Configurable Parameters:** If user selects to deploy as WAF Aligned, Parameters like Monitoring, Scaling, VPN will get enabled.
+- **Network-first Design:** All components deployed within private network boundaries
+- **Enterprise-ready:** Production-grade security and monitoring enabled
+
+## Application Information Flow
+
+The application information flow remains the same for both 'sandbox' and 'waf-aligned' configuration. 
+
+The solution is composed of several services:
+
+- The web app front end and the backend app logic are containerized and run from Azure Container service instances. 
+- When a request for conversion is created in the web app admin console, the user specifies what files should be converted and the target SQL dialect for conversion. 
+- These files are then uploaded to blob storage and initial data about the request is stored in Cosmos DB. 
+- The conversion takes place using appropriate LLM models using multiple agents, with each agent having a dedicated purpose in the conversion process. As files are converted, they are placed into blob storage, with metadata collected into Cosmos detailing the conversion process and the current state of the batch. 
+- Cosmos also stores the logs from the individual agents so the results can be fully reviewed before any of the converted files are put into production.

docs/CmsaArchitectureSource.pptx

@@ -17,7 +17,7 @@ By default this template will use the environment name as the prefix to prevent
 | `AZURE_ENV_MODEL_CAPACITY`             | integer | `200`            | Set the Model Capacity (choose a number based on available GPT model capacity in your subscription). |
 | `AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID` | string  | Guide to get your [Existing Workspace ID](/docs/re-use-log-analytics.md)     | Set this if you want to reuse an existing Log Analytics Workspace instead of creating a new one.     |
 | `AZURE_ENV_IMAGETAG`                   | string  | `latest`         | Set the Image tag Like (allowed values: latest, dev, hotfix)    |
-
+| `AZURE_ENV_JUMPBOX_SIZE`                   | string  | `Standard_DS2_v2`         | Specifies the size of the Jumpbox Virtual Machine. Set a custom value if `enablePrivateNetworking` is `true`.    |
 ---
 
 ## How to Set a Parameter

docs/CustomizingAzdParameters.md

@@ -181,14 +181,14 @@ To change the azd parameters from the default values, follow the steps [here](..
 
 1. Login to Azure:
 
-    ```shell
-    azd auth login
-    ```
+   ```shell
+   azd auth login
+   ```
 
-    #### Note: To authenticate with Azure Developer CLI (`azd`) to a specific tenant, use the previous command with your **Tenant ID**:
+   #### Note: To authenticate with Azure Developer CLI (`azd`) to a specific tenant, use the previous command with your **Tenant ID**:
 
-    ```sh
-    azd auth login --tenant-id <tenant-id>
+   ```sh
+   azd auth login --tenant-id <tenant-id>
    ```
 
 2. Provide an `azd` environment name (like "cmsaapp")
@@ -202,7 +202,7 @@ To change the azd parameters from the default values, follow the steps [here](..
     ```shell
     azd up
     ```
-    
+  
 4. Select a subscription from your Azure account, and select a location which has quota for all the resources. 
     * This deployment will take *6-9 minutes* to provision the resources in your account and set up the solution with sample data. 
     * If you get an error or timeout with deployment, changing the location can help, as there may be availability constraints for the resources.