Merge branch 'main' into enable-encryption-at-host-for-vms

microsoft · Jan 8, 2025 · e503c07 · e503c07
2 parents 9f176eb + 3704002
commit e503c07
Show file tree

Hide file tree

Showing 76 changed files with 826 additions and 871 deletions.
diff --git a/.github/workflows/build_validation_develop.yml b/.github/workflows/build_validation_develop.yml
@@ -1,11 +1,11 @@
 ---
 name: Build Validation
 
-on:  # yamllint disable-line rule:truthy
+on: # yamllint disable-line rule:truthy
   pull_request:
     branches:
       - main
-      - 'feature/**'
+      - "feature/**"
 
 # for each ref (branch/pr) run just the most recent,
 # cancel other pending/running ones
@@ -52,6 +52,16 @@ jobs:
             terraform_workspace_services:
               - templates/workspace_services/**/terraform/**/*.tf
 
+      - uses: hashicorp/setup-terraform@v3
+        if: ${{ steps.filter.outputs.terraform == 'true' }}
+        with:
+          terraform_version: "1.9.8"
+
+      - uses: hashicorp/setup-terraform@v3
+        if: ${{ steps.filter.outputs.terraform == 'true' }}
+        with:
+          terraform_version: "1.9.8"
+
       - name: Terraform format check
         if: ${{ steps.filter.outputs.terraform == 'true' }}
         run: terraform fmt -check -recursive
@@ -107,7 +117,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VALIDATE_TERRAFORM_TFLINT: true
           TERRAFORM_TFLINT_CONFIG_FILE: .tflint_core.hcl
-          FILTER_REGEX_INCLUDE: './core/.*'
+          FILTER_REGEX_INCLUDE: "./core/.*"
 
       - name: Workspace Tags
         if: ${{ steps.filter.outputs.terraform_workspaces == 'true' }}
@@ -118,7 +128,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VALIDATE_TERRAFORM_TFLINT: true
           TERRAFORM_TFLINT_CONFIG_FILE: .tflint_workspaces.hcl
-          FILTER_REGEX_INCLUDE: './templates/workspaces/.*'
+          FILTER_REGEX_INCLUDE: "./templates/workspaces/.*"
 
       - name: Workspace Services Tags
         if: ${{ steps.filter.outputs.terraform_workspace_services == 'true' }}
@@ -129,8 +139,8 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VALIDATE_TERRAFORM_TFLINT: true
           TERRAFORM_TFLINT_CONFIG_FILE: .tflint_workspace_services.hcl
-          FILTER_REGEX_INCLUDE: './templates/workspaces/.*'
-          FILTER_REGEX_EXCLUDE: '.*user_resource.*'
+          FILTER_REGEX_INCLUDE: "./templates/workspaces/.*"
+          FILTER_REGEX_EXCLUDE: ".*user_resource.*"
 
       - name: User Resources Tags
         if: ${{ steps.filter.outputs.terraform_workspace_services == 'true' }}
@@ -141,7 +151,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VALIDATE_TERRAFORM_TFLINT: true
           TERRAFORM_TFLINT_CONFIG_FILE: .tflint_user_resources.hcl
-          FILTER_REGEX_INCLUDE: './templates/workspace_services/.*/user_resources/.*'
+          FILTER_REGEX_INCLUDE: "./templates/workspace_services/.*/user_resources/.*"
 
       - name: Shared Services Tags
         if: ${{ steps.filter.outputs.terraform_shared_services == 'true' }}
@@ -152,4 +162,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VALIDATE_TERRAFORM_TFLINT: true
           TERRAFORM_TFLINT_CONFIG_FILE: .tflint_shared_services.hcl
-          FILTER_REGEX_INCLUDE: './templates/shared_services/.*'
+          FILTER_REGEX_INCLUDE: "./templates/shared_services/.*"
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -12,13 +12,13 @@
 #
 name: "CodeQL"
 
-on:  # yamllint disable-line rule:truthy
+on: # yamllint disable-line rule:truthy
   push:
     branches: [main]
   pull_request:
     branches: [main]
   schedule:
-    - cron: '41 3 * * 5'
+    - cron: "41 3 * * 5"
 
 # for each ref (branch/pr) run just the most recent,
 # cancel other pending/running ones
@@ -29,7 +29,7 @@ concurrency:
 jobs:
   analyze:
     name: Analyze
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     permissions:
       actions: read
       contents: read
@@ -38,7 +38,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        language: ['python', 'java', 'javascript', 'typescript']
+        language: ["python", "java", "javascript", "typescript"]
 
     steps:
       - name: Checkout repository

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -14,6 +14,7 @@ ENHANCEMENTS:
 * Split log entries with [Log chunk X of Y] for better readability. ([#3992](https://github.com/microsoft/AzureTRE/issues/3992))
 * Expose APP_SERVICE_SKU build variable to allow enablement of App Gateway WAF ([#4111](https://github.com/microsoft/AzureTRE/pull/4111))
 * Update Terraform to use Azure AD authentication rather than storage account keys ([#4103](https://github.com/microsoft/AzureTRE/issues/4103))
+* Consolidate Terraform upgrade scripts ([#4099](https://github.com/microsoft/AzureTRE/issues/4099))
 * Storage accounts should use infrastructure encryption ([#4001](https://github.com/microsoft/AzureTRE/issues/4001))
 * Update obsolete Terraform properties ([#4136](https://github.com/microsoft/AzureTRE/issues/4136))
 * Update Guacamole version and dependencies ([#4140](https://github.com/microsoft/AzureTRE/issues/4140))
@@ -27,9 +28,12 @@ ENHANCEMENTS:
 * Enhance DPI of Linux display ([[#4200](https://github.com/microsoft/AzureTRE/issues/4200)])
 * Update Admin VM versions ([[#4217](https://github.com/microsoft/AzureTRE/issues/4217)])
 * Update devcontainer/RP/API package versions: base image, docker, az cli, YQ ([#4225](https://github.com/microsoft/AzureTRE/pull/4225))
+* Purge container repos individually in when using `make tre-destroy` ([#4230](https://github.com/microsoft/AzureTRE/pull/4230))
 * Upgrade Python version from 3.8 to 3.12 ([#3949](https://github.com/microsoft/AzureTRE/issues/3949))Upgrade Python version from 3.8 to 3.12 (#3949)
 * Disable storage account key usage ([[#4227](https://github.com/microsoft/AzureTRE/issues/4227)])
 * Update Guacamole dependencies ([[#4232](https://github.com/microsoft/AzureTRE/issues/4232)])
+* Add option to force tunnel TRE's Firewall ([#4237](https://github.com/microsoft/AzureTRE/issues/4237))
+* Add EventGrid diagnostics to identify airlock issues ([#4258](https://github.com/microsoft/AzureTRE/issues/4258))
 
 BUG FIXES:
 * Update KeyVault references in API to use the version so Terraform cascades the update ([#4112](https://github.com/microsoft/AzureTRE/pull/4112))
@@ -45,6 +49,8 @@ BUG FIXES:
 * Bump terraform version in windows VM template ([#4212](https://github.com/microsoft/AzureTRE/issues/4212))
 * Upgrade azurerm terraform provider from v3.112.0 to v3.117.0 to mitiagte storage account deployment issue ([#4004](https://github.com/microsoft/AzureTRE/issues/4004))
 * Fix VM actions where Workspace shared storage doesn't allow shared key access ([#4222](https://github.com/microsoft/AzureTRE/issues/4222))
+* Fix public exposure in Guacamole service ([[#4199](https://github.com/microsoft/AzureTRE/issues/4199)])
+* Fix Azure ML network tags to use name rather than ID ([[#4151](https://github.com/microsoft/AzureTRE/issues/4151)])
 
 COMPONENTS:
 

diff --git a/Makefile b/Makefile
@@ -149,7 +149,7 @@ terraform-upgrade:
 	&& . ${MAKEFILE_DIR}/devops/scripts/check_dependencies.sh env \
 	&& . ${MAKEFILE_DIR}/devops/scripts/load_and_validate_env.sh \
 	&& . ${MAKEFILE_DIR}/devops/scripts/load_env.sh ${DIR}/.env \
-	&& cd ${DIR}/terraform/ && ./upgrade.sh
+	&& ./devops/scripts/upgrade.sh ${DIR}
 
 terraform-import:
 	$(call target_title, "Importing ${DIR} with Terraform") \
@@ -309,8 +309,10 @@ deploy-shared-service:
 	&& ${MAKEFILE_DIR}/devops/scripts/deploy_shared_service.sh $${PROPS}
 
 firewall-install:
-	$(MAKE) bundle-build bundle-publish bundle-register deploy-shared-service \
-	DIR=${MAKEFILE_DIR}/templates/shared_services/firewall/ BUNDLE_TYPE=shared_service
+	. ${MAKEFILE_DIR}/devops/scripts/check_dependencies.sh env \
+	&& $(MAKE) bundle-build bundle-publish bundle-register deploy-shared-service \
+	DIR=${MAKEFILE_DIR}/templates/shared_services/firewall/ BUNDLE_TYPE=shared_service \
+	PROPS="$${FIREWALL_SKU+--firewall_sku $${FIREWALL_SKU} }$${FIREWALL_FORCE_TUNNEL_IP+--firewall_force_tunnel_ip $${FIREWALL_FORCE_TUNNEL_IP} }"
 
 static-web-upload:
 	$(call target_title, "Uploading to static website") \

diff --git a/config.sample.yaml b/config.sample.yaml
@@ -10,7 +10,7 @@ management:
   acr_name: __CHANGE_ME__
   # ID of external Key Vault to store CMKs in (only required if enable_cmk_encryption is true)
   # external_key_store_id: __CHANGE_ME__
-  # Name of Key Vault for encryption keys, required only if enable_cmk_encryption is true and not using external_key_store_id
+  # Name of Key Vault for encryption, required if enable_cmk_encryption is true and external_key_store_id is not set
   # encryption_kv_name: __CHANGE_ME__
   # Azure Resource Manager credentials used for CI/CD pipelines
   arm_subscription_id: __CHANGE_ME__
@@ -46,6 +46,7 @@ tre:
   # The TRE Web UI is deployed by default.
   # Uncomment the following to disable deployment of the Web UI.
   # deploy_ui: false
+  # firewall_force_tunnel_ip: __CHANGE_ME__
   firewall_sku: Standard
   app_gateway_sku: Standard_v2
 

diff --git a/core/terraform/.terraform.lock.hcl b/core/terraform/.terraform.lock.hcl
diff --git a/core/terraform/airlock/airlock_processor.tf b/core/terraform/airlock/airlock_processor.tf
@@ -1,11 +1,3 @@
-data "local_file" "airlock_processor_version" {
-  filename = "${path.root}/../../airlock_processor/_version.py"
-}
-
-locals {
-  version = replace(replace(replace(data.local_file.airlock_processor_version.content, "__version__ = \"", ""), "\"", ""), "\n", "")
-}
-
 resource "azurerm_service_plan" "airlock_plan" {
   name                = "plan-airlock-${var.tre_id}"
   resource_group_name = var.resource_group_name
@@ -44,15 +36,15 @@ resource "azurerm_storage_account" "sa_airlock_processor_func_app" {
   # changing this value is destructive, hence attribute is in lifecycle.ignore_changes block below
   infrastructure_encryption_enabled = true
 
-  lifecycle { ignore_changes = [infrastructure_encryption_enabled, tags] }
-}
+  dynamic "customer_managed_key" {
+    for_each = var.enable_cmk_encryption ? [1] : []
+    content {
+      key_vault_key_id          = var.encryption_key_versionless_id
+      user_assigned_identity_id = var.encryption_identity_id
+    }
+  }
 
-resource "azurerm_storage_account_customer_managed_key" "sa_airlock_processor_func_app_encryption" {
-  count                     = var.enable_cmk_encryption ? 1 : 0
-  storage_account_id        = azurerm_storage_account.sa_airlock_processor_func_app.id
-  key_vault_id              = var.key_store_id
-  key_name                  = var.kv_encryption_key_name
-  user_assigned_identity_id = var.encryption_identity_id
+  lifecycle { ignore_changes = [infrastructure_encryption_enabled, tags] }
 }
 
 resource "azurerm_linux_function_app" "airlock_function_app" {

diff --git a/core/terraform/airlock/data.tf b/core/terraform/airlock/data.tf
@@ -0,0 +1,21 @@
+data "local_file" "airlock_processor_version" {
+  filename = "${path.root}/../../airlock_processor/_version.py"
+}
+
+data "azurerm_private_dns_zone" "eventgrid" {
+  name                = module.terraform_azurerm_environment_configuration.private_links["privatelink.eventgrid.azure.net"]
+  resource_group_name = var.resource_group_name
+}
+
+data "azurerm_container_registry" "mgmt_acr" {
+  name                = var.mgmt_acr_name
+  resource_group_name = var.mgmt_resource_group_name
+}
+
+data "azurerm_monitor_diagnostic_categories" "eventgrid_custom_topics" {
+  resource_id = azurerm_eventgrid_topic.airlock_notification.id
+}
+
+data "azurerm_monitor_diagnostic_categories" "eventgrid_system_topics" {
+  resource_id = azurerm_eventgrid_system_topic.export_approved_blob_created.id
+}
diff --git a/core/terraform/airlock/eventgrid_topics.tf b/core/terraform/airlock/eventgrid_topics.tf
@@ -1,8 +1,3 @@
-data "azurerm_private_dns_zone" "eventgrid" {
-  name                = module.terraform_azurerm_environment_configuration.private_links["privatelink.eventgrid.azure.net"]
-  resource_group_name = var.resource_group_name
-}
-
 # Below we assign a SYSTEM-assigned identity for the topics. note that a user-assigned identity will not work.
 
 # Event grid topics
@@ -511,3 +506,50 @@ resource "azurerm_eventgrid_event_subscription" "export_approved_blob_created" {
   ]
 }
 
+resource "azurerm_monitor_diagnostic_setting" "eventgrid_custom_topics" {
+  for_each = merge({
+    (azurerm_eventgrid_topic.airlock_notification.name) = azurerm_eventgrid_topic.airlock_notification.id,
+    (azurerm_eventgrid_topic.step_result.name)          = azurerm_eventgrid_topic.step_result.id,
+    (azurerm_eventgrid_topic.status_changed.name)       = azurerm_eventgrid_topic.status_changed.id,
+    (azurerm_eventgrid_topic.data_deletion.name)        = azurerm_eventgrid_topic.data_deletion.id,
+    },
+    var.enable_malware_scanning ? { (azurerm_eventgrid_topic.scan_result[0].name) = azurerm_eventgrid_topic.scan_result[0].id } : null
+  )
+
+  name                       = "${each.key}-diagnostics"
+  target_resource_id         = each.value
+  log_analytics_workspace_id = var.log_analytics_workspace_id
+  dynamic "enabled_log" {
+    for_each = data.azurerm_monitor_diagnostic_categories.eventgrid_custom_topics.log_category_types
+    content {
+      category = enabled_log.value
+    }
+  }
+
+  metric {
+    category = "AllMetrics"
+  }
+}
+
+resource "azurerm_monitor_diagnostic_setting" "eventgrid_system_topics" {
+  for_each = {
+    (azurerm_eventgrid_system_topic.import_inprogress_blob_created.name) = azurerm_eventgrid_system_topic.import_inprogress_blob_created.id,
+    (azurerm_eventgrid_system_topic.import_rejected_blob_created.name)   = azurerm_eventgrid_system_topic.import_rejected_blob_created.id,
+    (azurerm_eventgrid_system_topic.import_blocked_blob_created.name)    = azurerm_eventgrid_system_topic.import_blocked_blob_created.id,
+    (azurerm_eventgrid_system_topic.export_approved_blob_created.name)   = azurerm_eventgrid_system_topic.export_approved_blob_created.id,
+  }
+
+  name                       = "${each.key}-diagnostics"
+  target_resource_id         = each.value
+  log_analytics_workspace_id = var.log_analytics_workspace_id
+  dynamic "enabled_log" {
+    for_each = data.azurerm_monitor_diagnostic_categories.eventgrid_system_topics.log_category_types
+    content {
+      category = enabled_log.value
+    }
+  }
+
+  metric {
+    category = "AllMetrics"
+  }
+}
diff --git a/core/terraform/airlock/identity.tf b/core/terraform/airlock/identity.tf
@@ -1,8 +1,3 @@
-data "azurerm_container_registry" "mgmt_acr" {
-  name                = var.mgmt_acr_name
-  resource_group_name = var.mgmt_resource_group_name
-}
-
 resource "azurerm_user_assigned_identity" "airlock_id" {
   resource_group_name = var.resource_group_name
   location            = var.location

diff --git a/core/terraform/airlock/locals.tf b/core/terraform/airlock/locals.tf
@@ -1,4 +1,6 @@
 locals {
+  version = replace(replace(replace(data.local_file.airlock_processor_version.content, "__version__ = \"", ""), "\"", ""), "\n", "")
+
   # STorage AirLock EXternal
   import_external_storage_name = lower(replace("stalimex${var.tre_id}", "-", ""))
   # STorage AirLock IMport InProgress