From ff3d3dd9a9d4ce15b279b1b7ee7e8751d2a212dc Mon Sep 17 00:00:00 2001 From: Jesper Fajers Date: Tue, 25 Jun 2024 18:05:09 +0200 Subject: [PATCH] Fix Duplicate conflicting deployments with DeployAllMultipleTemplateParameterFiles (#887) * Update * Update * Update --- src/functions/Invoke-AzOpsPush.ps1 | 67 ++++++++++++------- src/tests/integration/Repository.Tests.ps1 | 30 +++++++-- src/tests/templates/decoy.westeurope.bicep | 12 ++++ .../decoy.westeurope.x123.parameters.json | 9 +++ .../templates/deployallrt.westeurope.bicep | 12 ++++ ...eployallrt.westeurope.x123.parameters.json | 9 +++ .../deployallrt.westeurope.xabc.bicepparam | 3 + .../templates/deployallrt2.westeurope.bicep | 12 ++++ 8 files changed, 126 insertions(+), 28 deletions(-) create mode 100644 src/tests/templates/decoy.westeurope.bicep create mode 100644 src/tests/templates/decoy.westeurope.x123.parameters.json create mode 100644 src/tests/templates/deployallrt.westeurope.bicep create mode 100644 src/tests/templates/deployallrt.westeurope.x123.parameters.json create mode 100644 src/tests/templates/deployallrt.westeurope.xabc.bicepparam create mode 100644 src/tests/templates/deployallrt2.westeurope.bicep diff --git a/src/functions/Invoke-AzOpsPush.ps1 b/src/functions/Invoke-AzOpsPush.ps1 index 45b6df8d..c0af098c 100644 --- a/src/functions/Invoke-AzOpsPush.ps1 +++ b/src/functions/Invoke-AzOpsPush.ps1 @@ -158,7 +158,7 @@ #region Case: Parameters File if (($fileItem.Name.EndsWith('.parameters.json')) -or ($fileItem.Name.EndsWith('.bicepparam'))) { $result.TemplateParameterFilePath = $fileItem.FullName - $deploymentName = $fileItem.Name -replace (Get-PSFConfigValue -FullName 'AzOps.Core.TemplateParameterFileSuffix'), '' -replace ' ', '_' -replace '\.bicepparam', '' + $deploymentName = $fileItem.Name -replace "\.parameters\$(Get-PSFConfigValue -FullName 'AzOps.Core.TemplateParameterFileSuffix')$", '' -replace ' ', '_' -replace '\.bicepparam$', '' if ($deploymentName.Length -gt 53) { $deploymentName = $deploymentName.SubString(0, 53) } $result.DeploymentName = 'AzOps-{0}-{1}' -f $deploymentName, $deploymentRegionId @@ -167,12 +167,12 @@ { $_.EndsWith('.parameters.json') } { if ((Get-PSFConfigValue -FullName 'AzOps.Core.AllowMultipleTemplateParameterFiles') -eq $true -and $fileItem.FullName.Split('.')[-3] -match $(Get-PSFConfigValue -FullName 'AzOps.Core.MultipleTemplateParameterFileSuffix').Replace('.','')) { Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Resolve.MultipleTemplateParameterFile' -LogStringValues $FilePath - $templatePath = $fileItem.FullName -replace "\.$($fileItem.FullName.Split('.')[-3])\.parameters.json$", '.json' - $bicepTemplatePath = $fileItem.FullName -replace "\.$($fileItem.FullName.Split('.')[-3])\.parameters.json$", '.bicep' + $templatePath = $fileItem.FullName -replace "\.$($fileItem.FullName.Split('.')[-3])\.parameters\.json$", '.json' + $bicepTemplatePath = $fileItem.FullName -replace "\.$($fileItem.FullName.Split('.')[-3])\.parameters\.json$", '.bicep' } else { - $templatePath = $fileItem.FullName -replace '\.parameters.json$', (Get-PSFConfigValue -FullName 'AzOps.Core.TemplateParameterFileSuffix') - $bicepTemplatePath = $fileItem.FullName -replace '\.parameters.json$', '.bicep' + $templatePath = $fileItem.FullName -replace '\.parameters\.json$', (Get-PSFConfigValue -FullName 'AzOps.Core.TemplateParameterFileSuffix') + $bicepTemplatePath = $fileItem.FullName -replace '\.parameters\.json$', '.bicep' } if (Test-Path $templatePath) { if ($CompareDeploymentToDeletion) { @@ -299,17 +299,21 @@ if ($paramFileList) { $multiResult = @() foreach ($paramFile in $paramFileList) { - if ($CompareDeploymentToDeletion) { - # Avoid adding files destined for deletion to a deployment list - if ($paramFile.VersionInfo.FileName -in $deleteSet -or $paramFile.VersionInfo.FileName -in ($deleteSet | Resolve-Path).Path) { - Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Resolve.DeployDeletionOverlap' -LogStringValues $paramFile.VersionInfo.FileName - continue + # Check if the parameter file's name matches the expected pattern + $escapedBaseName = $fileItem.BaseName -replace '\.', '\.' + if ($paramFile.BaseName -match "^$escapedBaseName(\$(Get-PSFConfigValue -FullName 'AzOps.Core.MultipleTemplateParameterFileSuffix'))") { + if ($CompareDeploymentToDeletion) { + # Avoid adding files destined for deletion to a deployment list + if ($paramFile.VersionInfo.FileName -in $deleteSet -or $paramFile.VersionInfo.FileName -in ($deleteSet | Resolve-Path).Path) { + Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Resolve.DeployDeletionOverlap' -LogStringValues $paramFile.VersionInfo.FileName + continue + } + } + # Process parameter files for template equivalent + if (($fileItem.FullName.Split('.')[-2] -eq $paramFile.FullName.Split('.')[-3]) -or ($fileItem.FullName.Split('.')[-2] -eq $paramFile.FullName.Split('.')[-4])) { + Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Resolve.MultipleTemplateParameterFile' -LogStringValues $paramFile.FullName + $multiResult += Resolve-ArmFileAssociation -ScopeObject $scopeObject -FilePath $paramFile -AzOpsMainTemplate $AzOpsMainTemplate -ConvertedTemplate $ConvertedTemplate -ConvertedParameter $ConvertedParameter -CompareDeploymentToDeletion:$CompareDeploymentToDeletion } - } - # Process possible parameter files for template equivalent - if (($fileItem.FullName.Split('.')[-2] -eq $paramFile.FullName.Split('.')[-3]) -or ($fileItem.FullName.Split('.')[-2] -eq $paramFile.FullName.Split('.')[-4])) { - Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Resolve.MultipleTemplateParameterFile' -LogStringValues $paramFile.FullName - $multiResult += Resolve-ArmFileAssociation -ScopeObject $scopeObject -FilePath $paramFile -AzOpsMainTemplate $AzOpsMainTemplate -ConvertedTemplate $ConvertedTemplate -ConvertedParameter $ConvertedParameter -CompareDeploymentToDeletion:$CompareDeploymentToDeletion } } if ($multiResult) { @@ -318,20 +322,16 @@ } else { Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Resolve.ParameterNotFound' -LogStringValues $FilePath, $parameterPath + if (-not (Test-TemplateDefaultParameter -FilePath $FilePath)) { + continue + } } - } } else { Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Resolve.ParameterNotFound' -LogStringValues $FilePath, $parameterPath if ((Get-PSFConfigValue -FullName 'AzOps.Core.AllowMultipleTemplateParameterFiles') -eq $true) { - # Check for template parameters without defaultValue - $defaultValueContent = Get-Content $FilePath - $missingDefaultParam = $defaultValueContent | jq '.parameters | with_entries(select(.value.defaultValue == null))' | ConvertFrom-Json -AsHashtable - if ($missingDefaultParam.Count -ge 1) { - # Skip template deployment when template parameters without defaultValue are found and no parameter file identified - $missingString = foreach ($item in $missingDefaultParam.Keys.GetEnumerator()) {"$item,"} - Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Resolve.NotFoundParamFileDefaultValue' -LogStringValues $FilePath, ($missingString | Out-String -NoNewline) + if (-not (Test-TemplateDefaultParameter -FilePath $FilePath)) { continue } } @@ -344,6 +344,27 @@ $result #endregion Case: Template File } + function Test-TemplateDefaultParameter { + param( + [string]$FilePath + ) + + # Read the template file + $defaultValueContent = Get-Content $FilePath + # Check for parameters without a default value using jq + $missingDefaultParam = $defaultValueContent | jq '.parameters | with_entries(select(.value.defaultValue == null))' | ConvertFrom-Json -AsHashtable + if ($missingDefaultParam.Count -ge 1) { + # Skip template deployment when template parameters without defaultValue are found and no parameter file identified + $missingString = foreach ($item in $missingDefaultParam.Keys.GetEnumerator()) {"$item,"} + # Log a debug message with the missing parameters + Write-AzOpsMessage -LogLevel Debug -LogString 'Invoke-AzOpsPush.Resolve.NotFoundParamFileDefaultValue' -LogStringValues $FilePath, ($missingString | Out-String -NoNewline) + # Missing default value were found + return $false + } else { + # Default values found + return $true + } + } #endregion Utility Functions $WhatIfPreferenceState = $WhatIfPreference diff --git a/src/tests/integration/Repository.Tests.ps1 b/src/tests/integration/Repository.Tests.ps1 index d0cb8dc4..6eeb777e 100644 --- a/src/tests/integration/Repository.Tests.ps1 +++ b/src/tests/integration/Repository.Tests.ps1 @@ -139,7 +139,7 @@ Describe "Repository" { $script:resourceGroupParallelDeploy = (Get-AzResourceGroup | Where-Object ResourceGroupName -eq "ParallelDeploy-azopsrg") $script:roleAssignments = (Get-AzRoleAssignment -ObjectId "023e7c1c-1fa4-4818-bb78-0a9c5e8b0217" | Where-Object { $_.Scope -eq "/subscriptions/$script:subscriptionId" -and $_.RoleDefinitionId -eq "acdd72a7-3385-48ef-bd42-f606fba81ae7" }) $script:policyExemptions = Get-AzPolicyExemption -Name "PolicyExemptionTest" -Scope "/subscriptions/$script:subscriptionId" - $script:routeTable = (Get-AzResource -Name "RouteTable" -ResourceGroupName $($script:resourceGroup).ResourceGroupName) + $script:routeTable = (Get-AzResource -Name "RouteTable" -ResourceGroupName $script:resourceGroup.ResourceGroupName) $script:policyAssignmentsDeletion = Get-AzPolicyAssignment -Name "TestPolicyAssignmentDeletion" -Scope "/subscriptions/$script:subscriptionId/resourceGroups/$($script:resourceGroupCustomDeletion.ResourceGroupName)" $script:ruleCollectionGroups = (Get-AzResource -ExpandProperties -Name "TestPolicy" -ResourceGroupName $($script:resourceGroup).ResourceGroupName).Properties.ruleCollectionGroups.id.split("/")[-1] $script:logAnalyticsWorkspace = (Get-AzResource -Name "thisisalongloganalyticsworkspacename123456789011121314151617181" -ResourceGroupName $($script:resourceGroup).ResourceGroupName) @@ -1162,7 +1162,7 @@ Describe "Repository" { ) {Invoke-AzOpsPush -ChangeSet $changeSet} | Should -Not -Throw Start-Sleep -Seconds 5 - $script:bicepMultiParamPathDeployment = Get-AzResource -ResourceGroupName $($script:resourceGroup).ResourceGroupName -ResourceType 'Microsoft.Network/routeTables' | Where-Object {$_.name -like "rtmultibasex*"} + $script:bicepMultiParamPathDeployment = Get-AzResource -ResourceGroupName $script:resourceGroup.ResourceGroupName -ResourceType 'Microsoft.Network/routeTables' | Where-Object {$_.name -like "rtmultibasex*"} $script:bicepMultiParamPathDeployment.Count | Should -Be 2 } #endregion @@ -1178,7 +1178,7 @@ Describe "Repository" { ) {Invoke-AzOpsPush -ChangeSet $changeSet} | Should -Not -Throw Start-Sleep -Seconds 5 - $script:bicepRepeatSuffixPathDeployment = Get-AzResource -ResourceGroupName $($script:resourceGroup).ResourceGroupName -ResourceType 'Microsoft.Network/routeTables' | Where-Object {$_.name -like "rtsuffix*"} + $script:bicepRepeatSuffixPathDeployment = Get-AzResource -ResourceGroupName $script:resourceGroup.ResourceGroupName -ResourceType 'Microsoft.Network/routeTables' | Where-Object {$_.name -like "rtsuffix*"} $script:bicepRepeatSuffixPathDeployment.Count | Should -Be 2 Set-PSFConfig -FullName AzOps.Core.MultipleTemplateParameterFileSuffix -Value ".x" } @@ -1214,11 +1214,31 @@ Describe "Repository" { ) {Invoke-AzOpsPush -ChangeSet $changeSet} | Should -Not -Throw Start-Sleep -Seconds 5 - $script:deployAllRtParamPathDeployment = Get-AzResource -ResourceGroupName $($script:resourceGroup).ResourceGroupName -ResourceType 'Microsoft.Network/routeTables' | Where-Object {$_.name -like "deployallrtbasex*"} + $script:deployAllRtParamPathDeployment = Get-AzResource -ResourceGroupName $script:resourceGroup.ResourceGroupName -ResourceType 'Microsoft.Network/routeTables' | Where-Object {$_.name -like "deployallrtbasex*"} $script:deployAllRtParamPathDeployment.Count | Should -Be 2 } #endregion + #region Bicep template with change, AzOps set to resolve corresponding parameter files and create multiple deployments [3] and avoid [1] decoy + It "Deploy Bicep template with change, AzOps set to resolve corresponding parameter files and create multiple deployments [3] and avoid [1] decoy" { + Set-PSFConfig -FullName AzOps.Core.AllowMultipleTemplateParameterFiles -Value $true + Set-PSFConfig -FullName AzOps.Core.DeployAllMultipleTemplateParameterFiles -Value $true + $script:deployAllRtFilesPath = Get-ChildItem -Path "$($global:testRoot)/templates/deployallrt.westeurope*" | Copy-Item -Destination $script:resourceGroupDirectory -PassThru -Force + $script:deployAllRt2FilesPath = Get-ChildItem -Path "$($global:testRoot)/templates/deployallrt2.westeurope.bicep" | Copy-Item -Destination $script:resourceGroupDirectory -PassThru -Force + $script:decoyRtFilesPath = Get-ChildItem -Path "$($global:testRoot)/templates/decoy.westeurope*" | Copy-Item -Destination $script:resourceGroupDirectory -PassThru -Force + $changeSet = @( + "A`t$($script:deployAllRtFilesPath.FullName[0])", + "A`t$($script:deployAllRt2FilesPath.FullName)" + ) + $testFiles = Invoke-AzOpsPush -ChangeSet $changeSet + $? | Should -Be $true + $testFiles.Count | Should -Be 3 + Start-Sleep -Seconds 5 + $script:deployAllRtDeployment = Get-AzResource -ResourceGroupName $script:resourceGroup.ResourceGroupName -ResourceType 'Microsoft.Network/routeTables' | Where-Object {$_.name -like "deployallrtwex*" -or $_.name -like "deployallrt2wex*"} + $script:deployAllRtDeployment.Count | Should -Be 3 + } + #endregion + #region Multiple deployments to test parallel deployment logic It "Deploy parallel storage accounts and compare to serial timing" { Set-PSFConfig -FullName AzOps.Core.AllowMultipleTemplateParameterFiles -Value $true @@ -1232,7 +1252,7 @@ Describe "Repository" { ) {Invoke-AzOpsPush -ChangeSet $changeSet} | Should -Not -Throw Start-Sleep -Seconds 30 - $script:deployAllStaParamPathDeployment = Get-AzResource -ResourceGroupName $($script:resourceGroupParallelDeploy).ResourceGroupName -ResourceType 'Microsoft.Storage/storageAccounts' + $script:deployAllStaParamPathDeployment = Get-AzResource -ResourceGroupName $script:resourceGroupParallelDeploy.ResourceGroupName -ResourceType 'Microsoft.Storage/storageAccounts' $script:deployAllStaParamPathDeployment.Count | Should -Be 4 $query = "resourcechanges | where resourceGroup =~ '$($($script:resourceGroupParallelDeploy).ResourceGroupName)' and properties.targetResourceType == 'microsoft.storage/storageaccounts' and properties.changeType == 'Create' | extend changeTime=todatetime(properties.changeAttributes.timestamp), targetResourceId=tostring(properties.targetResourceId) | summarize arg_max(changeTime, *) by targetResourceId | project changeTime, targetResourceId, properties.changeType, properties.targetResourceType | order by changeTime asc" $createTime = Search-AzGraph -Query $query -Subscription $script:subscriptionId diff --git a/src/tests/templates/decoy.westeurope.bicep b/src/tests/templates/decoy.westeurope.bicep new file mode 100644 index 00000000..f790cbf7 --- /dev/null +++ b/src/tests/templates/decoy.westeurope.bicep @@ -0,0 +1,12 @@ +param name string +param location string = resourceGroup().location + +resource symbolicname 'Microsoft.Network/routeTables@2023-04-01' = { + name: name + location: location + properties: { + disableBgpRoutePropagation: false + routes: [ + ] + } +} diff --git a/src/tests/templates/decoy.westeurope.x123.parameters.json b/src/tests/templates/decoy.westeurope.x123.parameters.json new file mode 100644 index 00000000..3229d17c --- /dev/null +++ b/src/tests/templates/decoy.westeurope.x123.parameters.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "name": { + "value": "decoywex123" + } + } +} \ No newline at end of file diff --git a/src/tests/templates/deployallrt.westeurope.bicep b/src/tests/templates/deployallrt.westeurope.bicep new file mode 100644 index 00000000..f790cbf7 --- /dev/null +++ b/src/tests/templates/deployallrt.westeurope.bicep @@ -0,0 +1,12 @@ +param name string +param location string = resourceGroup().location + +resource symbolicname 'Microsoft.Network/routeTables@2023-04-01' = { + name: name + location: location + properties: { + disableBgpRoutePropagation: false + routes: [ + ] + } +} diff --git a/src/tests/templates/deployallrt.westeurope.x123.parameters.json b/src/tests/templates/deployallrt.westeurope.x123.parameters.json new file mode 100644 index 00000000..2e3bdd78 --- /dev/null +++ b/src/tests/templates/deployallrt.westeurope.x123.parameters.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "name": { + "value": "deployallrtwex123" + } + } +} \ No newline at end of file diff --git a/src/tests/templates/deployallrt.westeurope.xabc.bicepparam b/src/tests/templates/deployallrt.westeurope.xabc.bicepparam new file mode 100644 index 00000000..c71db6eb --- /dev/null +++ b/src/tests/templates/deployallrt.westeurope.xabc.bicepparam @@ -0,0 +1,3 @@ +using './deployallrt.westeurope.bicep' + +param name = toLower('deployallrtwexabc') diff --git a/src/tests/templates/deployallrt2.westeurope.bicep b/src/tests/templates/deployallrt2.westeurope.bicep new file mode 100644 index 00000000..83b6269c --- /dev/null +++ b/src/tests/templates/deployallrt2.westeurope.bicep @@ -0,0 +1,12 @@ +param name string = 'deployallrt2wex123' +param location string = resourceGroup().location + +resource symbolicname 'Microsoft.Network/routeTables@2023-04-01' = { + name: name + location: location + properties: { + disableBgpRoutePropagation: false + routes: [ + ] + } +}