From b7cd45a9180dc6dd76064b720ca1f19cba44bb83 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Mon, 4 Nov 2024 13:03:11 +0100 Subject: [PATCH] Increase Get-BcArtifactUrl Performance (#3739) When looking for the latest version of the artifacts for a specific country, we need to fetch all artifacts in the blob storage in order to figure out what is latest. This fix uses an approximation of latest version number (actually the one shipping in a month) and trying to download that (or 2 prior versions) Additional improvements we are looking at for future optimizations: - Use OCI artifacts for artifacts storage instead of traditional blob storage - Restructure artifacts in more layers to allow for partial download of what you need in various scenarios - Create index if artifacts to allow for querying artifacts without reading tags/annotations --------- Co-authored-by: freddydk --- Artifacts/Get-BCArtifactUrl.ps1 | 306 ++++++++++++++++++-------------- BC.HelperFunctions.ps1 | 1 + ReleaseNotes.txt | 3 + Version.txt | 2 +- 4 files changed, 175 insertions(+), 137 deletions(-) diff --git a/Artifacts/Get-BCArtifactUrl.ps1 b/Artifacts/Get-BCArtifactUrl.ps1 index 3b0d7437b..cf13ed698 100644 --- a/Artifacts/Get-BCArtifactUrl.ps1 +++ b/Artifacts/Get-BCArtifactUrl.ps1 @@ -141,172 +141,206 @@ try { if ($storageAccount -eq 'bcinsider.blob.core.windows.net' -and !$accept_insiderEULA) { throw "You need to accept the insider EULA (https://go.microsoft.com/fwlink/?linkid=2245051) by specifying -accept_insiderEula or by providing a SAS token to get access to insider builds" } - $GetListUrl = "https://$storageAccount/$($Type.ToLowerInvariant())/?comp=list&restype=container" - - $upMajorFilter = '' - $upVersionFilter = '' - if ($select -eq 'SecondToLastMajor') { - if ($version) { - throw "You cannot specify a version when asking for the Second To Last Major version" - } - } - elseif ($select -eq 'Closest') { - if (!($version)) { - throw "You must specify a version number when you want to get the closest artifact Url" + + if ($type -eq 'sandbox' -and $storageAccount -eq 'bcartifacts.blob.core.windows.net' -and $select -eq 'latest' -and $version -eq '' -and $bcContainerHelperConfig.useApproximateVersion) { + # Temp fix / hack for Get-BcArtifact performance + # If Microsoft changes versioning schema, this needs to change (or useApproximateVersion should be set to false) + $now = ([DateTime]::Now).AddDays(15) + $approximateMajor = 23+2*($now.Year-2024)+($now.Month -ge 4)+($now.Month -ge 10) + $approximateMinor = ($now.Month + 2)%6 + $artifactUrl = Get-BCArtifactUrl -country $country -version "$approximateMajor.$approximateMinor" -select Latest -doNotCheckPlatform:$doNotCheckPlatform + if ($artifactUrl) { + # We found an artifact - check if it is the latest + while ($artifactUrl) { + $lastGoodArtifact = $artifactUrl + if ($approximateMinor -eq 5) { + $approximateMajor += 1 + $approximateMinor = 0 + } + else { + $approximateMinor += 1 + } + $artifactUrl = Get-BCArtifactUrl -country $country -version "$approximateMajor.$approximateMinor" -select Latest -doNotCheckPlatform:$doNotCheckPlatform + } + $artifactUrl = $lastGoodArtifact } - $dots = ($version.ToCharArray() -eq '.').Count - $closestToVersion = [Version]"0.0.0.0" - if ($dots -ne 3 -or !([Version]::TryParse($version, [ref] $closestToVersion))) { - throw "Version number must be in the format 1.2.3.4 when you want to get the closest artifact Url" + else { + # No artifact found - try previous 3 versions (else give up - maybe country is unavailable) + $tryVersions = 3 + while (-not $artifactUrl -and $tryVersions-- -gt 0) { + if ($approximateMinor -eq 0) { + $approximateMajor -= 1 + $approximateMinor = 5 + } + else { + $approximateMinor -= 1 + } + $artifactUrl = Get-BCArtifactUrl -country $country -version "$approximateMajor.$approximateMinor" -select Latest -doNotCheckPlatform:$doNotCheckPlatform + } } - $GetListUrl += "&prefix=$($closestToVersion.Major).$($closestToVersion.Minor)." - $upMajorFilter = "$($closestToVersion.Major)" - $upVersionFilter = "$($closestToVersion.Minor)." + $artifactUrl } - elseif (!([string]::IsNullOrEmpty($version))) { - $dots = ($version.ToCharArray() -eq '.').Count - if ($dots -lt 3) { - # avoid 14.1 returning 14.10, 14.11 etc. - $version = "$($version.TrimEnd('.'))." + else { + $GetListUrl = "https://$storageAccount/$($Type.ToLowerInvariant())/?comp=list&restype=container" + if ($select -eq 'SecondToLastMajor') { + if ($version) { + throw "You cannot specify a version when asking for the Second To Last Major version" + } } - $GetListUrl += "&prefix=$($Version)" - $upMajorFilter = $version.Split('.')[0] - $upVersionFilter = $version.Substring($version.Length).TrimStart('.') - } - - $Artifacts = @() - $nextMarker = '' - $currentMarker = '' - $downloadAttempt = 1 - $downloadRetryAttempts = 10 - do { - if ($currentMarker -ne $nextMarker) - { - $currentMarker = $nextMarker - $downloadAttempt = 1 + elseif ($select -eq 'Closest') { + if (!($version)) { + throw "You must specify a version number when you want to get the closest artifact Url" + } + $dots = ($version.ToCharArray() -eq '.').Count + $closestToVersion = [Version]"0.0.0.0" + if ($dots -ne 3 -or !([Version]::TryParse($version, [ref] $closestToVersion))) { + throw "Version number must be in the format 1.2.3.4 when you want to get the closest artifact Url" + } + $GetListUrl += "&prefix=$($closestToVersion.Major).$($closestToVersion.Minor)." } - Write-Verbose "Download String $GetListUrl$nextMarker" - try - { - $Response = Invoke-RestMethod -UseBasicParsing -ContentType "application/json; charset=UTF8" -Uri "$GetListUrl$nextMarker" - if (([int]$Response[0]) -eq 239 -and ([int]$Response[1]) -eq 187 -and ([int]$Response[2]) -eq 191) { - # Remove UTF8 BOM - $response = $response.Substring(3) + elseif (!([string]::IsNullOrEmpty($version))) { + $dots = ($version.ToCharArray() -eq '.').Count + if ($dots -lt 3) { + # avoid 14.1 returning 14.10, 14.11 etc. + $version = "$($version.TrimEnd('.'))." } - if (([int]$Response[0]) -eq 65279) { - # Remove Unicode BOM (PowerShell 7.4) - $response = $response.Substring(1) + $GetListUrl += "&prefix=$($Version)" + } + + $Artifacts = @() + $nextMarker = '' + $currentMarker = '' + $downloadAttempt = 1 + $downloadRetryAttempts = 10 + do { + if ($currentMarker -ne $nextMarker) + { + $currentMarker = $nextMarker + $downloadAttempt = 1 } - $enumerationResults = ([xml]$Response).EnumerationResults - - if ($enumerationResults.Blobs) { - if (($After) -or ($Before)) { - $artifacts += $enumerationResults.Blobs.Blob | % { - if ($after) { - $blobModifiedDate = [DateTime]::Parse($_.Properties."Last-Modified") - if ($before) { - if ($blobModifiedDate -lt $before -and $blobModifiedDate -gt $after) { + Write-Verbose "Download String $GetListUrl$nextMarker" + try + { + $Response = Invoke-RestMethod -UseBasicParsing -ContentType "application/json; charset=UTF8" -Uri "$GetListUrl$nextMarker" + if (([int]$Response[0]) -eq 239 -and ([int]$Response[1]) -eq 187 -and ([int]$Response[2]) -eq 191) { + # Remove UTF8 BOM + $response = $response.Substring(3) + } + if (([int]$Response[0]) -eq 65279) { + # Remove Unicode BOM (PowerShell 7.4) + $response = $response.Substring(1) + } + $enumerationResults = ([xml]$Response).EnumerationResults + + if ($enumerationResults.Blobs) { + if (($After) -or ($Before)) { + $artifacts += $enumerationResults.Blobs.Blob | % { + if ($after) { + $blobModifiedDate = [DateTime]::Parse($_.Properties."Last-Modified") + if ($before) { + if ($blobModifiedDate -lt $before -and $blobModifiedDate -gt $after) { + $_.Name + } + } + elseif ($blobModifiedDate -gt $after) { $_.Name } } - elseif ($blobModifiedDate -gt $after) { - $_.Name - } - } - else { - $blobModifiedDate = [DateTime]::Parse($_.Properties."Last-Modified") - if ($blobModifiedDate -lt $before) { - $_.Name + else { + $blobModifiedDate = [DateTime]::Parse($_.Properties."Last-Modified") + if ($blobModifiedDate -lt $before) { + $_.Name + } } } } + else { + $artifacts += $enumerationResults.Blobs.Blob.Name + } } - else { - $artifacts += $enumerationResults.Blobs.Blob.Name + $nextMarker = $enumerationResults.NextMarker + if ($nextMarker) { + $nextMarker = "&marker=$nextMarker" } } - $nextMarker = $enumerationResults.NextMarker - if ($nextMarker) { - $nextMarker = "&marker=$nextMarker" - } - } - catch - { - $downloadAttempt += 1 - Write-Host "Error querying artifacts. Error message was $($_.Exception.Message)" - Write-Host - - if ($downloadAttempt -le $downloadRetryAttempts) + catch { - Write-Host "Repeating download attempt (" $downloadAttempt.ToString() " of " $downloadRetryAttempts.ToString() ")..." + $downloadAttempt += 1 + Write-Host "Error querying artifacts. Error message was $($_.Exception.Message)" Write-Host + + if ($downloadAttempt -le $downloadRetryAttempts) + { + Write-Host "Repeating download attempt (" $downloadAttempt.ToString() " of " $downloadRetryAttempts.ToString() ")..." + Write-Host + } + else + { + throw + } } - else - { - throw - } - } - } while ($nextMarker) + } while ($nextMarker) - if (!([string]::IsNullOrEmpty($country))) { - # avoid confusion between base and se - $countryArtifacts = $Artifacts | Where-Object { $_.EndsWith("/$country", [System.StringComparison]::InvariantCultureIgnoreCase) -and ($doNotCheckPlatform -or ($Artifacts.Contains("$($_.Split('/')[0])/platform"))) } - if (!$countryArtifacts) { - if (($type -eq "sandbox") -and ($bcContainerHelperConfig.mapCountryCode.PSObject.Properties.Name -eq $country)) { - $country = $bcContainerHelperConfig.mapCountryCode."$country" - $countryArtifacts = $Artifacts | Where-Object { $_.EndsWith("/$country", [System.StringComparison]::InvariantCultureIgnoreCase) -and ($doNotCheckPlatform -or ($Artifacts.Contains("$($_.Split('/')[0])/platform"))) } + if (!([string]::IsNullOrEmpty($country))) { + # avoid confusion between base and se + $countryArtifacts = $Artifacts | Where-Object { $_.EndsWith("/$country", [System.StringComparison]::InvariantCultureIgnoreCase) -and ($doNotCheckPlatform -or ($Artifacts.Contains("$($_.Split('/')[0])/platform"))) } + if (!$countryArtifacts) { + if (($type -eq "sandbox") -and ($bcContainerHelperConfig.mapCountryCode.PSObject.Properties.Name -eq $country)) { + $country = $bcContainerHelperConfig.mapCountryCode."$country" + $countryArtifacts = $Artifacts | Where-Object { $_.EndsWith("/$country", [System.StringComparison]::InvariantCultureIgnoreCase) -and ($doNotCheckPlatform -or ($Artifacts.Contains("$($_.Split('/')[0])/platform"))) } + } } + $Artifacts = $countryArtifacts } - $Artifacts = $countryArtifacts - } - else { - $Artifacts = $Artifacts | Where-Object { !($_.EndsWith("/platform", [System.StringComparison]::InvariantCultureIgnoreCase)) } - } - - switch ($Select) { - 'All' { - $Artifacts = $Artifacts | - Sort-Object { [Version]($_.Split('/')[0]) } - } - 'Latest' { - $Artifacts = $Artifacts | - Sort-Object { [Version]($_.Split('/')[0]) } | - Select-Object -Last 1 - } - 'First' { - $Artifacts = $Artifacts | - Sort-Object { [Version]($_.Split('/')[0]) } | - Select-Object -First 1 + else { + $Artifacts = $Artifacts | Where-Object { !($_.EndsWith("/platform", [System.StringComparison]::InvariantCultureIgnoreCase)) } } - 'SecondToLastMajor' { - $Artifacts = $Artifacts | - Sort-Object -Descending { [Version]($_.Split('/')[0]) } - $latest = $Artifacts | Select-Object -First 1 - if ($latest) { - $latestversion = [Version]($latest.Split('/')[0]) + + switch ($Select) { + 'All' { + $Artifacts = $Artifacts | + Sort-Object { [Version]($_.Split('/')[0]) } + } + 'Latest' { + $Artifacts = $Artifacts | + Sort-Object { [Version]($_.Split('/')[0]) } | + Select-Object -Last 1 + } + 'First' { $Artifacts = $Artifacts | - Where-Object { ([Version]($_.Split('/')[0])).Major -ne $latestversion.Major } | + Sort-Object { [Version]($_.Split('/')[0]) } | Select-Object -First 1 } - else { - $Artifacts = @() + 'SecondToLastMajor' { + $Artifacts = $Artifacts | + Sort-Object -Descending { [Version]($_.Split('/')[0]) } + $latest = $Artifacts | Select-Object -First 1 + if ($latest) { + $latestversion = [Version]($latest.Split('/')[0]) + $Artifacts = $Artifacts | + Where-Object { ([Version]($_.Split('/')[0])).Major -ne $latestversion.Major } | + Select-Object -First 1 + } + else { + $Artifacts = @() + } } - } - 'Closest' { - $Artifacts = $Artifacts | - Sort-Object { [Version]($_.Split('/')[0]) } - $closest = $Artifacts | - Where-Object { [Version]($_.Split('/')[0]) -ge $closestToVersion } | - Select-Object -First 1 - if (-not $closest) { - $closest = $Artifacts | Select-Object -Last 1 + 'Closest' { + $Artifacts = $Artifacts | + Sort-Object { [Version]($_.Split('/')[0]) } + $closest = $Artifacts | + Where-Object { [Version]($_.Split('/')[0]) -ge $closestToVersion } | + Select-Object -First 1 + if (-not $closest) { + $closest = $Artifacts | Select-Object -Last 1 + } + $Artifacts = $closest } - $Artifacts = $closest } - } - foreach ($Artifact in $Artifacts) { - "$BaseUrl$($Artifact)$sasToken" + foreach ($Artifact in $Artifacts) { + "$BaseUrl$($Artifact)$sasToken" + } } } } diff --git a/BC.HelperFunctions.ps1 b/BC.HelperFunctions.ps1 index 6f88c4e9e..3c879064e 100644 --- a/BC.HelperFunctions.ps1 +++ b/BC.HelperFunctions.ps1 @@ -118,6 +118,7 @@ function Get-ContainerHelperConfig { "IsGitHubActions" = ($env:GITHUB_ACTIONS -eq "true") "IsAzureDevOps" = ($env:TF_BUILD -eq "true") "IsGitLab" = ($env:GITLAB_CI -eq "true") + "useApproximateVersion" = $false } if ($isInsider) { diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index 2bb34e8d4..94356be76 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -1,3 +1,6 @@ +6.0.28 +Increase performance of Get-BcArtifactUrl when selecting latest artifact, by using an approximate filtering (set useApproximateVersion to false in settings to disable) + 6.0.27 Issue 3538 Compile-AppWithBcCompilerFolder fails when dependency does propagateDependencies Issue 3727 Regression - Release pipelines failing with SaaS environments due BcAuthContext diff --git a/Version.txt b/Version.txt index 365186458..1180d1b51 100644 --- a/Version.txt +++ b/Version.txt @@ -1 +1 @@ -6.0.27-dev \ No newline at end of file +6.0.28-dev \ No newline at end of file