diff --git a/Artifacts/Get-BCArtifactUrl.ps1 b/Artifacts/Get-BCArtifactUrl.ps1 index 3b0d7437b..cf13ed698 100644 --- a/Artifacts/Get-BCArtifactUrl.ps1 +++ b/Artifacts/Get-BCArtifactUrl.ps1 @@ -141,172 +141,206 @@ try { if ($storageAccount -eq 'bcinsider.blob.core.windows.net' -and !$accept_insiderEULA) { throw "You need to accept the insider EULA (https://go.microsoft.com/fwlink/?linkid=2245051) by specifying -accept_insiderEula or by providing a SAS token to get access to insider builds" } - $GetListUrl = "https://$storageAccount/$($Type.ToLowerInvariant())/?comp=list&restype=container" - - $upMajorFilter = '' - $upVersionFilter = '' - if ($select -eq 'SecondToLastMajor') { - if ($version) { - throw "You cannot specify a version when asking for the Second To Last Major version" - } - } - elseif ($select -eq 'Closest') { - if (!($version)) { - throw "You must specify a version number when you want to get the closest artifact Url" + + if ($type -eq 'sandbox' -and $storageAccount -eq 'bcartifacts.blob.core.windows.net' -and $select -eq 'latest' -and $version -eq '' -and $bcContainerHelperConfig.useApproximateVersion) { + # Temp fix / hack for Get-BcArtifact performance + # If Microsoft changes versioning schema, this needs to change (or useApproximateVersion should be set to false) + $now = ([DateTime]::Now).AddDays(15) + $approximateMajor = 23+2*($now.Year-2024)+($now.Month -ge 4)+($now.Month -ge 10) + $approximateMinor = ($now.Month + 2)%6 + $artifactUrl = Get-BCArtifactUrl -country $country -version "$approximateMajor.$approximateMinor" -select Latest -doNotCheckPlatform:$doNotCheckPlatform + if ($artifactUrl) { + # We found an artifact - check if it is the latest + while ($artifactUrl) { + $lastGoodArtifact = $artifactUrl + if ($approximateMinor -eq 5) { + $approximateMajor += 1 + $approximateMinor = 0 + } + else { + $approximateMinor += 1 + } + $artifactUrl = Get-BCArtifactUrl -country $country -version "$approximateMajor.$approximateMinor" -select Latest -doNotCheckPlatform:$doNotCheckPlatform + } + $artifactUrl = $lastGoodArtifact } - $dots = ($version.ToCharArray() -eq '.').Count - $closestToVersion = [Version]"0.0.0.0" - if ($dots -ne 3 -or !([Version]::TryParse($version, [ref] $closestToVersion))) { - throw "Version number must be in the format 1.2.3.4 when you want to get the closest artifact Url" + else { + # No artifact found - try previous 3 versions (else give up - maybe country is unavailable) + $tryVersions = 3 + while (-not $artifactUrl -and $tryVersions-- -gt 0) { + if ($approximateMinor -eq 0) { + $approximateMajor -= 1 + $approximateMinor = 5 + } + else { + $approximateMinor -= 1 + } + $artifactUrl = Get-BCArtifactUrl -country $country -version "$approximateMajor.$approximateMinor" -select Latest -doNotCheckPlatform:$doNotCheckPlatform + } } - $GetListUrl += "&prefix=$($closestToVersion.Major).$($closestToVersion.Minor)." - $upMajorFilter = "$($closestToVersion.Major)" - $upVersionFilter = "$($closestToVersion.Minor)." + $artifactUrl } - elseif (!([string]::IsNullOrEmpty($version))) { - $dots = ($version.ToCharArray() -eq '.').Count - if ($dots -lt 3) { - # avoid 14.1 returning 14.10, 14.11 etc. - $version = "$($version.TrimEnd('.'))." + else { + $GetListUrl = "https://$storageAccount/$($Type.ToLowerInvariant())/?comp=list&restype=container" + if ($select -eq 'SecondToLastMajor') { + if ($version) { + throw "You cannot specify a version when asking for the Second To Last Major version" + } } - $GetListUrl += "&prefix=$($Version)" - $upMajorFilter = $version.Split('.')[0] - $upVersionFilter = $version.Substring($version.Length).TrimStart('.') - } - - $Artifacts = @() - $nextMarker = '' - $currentMarker = '' - $downloadAttempt = 1 - $downloadRetryAttempts = 10 - do { - if ($currentMarker -ne $nextMarker) - { - $currentMarker = $nextMarker - $downloadAttempt = 1 + elseif ($select -eq 'Closest') { + if (!($version)) { + throw "You must specify a version number when you want to get the closest artifact Url" + } + $dots = ($version.ToCharArray() -eq '.').Count + $closestToVersion = [Version]"0.0.0.0" + if ($dots -ne 3 -or !([Version]::TryParse($version, [ref] $closestToVersion))) { + throw "Version number must be in the format 1.2.3.4 when you want to get the closest artifact Url" + } + $GetListUrl += "&prefix=$($closestToVersion.Major).$($closestToVersion.Minor)." } - Write-Verbose "Download String $GetListUrl$nextMarker" - try - { - $Response = Invoke-RestMethod -UseBasicParsing -ContentType "application/json; charset=UTF8" -Uri "$GetListUrl$nextMarker" - if (([int]$Response[0]) -eq 239 -and ([int]$Response[1]) -eq 187 -and ([int]$Response[2]) -eq 191) { - # Remove UTF8 BOM - $response = $response.Substring(3) + elseif (!([string]::IsNullOrEmpty($version))) { + $dots = ($version.ToCharArray() -eq '.').Count + if ($dots -lt 3) { + # avoid 14.1 returning 14.10, 14.11 etc. + $version = "$($version.TrimEnd('.'))." } - if (([int]$Response[0]) -eq 65279) { - # Remove Unicode BOM (PowerShell 7.4) - $response = $response.Substring(1) + $GetListUrl += "&prefix=$($Version)" + } + + $Artifacts = @() + $nextMarker = '' + $currentMarker = '' + $downloadAttempt = 1 + $downloadRetryAttempts = 10 + do { + if ($currentMarker -ne $nextMarker) + { + $currentMarker = $nextMarker + $downloadAttempt = 1 } - $enumerationResults = ([xml]$Response).EnumerationResults - - if ($enumerationResults.Blobs) { - if (($After) -or ($Before)) { - $artifacts += $enumerationResults.Blobs.Blob | % { - if ($after) { - $blobModifiedDate = [DateTime]::Parse($_.Properties."Last-Modified") - if ($before) { - if ($blobModifiedDate -lt $before -and $blobModifiedDate -gt $after) { + Write-Verbose "Download String $GetListUrl$nextMarker" + try + { + $Response = Invoke-RestMethod -UseBasicParsing -ContentType "application/json; charset=UTF8" -Uri "$GetListUrl$nextMarker" + if (([int]$Response[0]) -eq 239 -and ([int]$Response[1]) -eq 187 -and ([int]$Response[2]) -eq 191) { + # Remove UTF8 BOM + $response = $response.Substring(3) + } + if (([int]$Response[0]) -eq 65279) { + # Remove Unicode BOM (PowerShell 7.4) + $response = $response.Substring(1) + } + $enumerationResults = ([xml]$Response).EnumerationResults + + if ($enumerationResults.Blobs) { + if (($After) -or ($Before)) { + $artifacts += $enumerationResults.Blobs.Blob | % { + if ($after) { + $blobModifiedDate = [DateTime]::Parse($_.Properties."Last-Modified") + if ($before) { + if ($blobModifiedDate -lt $before -and $blobModifiedDate -gt $after) { + $_.Name + } + } + elseif ($blobModifiedDate -gt $after) { $_.Name } } - elseif ($blobModifiedDate -gt $after) { - $_.Name - } - } - else { - $blobModifiedDate = [DateTime]::Parse($_.Properties."Last-Modified") - if ($blobModifiedDate -lt $before) { - $_.Name + else { + $blobModifiedDate = [DateTime]::Parse($_.Properties."Last-Modified") + if ($blobModifiedDate -lt $before) { + $_.Name + } } } } + else { + $artifacts += $enumerationResults.Blobs.Blob.Name + } } - else { - $artifacts += $enumerationResults.Blobs.Blob.Name + $nextMarker = $enumerationResults.NextMarker + if ($nextMarker) { + $nextMarker = "&marker=$nextMarker" } } - $nextMarker = $enumerationResults.NextMarker - if ($nextMarker) { - $nextMarker = "&marker=$nextMarker" - } - } - catch - { - $downloadAttempt += 1 - Write-Host "Error querying artifacts. Error message was $($_.Exception.Message)" - Write-Host - - if ($downloadAttempt -le $downloadRetryAttempts) + catch { - Write-Host "Repeating download attempt (" $downloadAttempt.ToString() " of " $downloadRetryAttempts.ToString() ")..." + $downloadAttempt += 1 + Write-Host "Error querying artifacts. Error message was $($_.Exception.Message)" Write-Host + + if ($downloadAttempt -le $downloadRetryAttempts) + { + Write-Host "Repeating download attempt (" $downloadAttempt.ToString() " of " $downloadRetryAttempts.ToString() ")..." + Write-Host + } + else + { + throw + } } - else - { - throw - } - } - } while ($nextMarker) + } while ($nextMarker) - if (!([string]::IsNullOrEmpty($country))) { - # avoid confusion between base and se - $countryArtifacts = $Artifacts | Where-Object { $_.EndsWith("/$country", [System.StringComparison]::InvariantCultureIgnoreCase) -and ($doNotCheckPlatform -or ($Artifacts.Contains("$($_.Split('/')[0])/platform"))) } - if (!$countryArtifacts) { - if (($type -eq "sandbox") -and ($bcContainerHelperConfig.mapCountryCode.PSObject.Properties.Name -eq $country)) { - $country = $bcContainerHelperConfig.mapCountryCode."$country" - $countryArtifacts = $Artifacts | Where-Object { $_.EndsWith("/$country", [System.StringComparison]::InvariantCultureIgnoreCase) -and ($doNotCheckPlatform -or ($Artifacts.Contains("$($_.Split('/')[0])/platform"))) } + if (!([string]::IsNullOrEmpty($country))) { + # avoid confusion between base and se + $countryArtifacts = $Artifacts | Where-Object { $_.EndsWith("/$country", [System.StringComparison]::InvariantCultureIgnoreCase) -and ($doNotCheckPlatform -or ($Artifacts.Contains("$($_.Split('/')[0])/platform"))) } + if (!$countryArtifacts) { + if (($type -eq "sandbox") -and ($bcContainerHelperConfig.mapCountryCode.PSObject.Properties.Name -eq $country)) { + $country = $bcContainerHelperConfig.mapCountryCode."$country" + $countryArtifacts = $Artifacts | Where-Object { $_.EndsWith("/$country", [System.StringComparison]::InvariantCultureIgnoreCase) -and ($doNotCheckPlatform -or ($Artifacts.Contains("$($_.Split('/')[0])/platform"))) } + } } + $Artifacts = $countryArtifacts } - $Artifacts = $countryArtifacts - } - else { - $Artifacts = $Artifacts | Where-Object { !($_.EndsWith("/platform", [System.StringComparison]::InvariantCultureIgnoreCase)) } - } - - switch ($Select) { - 'All' { - $Artifacts = $Artifacts | - Sort-Object { [Version]($_.Split('/')[0]) } - } - 'Latest' { - $Artifacts = $Artifacts | - Sort-Object { [Version]($_.Split('/')[0]) } | - Select-Object -Last 1 - } - 'First' { - $Artifacts = $Artifacts | - Sort-Object { [Version]($_.Split('/')[0]) } | - Select-Object -First 1 + else { + $Artifacts = $Artifacts | Where-Object { !($_.EndsWith("/platform", [System.StringComparison]::InvariantCultureIgnoreCase)) } } - 'SecondToLastMajor' { - $Artifacts = $Artifacts | - Sort-Object -Descending { [Version]($_.Split('/')[0]) } - $latest = $Artifacts | Select-Object -First 1 - if ($latest) { - $latestversion = [Version]($latest.Split('/')[0]) + + switch ($Select) { + 'All' { + $Artifacts = $Artifacts | + Sort-Object { [Version]($_.Split('/')[0]) } + } + 'Latest' { + $Artifacts = $Artifacts | + Sort-Object { [Version]($_.Split('/')[0]) } | + Select-Object -Last 1 + } + 'First' { $Artifacts = $Artifacts | - Where-Object { ([Version]($_.Split('/')[0])).Major -ne $latestversion.Major } | + Sort-Object { [Version]($_.Split('/')[0]) } | Select-Object -First 1 } - else { - $Artifacts = @() + 'SecondToLastMajor' { + $Artifacts = $Artifacts | + Sort-Object -Descending { [Version]($_.Split('/')[0]) } + $latest = $Artifacts | Select-Object -First 1 + if ($latest) { + $latestversion = [Version]($latest.Split('/')[0]) + $Artifacts = $Artifacts | + Where-Object { ([Version]($_.Split('/')[0])).Major -ne $latestversion.Major } | + Select-Object -First 1 + } + else { + $Artifacts = @() + } } - } - 'Closest' { - $Artifacts = $Artifacts | - Sort-Object { [Version]($_.Split('/')[0]) } - $closest = $Artifacts | - Where-Object { [Version]($_.Split('/')[0]) -ge $closestToVersion } | - Select-Object -First 1 - if (-not $closest) { - $closest = $Artifacts | Select-Object -Last 1 + 'Closest' { + $Artifacts = $Artifacts | + Sort-Object { [Version]($_.Split('/')[0]) } + $closest = $Artifacts | + Where-Object { [Version]($_.Split('/')[0]) -ge $closestToVersion } | + Select-Object -First 1 + if (-not $closest) { + $closest = $Artifacts | Select-Object -Last 1 + } + $Artifacts = $closest } - $Artifacts = $closest } - } - foreach ($Artifact in $Artifacts) { - "$BaseUrl$($Artifact)$sasToken" + foreach ($Artifact in $Artifacts) { + "$BaseUrl$($Artifact)$sasToken" + } } } } diff --git a/BC.HelperFunctions.ps1 b/BC.HelperFunctions.ps1 index 6f88c4e9e..3c879064e 100644 --- a/BC.HelperFunctions.ps1 +++ b/BC.HelperFunctions.ps1 @@ -118,6 +118,7 @@ function Get-ContainerHelperConfig { "IsGitHubActions" = ($env:GITHUB_ACTIONS -eq "true") "IsAzureDevOps" = ($env:TF_BUILD -eq "true") "IsGitLab" = ($env:GITLAB_CI -eq "true") + "useApproximateVersion" = $false } if ($isInsider) { diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index 2bb34e8d4..94356be76 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -1,3 +1,6 @@ +6.0.28 +Increase performance of Get-BcArtifactUrl when selecting latest artifact, by using an approximate filtering (set useApproximateVersion to false in settings to disable) + 6.0.27 Issue 3538 Compile-AppWithBcCompilerFolder fails when dependency does propagateDependencies Issue 3727 Regression - Release pipelines failing with SaaS environments due BcAuthContext diff --git a/Version.txt b/Version.txt index 365186458..1180d1b51 100644 --- a/Version.txt +++ b/Version.txt @@ -1 +1 @@ -6.0.27-dev \ No newline at end of file +6.0.28-dev \ No newline at end of file