Increase Get-BcArtifactUrl Performance (#3739)

When looking for the latest version of the artifacts for a specific country, we need to fetch all artifacts in the blob storage in order to figure out what is latest. This fix uses an approximation of latest version number (actually the one shipping in a month) and trying to download that (or 2 prior versions) Additional improvements we are looking at for future optimizations: - Use OCI artifacts for artifacts storage instead of traditional blob storage - Restructure artifacts in more layers to allow for partial download of what you need in various scenarios - Create index if artifacts to allow for querying artifacts without reading tags/annotations --------- Co-authored-by: freddydk <[email protected]>
microsoft · Nov 4, 2024 · b7cd45a · b7cd45a
1 parent aaf98da
commit b7cd45a
Show file tree

Hide file tree

Showing 4 changed files with 175 additions and 137 deletions.
diff --git a/Artifacts/Get-BCArtifactUrl.ps1 b/Artifacts/Get-BCArtifactUrl.ps1
@@ -141,172 +141,206 @@ try {
         if ($storageAccount -eq 'bcinsider.blob.core.windows.net' -and !$accept_insiderEULA) {
             throw "You need to accept the insider EULA (https://go.microsoft.com/fwlink/?linkid=2245051) by specifying -accept_insiderEula or by providing a SAS token to get access to insider builds"
         }
-        $GetListUrl = "https://$storageAccount/$($Type.ToLowerInvariant())/?comp=list&restype=container"
-
-        $upMajorFilter = ''
-        $upVersionFilter = ''
-        if ($select -eq 'SecondToLastMajor') {
-            if ($version) {
-                throw "You cannot specify a version when asking for the Second To Last Major version"
-            }
-        }
-        elseif ($select -eq 'Closest') {
-            if (!($version)) {
-                throw "You must specify a version number when you want to get the closest artifact Url"
+
+        if ($type -eq 'sandbox' -and $storageAccount -eq 'bcartifacts.blob.core.windows.net' -and $select -eq 'latest' -and $version -eq '' -and $bcContainerHelperConfig.useApproximateVersion) {
+            # Temp fix / hack for Get-BcArtifact performance
+            # If Microsoft changes versioning schema, this needs to change (or useApproximateVersion should be set to false)
+            $now = ([DateTime]::Now).AddDays(15)
+            $approximateMajor = 23+2*($now.Year-2024)+($now.Month -ge 4)+($now.Month -ge 10)
+            $approximateMinor = ($now.Month + 2)%6
+            $artifactUrl = Get-BCArtifactUrl -country $country -version "$approximateMajor.$approximateMinor" -select Latest -doNotCheckPlatform:$doNotCheckPlatform
+            if ($artifactUrl) {
+                # We found an artifact - check if it is the latest
+                while ($artifactUrl) {
+                    $lastGoodArtifact = $artifactUrl
+                    if ($approximateMinor -eq 5) {
+                        $approximateMajor += 1
+                        $approximateMinor = 0
+                    }
+                    else {
+                        $approximateMinor += 1
+                    }
+                    $artifactUrl = Get-BCArtifactUrl -country $country -version "$approximateMajor.$approximateMinor" -select Latest -doNotCheckPlatform:$doNotCheckPlatform
+                }
+                $artifactUrl = $lastGoodArtifact
             }
-            $dots = ($version.ToCharArray() -eq '.').Count
-            $closestToVersion = [Version]"0.0.0.0"
-            if ($dots -ne 3 -or !([Version]::TryParse($version, [ref] $closestToVersion))) {
-                throw "Version number must be in the format 1.2.3.4 when you want to get the closest artifact Url"
+            else {
+                # No artifact found - try previous 3 versions (else give up - maybe country is unavailable)
+                $tryVersions = 3
+                while (-not $artifactUrl -and $tryVersions-- -gt 0) {
+                    if ($approximateMinor -eq 0) {
+                        $approximateMajor -= 1
+                        $approximateMinor = 5
+                    }
+                    else {
+                        $approximateMinor -= 1
+                    }
+                    $artifactUrl = Get-BCArtifactUrl -country $country -version "$approximateMajor.$approximateMinor" -select Latest -doNotCheckPlatform:$doNotCheckPlatform
+                }
             }
-            $GetListUrl += "&prefix=$($closestToVersion.Major).$($closestToVersion.Minor)."
-            $upMajorFilter = "$($closestToVersion.Major)"
-            $upVersionFilter = "$($closestToVersion.Minor)."
+            $artifactUrl
         }
-        elseif (!([string]::IsNullOrEmpty($version))) {
-            $dots = ($version.ToCharArray() -eq '.').Count
-            if ($dots -lt 3) {
-                # avoid 14.1 returning 14.10, 14.11 etc.
-                $version = "$($version.TrimEnd('.'))."
+        else {
+            $GetListUrl = "https://$storageAccount/$($Type.ToLowerInvariant())/?comp=list&restype=container"
+            if ($select -eq 'SecondToLastMajor') {
+                if ($version) {
+                    throw "You cannot specify a version when asking for the Second To Last Major version"
+                }
             }
-            $GetListUrl += "&prefix=$($Version)"
-            $upMajorFilter = $version.Split('.')[0]
-            $upVersionFilter = $version.Substring($version.Length).TrimStart('.')
-        }
-
-        $Artifacts = @()
-        $nextMarker = ''
-        $currentMarker = ''
-        $downloadAttempt = 1
-        $downloadRetryAttempts = 10
-        do {
-            if ($currentMarker -ne $nextMarker)
-            {
-                $currentMarker = $nextMarker
-                $downloadAttempt = 1
+            elseif ($select -eq 'Closest') {
+                if (!($version)) {
+                    throw "You must specify a version number when you want to get the closest artifact Url"
+                }
+                $dots = ($version.ToCharArray() -eq '.').Count
+                $closestToVersion = [Version]"0.0.0.0"
+                if ($dots -ne 3 -or !([Version]::TryParse($version, [ref] $closestToVersion))) {
+                    throw "Version number must be in the format 1.2.3.4 when you want to get the closest artifact Url"
+                }
+                $GetListUrl += "&prefix=$($closestToVersion.Major).$($closestToVersion.Minor)."
             }
-            Write-Verbose "Download String $GetListUrl$nextMarker"
-            try
-            {
-                $Response = Invoke-RestMethod -UseBasicParsing -ContentType "application/json; charset=UTF8" -Uri "$GetListUrl$nextMarker"
-                if (([int]$Response[0]) -eq 239 -and ([int]$Response[1]) -eq 187 -and ([int]$Response[2]) -eq 191) {
-                    # Remove UTF8 BOM
-                    $response = $response.Substring(3)
+            elseif (!([string]::IsNullOrEmpty($version))) {
+                $dots = ($version.ToCharArray() -eq '.').Count
+                if ($dots -lt 3) {
+                    # avoid 14.1 returning 14.10, 14.11 etc.
+                    $version = "$($version.TrimEnd('.'))."
                 }
-                if (([int]$Response[0]) -eq 65279) {
-                    # Remove Unicode BOM (PowerShell 7.4)
-                    $response = $response.Substring(1)
+                $GetListUrl += "&prefix=$($Version)"
+            }
+
+            $Artifacts = @()
+            $nextMarker = ''
+            $currentMarker = ''
+            $downloadAttempt = 1
+            $downloadRetryAttempts = 10
+            do {
+                if ($currentMarker -ne $nextMarker)
+                {
+                    $currentMarker = $nextMarker
+                    $downloadAttempt = 1
                 }
-                $enumerationResults = ([xml]$Response).EnumerationResults
-
-                if ($enumerationResults.Blobs) {
-                    if (($After) -or ($Before)) {
-                        $artifacts += $enumerationResults.Blobs.Blob | % {
-                            if ($after) {
-                                $blobModifiedDate = [DateTime]::Parse($_.Properties."Last-Modified")
-                                if ($before) {
-                                    if ($blobModifiedDate -lt $before -and $blobModifiedDate -gt $after) {
+                Write-Verbose "Download String $GetListUrl$nextMarker"
+                try
+                {
+                    $Response = Invoke-RestMethod -UseBasicParsing -ContentType "application/json; charset=UTF8" -Uri "$GetListUrl$nextMarker"
+                    if (([int]$Response[0]) -eq 239 -and ([int]$Response[1]) -eq 187 -and ([int]$Response[2]) -eq 191) {
+                        # Remove UTF8 BOM
+                        $response = $response.Substring(3)
+                    }
+                    if (([int]$Response[0]) -eq 65279) {
+                        # Remove Unicode BOM (PowerShell 7.4)
+                        $response = $response.Substring(1)
+                    }
+                    $enumerationResults = ([xml]$Response).EnumerationResults
+
+                    if ($enumerationResults.Blobs) {
+                        if (($After) -or ($Before)) {
+                            $artifacts += $enumerationResults.Blobs.Blob | % {
+                                if ($after) {
+                                    $blobModifiedDate = [DateTime]::Parse($_.Properties."Last-Modified")
+                                    if ($before) {
+                                        if ($blobModifiedDate -lt $before -and $blobModifiedDate -gt $after) {
+                                            $_.Name
+                                        }
+                                    }
+                                    elseif ($blobModifiedDate -gt $after) {
                                         $_.Name
                                     }
                                 }
-                                elseif ($blobModifiedDate -gt $after) {
-                                    $_.Name
-                                }
-                            }
-                            else {
-                                $blobModifiedDate = [DateTime]::Parse($_.Properties."Last-Modified")
-                                if ($blobModifiedDate -lt $before) {
-                                    $_.Name
+                                else {
+                                    $blobModifiedDate = [DateTime]::Parse($_.Properties."Last-Modified")
+                                    if ($blobModifiedDate -lt $before) {
+                                        $_.Name
+                                    }
                                 }
                             }
                         }
+                        else {
+                            $artifacts += $enumerationResults.Blobs.Blob.Name
+                        }
                     }
-                    else {
-                        $artifacts += $enumerationResults.Blobs.Blob.Name
+                    $nextMarker = $enumerationResults.NextMarker
+                    if ($nextMarker) {
+                        $nextMarker = "&marker=$nextMarker"
                     }
                 }
-                $nextMarker = $enumerationResults.NextMarker
-                if ($nextMarker) {
-                    $nextMarker = "&marker=$nextMarker"
-                }
-            }
-            catch
-            {
-                $downloadAttempt += 1
-                Write-Host "Error querying artifacts. Error message was $($_.Exception.Message)"
-                Write-Host
-
-                if ($downloadAttempt -le $downloadRetryAttempts)
+                catch
                 {
-                    Write-Host "Repeating download attempt (" $downloadAttempt.ToString() " of " $downloadRetryAttempts.ToString() ")..."
+                    $downloadAttempt += 1
+                    Write-Host "Error querying artifacts. Error message was $($_.Exception.Message)"
                     Write-Host
+
+                    if ($downloadAttempt -le $downloadRetryAttempts)
+                    {
+                        Write-Host "Repeating download attempt (" $downloadAttempt.ToString() " of " $downloadRetryAttempts.ToString() ")..."
+                        Write-Host
+                    }
+                    else
+                    {
+                        throw
+                    }                
                 }
-                else
-                {
-                    throw
-                }                
-            }
-        } while ($nextMarker)
+            } while ($nextMarker)
 
-        if (!([string]::IsNullOrEmpty($country))) {
-            # avoid confusion between base and se
-            $countryArtifacts = $Artifacts | Where-Object { $_.EndsWith("/$country", [System.StringComparison]::InvariantCultureIgnoreCase) -and ($doNotCheckPlatform -or ($Artifacts.Contains("$($_.Split('/')[0])/platform"))) }
-            if (!$countryArtifacts) {
-                if (($type -eq "sandbox") -and ($bcContainerHelperConfig.mapCountryCode.PSObject.Properties.Name -eq $country)) {
-                    $country = $bcContainerHelperConfig.mapCountryCode."$country"
-                    $countryArtifacts = $Artifacts | Where-Object { $_.EndsWith("/$country", [System.StringComparison]::InvariantCultureIgnoreCase) -and ($doNotCheckPlatform -or ($Artifacts.Contains("$($_.Split('/')[0])/platform"))) }
+            if (!([string]::IsNullOrEmpty($country))) {
+                # avoid confusion between base and se
+                $countryArtifacts = $Artifacts | Where-Object { $_.EndsWith("/$country", [System.StringComparison]::InvariantCultureIgnoreCase) -and ($doNotCheckPlatform -or ($Artifacts.Contains("$($_.Split('/')[0])/platform"))) }
+                if (!$countryArtifacts) {
+                    if (($type -eq "sandbox") -and ($bcContainerHelperConfig.mapCountryCode.PSObject.Properties.Name -eq $country)) {
+                        $country = $bcContainerHelperConfig.mapCountryCode."$country"
+                        $countryArtifacts = $Artifacts | Where-Object { $_.EndsWith("/$country", [System.StringComparison]::InvariantCultureIgnoreCase) -and ($doNotCheckPlatform -or ($Artifacts.Contains("$($_.Split('/')[0])/platform"))) }
+                    }
                 }
+                $Artifacts = $countryArtifacts
             }
-            $Artifacts = $countryArtifacts
-        }
-        else {
-            $Artifacts = $Artifacts | Where-Object { !($_.EndsWith("/platform", [System.StringComparison]::InvariantCultureIgnoreCase)) }
-        }
-
-        switch ($Select) {
-            'All' {  
-                $Artifacts = $Artifacts |
-                    Sort-Object { [Version]($_.Split('/')[0]) }
-            }
-            'Latest' { 
-                $Artifacts = $Artifacts |
-                    Sort-Object { [Version]($_.Split('/')[0]) } |
-                    Select-Object -Last 1
-            }
-            'First' { 
-                $Artifacts = $Artifacts |
-                    Sort-Object { [Version]($_.Split('/')[0]) } |
-                    Select-Object -First 1
+            else {
+                $Artifacts = $Artifacts | Where-Object { !($_.EndsWith("/platform", [System.StringComparison]::InvariantCultureIgnoreCase)) }
             }
-            'SecondToLastMajor' { 
-                $Artifacts = $Artifacts |
-                    Sort-Object -Descending { [Version]($_.Split('/')[0]) }
-                $latest = $Artifacts | Select-Object -First 1
-                if ($latest) {
-                    $latestversion = [Version]($latest.Split('/')[0])
+
+            switch ($Select) {
+                'All' {  
+                    $Artifacts = $Artifacts |
+                        Sort-Object { [Version]($_.Split('/')[0]) }
+                }
+                'Latest' { 
+                    $Artifacts = $Artifacts |
+                        Sort-Object { [Version]($_.Split('/')[0]) } |
+                        Select-Object -Last 1
+                }
+                'First' { 
                     $Artifacts = $Artifacts |
-                        Where-Object { ([Version]($_.Split('/')[0])).Major -ne $latestversion.Major } |
+                        Sort-Object { [Version]($_.Split('/')[0]) } |
                         Select-Object -First 1
                 }
-                else {
-                    $Artifacts = @()
+                'SecondToLastMajor' { 
+                    $Artifacts = $Artifacts |
+                        Sort-Object -Descending { [Version]($_.Split('/')[0]) }
+                    $latest = $Artifacts | Select-Object -First 1
+                    if ($latest) {
+                        $latestversion = [Version]($latest.Split('/')[0])
+                        $Artifacts = $Artifacts |
+                            Where-Object { ([Version]($_.Split('/')[0])).Major -ne $latestversion.Major } |
+                            Select-Object -First 1
+                    }
+                    else {
+                        $Artifacts = @()
+                    }
                 }
-            }
-            'Closest' {
-                $Artifacts = $Artifacts |
-                    Sort-Object { [Version]($_.Split('/')[0]) }
-                $closest = $Artifacts |
-                    Where-Object { [Version]($_.Split('/')[0]) -ge $closestToVersion } |
-                    Select-Object -First 1
-                if (-not $closest) {
-                    $closest = $Artifacts | Select-Object -Last 1
+                'Closest' {
+                    $Artifacts = $Artifacts |
+                        Sort-Object { [Version]($_.Split('/')[0]) }
+                    $closest = $Artifacts |
+                        Where-Object { [Version]($_.Split('/')[0]) -ge $closestToVersion } |
+                        Select-Object -First 1
+                    if (-not $closest) {
+                        $closest = $Artifacts | Select-Object -Last 1
+                    }
+                    $Artifacts = $closest           
                 }
-                $Artifacts = $closest           
             }
-        }
 
-        foreach ($Artifact in $Artifacts) {
-            "$BaseUrl$($Artifact)$sasToken"
+           foreach ($Artifact in $Artifacts) {
+                "$BaseUrl$($Artifact)$sasToken"
+            }
         }
     }
 }

diff --git a/BC.HelperFunctions.ps1 b/BC.HelperFunctions.ps1
@@ -118,6 +118,7 @@ function Get-ContainerHelperConfig {
             "IsGitHubActions" = ($env:GITHUB_ACTIONS -eq "true")
             "IsAzureDevOps" = ($env:TF_BUILD -eq "true")
             "IsGitLab" = ($env:GITLAB_CI -eq "true")
+            "useApproximateVersion" = $false
         }
 
         if ($isInsider) {

diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt
@@ -1,3 +1,6 @@
+6.0.28
+Increase performance of Get-BcArtifactUrl when selecting latest artifact, by using an approximate filtering (set useApproximateVersion to false in settings to disable)
+
 6.0.27
 Issue 3538 Compile-AppWithBcCompilerFolder fails when dependency does propagateDependencies
 Issue 3727 Regression - Release pipelines failing with SaaS environments due BcAuthContext

diff --git a/Version.txt b/Version.txt
@@ -1 +1 @@
-6.0.27-dev
+6.0.28-dev