use oidc (#233289)

* use oidc (#233126) * use oidc * undo dep bump * use ClientAssertionCredential * temporarily disable mangle * adopt oidc * move to new service connection, remove deprecated storage account upload (#233191) * use longer lived access tokens for cosmosdb auth (#233255)
microsoft · Nov 7, 2024 · 7d80f16 · 7d80f16
1 parent 5d458f8
commit 7d80f16
Show file tree

Hide file tree

Showing 28 changed files with 120 additions and 86 deletions.
diff --git a/build/azure-pipelines/alpine/product-build-alpine.yml b/build/azure-pipelines/alpine/product-build-alpine.yml
@@ -10,7 +10,7 @@ steps:
   - task: AzureKeyVault@2
     displayName: "Azure Key Vault: Get Secrets"
     inputs:
-      azureSubscription: "vscode-builds-subscription"
+      azureSubscription: vscode
       KeyVaultName: vscode-build-secrets
       SecretsFilter: "github-distro-mixin-password"
 
@@ -59,7 +59,7 @@ steps:
 
   - task: Docker@1
     inputs:
-      azureSubscriptionEndpoint: "vscode-builds-subscription"
+      azureSubscriptionEndpoint: vscode
       azureContainerRegistry: vscodehub.azurecr.io
       command: "Run an image"
       imageName: "vscode-linux-build-agent:alpine-$(VSCODE_ARCH)"

diff --git a/build/azure-pipelines/cli/cli-darwin-sign.yml b/build/azure-pipelines/cli/cli-darwin-sign.yml
@@ -7,7 +7,7 @@ steps:
   - task: AzureKeyVault@2
     displayName: "Azure Key Vault: Get Secrets"
     inputs:
-      azureSubscription: "vscode-builds-subscription"
+      azureSubscription: vscode
       KeyVaultName: vscode-build-secrets
       SecretsFilter: "ESRP-PKI,esrp-aad-username,esrp-aad-password"
 

diff --git a/build/azure-pipelines/cli/cli-win32-sign.yml b/build/azure-pipelines/cli/cli-win32-sign.yml
@@ -7,7 +7,7 @@ steps:
   - task: AzureKeyVault@2
     displayName: "Azure Key Vault: Get Secrets"
     inputs:
-      azureSubscription: "vscode-builds-subscription"
+      azureSubscription: vscode
       KeyVaultName: vscode-build-secrets
       SecretsFilter: "ESRP-PKI,esrp-aad-username,esrp-aad-password"
 

diff --git a/build/azure-pipelines/common/createBuild.js b/build/azure-pipelines/common/createBuild.js
diff --git a/build/azure-pipelines/common/createBuild.ts b/build/azure-pipelines/common/createBuild.ts
@@ -3,7 +3,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { ClientSecretCredential } from '@azure/identity';
+import { ClientAssertionCredential } from '@azure/identity';
 import { CosmosClient } from '@azure/cosmos';
 import { retry } from './retry';
 
@@ -47,7 +47,7 @@ async function main(): Promise<void> {
 		updates: {}
 	};
 
-	const aadCredentials = new ClientSecretCredential(process.env['AZURE_TENANT_ID']!, process.env['AZURE_CLIENT_ID']!, process.env['AZURE_CLIENT_SECRET']!);
+	const aadCredentials = new ClientAssertionCredential(process.env['AZURE_TENANT_ID']!, process.env['AZURE_CLIENT_ID']!, () => Promise.resolve(process.env['AZURE_ID_TOKEN']!));
 	const client = new CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT']!, aadCredentials });
 	const scripts = client.database('builds').container(quality).scripts;
 	await retry(() => scripts.storedProcedure('createBuild').execute('', [{ ...build, _partitionKey: '' }]));

diff --git a/build/azure-pipelines/common/publish.js b/build/azure-pipelines/common/publish.js
diff --git a/build/azure-pipelines/common/publish.ts b/build/azure-pipelines/common/publish.ts
@@ -47,6 +47,36 @@ class Temp {
 	}
 }
 
+/**
+ * Gets an access token converted from a WIF/OIDC id token.
+ * We need this since this build job takes a while to run and while id tokens live for 10 minutes only, access tokens live for 24 hours.
+ * Source: https://goodworkaround.com/2021/12/21/another-deep-dive-into-azure-ad-workload-identity-federation-using-github-actions/
+ */
+export async function getAccessToken(endpoint: string, tenantId: string, clientId: string, idToken: string): Promise<string> {
+	const body = new URLSearchParams({
+		scope: `${endpoint}.default`,
+		client_id: clientId,
+		grant_type: 'client_credentials',
+		client_assertion_type: 'urn:ietf:params:oauth:client-assertion-type:jwt-bearer',
+		client_assertion: encodeURIComponent(idToken)
+	});
+
+	const response = await fetch(`https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`, {
+		method: 'POST',
+		headers: {
+			'Content-Type': 'application/x-www-form-urlencoded'
+		},
+		body: body.toString()
+	});
+
+	if (!response.ok) {
+		throw new Error(`HTTP error! status: ${response.status}`);
+	}
+
+	const aadToken = await response.json();
+	return aadToken.access_token;
+}
+
 interface RequestOptions {
 	readonly body?: string;
 }
@@ -636,7 +666,7 @@ function getRealType(type: string) {
 	}
 }
 
-async function processArtifact(artifact: Artifact, artifactFilePath: string): Promise<void> {
+async function processArtifact(artifact: Artifact, artifactFilePath: string, cosmosDBAccessToken: string): Promise<void> {
 	const log = (...args: any[]) => console.log(`[${artifact.name}]`, ...args);
 	const match = /^vscode_(?<product>[^_]+)_(?<os>[^_]+)(?:_legacy)?_(?<arch>[^_]+)_(?<unprocessedType>[^_]+)$/.exec(artifact.name);
 
@@ -674,8 +704,7 @@ async function processArtifact(artifact: Artifact, artifactFilePath: string): Pr
 
 	await retry(async (attempt) => {
 		log(`Creating asset in Cosmos DB (attempt ${attempt})...`);
-		const aadCredentials = new ClientSecretCredential(e('AZURE_TENANT_ID'), e('AZURE_CLIENT_ID'), e('AZURE_CLIENT_SECRET'));
-		const client = new CosmosClient({ endpoint: e('AZURE_DOCUMENTDB_ENDPOINT'), aadCredentials });
+		const client = new CosmosClient({ endpoint: e('AZURE_DOCUMENTDB_ENDPOINT')!, tokenProvider: () => Promise.resolve(`type=aad&ver=1.0&sig=${cosmosDBAccessToken}`) });
 		const scripts = client.database('builds').container(quality).scripts;
 		await scripts.storedProcedure('createAsset').execute('', [commit, asset, true]);
 	});
@@ -691,8 +720,8 @@ async function processArtifact(artifact: Artifact, artifactFilePath: string): Pr
 // the CDN and finally update the build in Cosmos DB.
 async function main() {
 	if (!isMainThread) {
-		const { artifact, artifactFilePath } = workerData;
-		await processArtifact(artifact, artifactFilePath);
+		const { artifact, artifactFilePath, cosmosDBAccessToken } = workerData;
+		await processArtifact(artifact, artifactFilePath, cosmosDBAccessToken);
 		return;
 	}
 
@@ -713,6 +742,7 @@ async function main() {
 
 	let resultPromise = Promise.resolve<PromiseSettledResult<void>[]>([]);
 	const operations: { name: string; operation: Promise<void> }[] = [];
+	const cosmosDBAccessToken = await getAccessToken(e('AZURE_DOCUMENTDB_ENDPOINT')!, e('AZURE_TENANT_ID')!, e('AZURE_CLIENT_ID')!, e('AZURE_ID_TOKEN')!);
 
 	while (true) {
 		const [timeline, artifacts] = await Promise.all([retry(() => getPipelineTimeline()), retry(() => getPipelineArtifacts())]);
@@ -754,7 +784,7 @@ async function main() {
 
 			processing.add(artifact.name);
 			const promise = new Promise<void>((resolve, reject) => {
-				const worker = new Worker(__filename, { workerData: { artifact, artifactFilePath } });
+				const worker = new Worker(__filename, { workerData: { artifact, artifactFilePath, cosmosDBAccessToken } });
 				worker.on('error', reject);
 				worker.on('exit', code => {
 					if (code === 0) {

diff --git a/build/azure-pipelines/common/releaseBuild.js b/build/azure-pipelines/common/releaseBuild.js
diff --git a/build/azure-pipelines/common/releaseBuild.ts b/build/azure-pipelines/common/releaseBuild.ts
@@ -3,7 +3,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { ClientSecretCredential } from '@azure/identity';
+import { ClientAssertionCredential } from '@azure/identity';
 import { CosmosClient } from '@azure/cosmos';
 import { retry } from './retry';
 
@@ -45,7 +45,7 @@ async function main(force: boolean): Promise<void> {
 	const commit = getEnv('BUILD_SOURCEVERSION');
 	const quality = getEnv('VSCODE_QUALITY');
 
-	const aadCredentials = new ClientSecretCredential(process.env['AZURE_TENANT_ID']!, process.env['AZURE_CLIENT_ID']!, process.env['AZURE_CLIENT_SECRET']!);
+	const aadCredentials = new ClientAssertionCredential(process.env['AZURE_TENANT_ID']!, process.env['AZURE_CLIENT_ID']!, () => Promise.resolve(process.env['AZURE_ID_TOKEN']!));
 	const client = new CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT']!, aadCredentials });
 
 	if (!force) {

diff --git a/build/azure-pipelines/darwin/product-build-darwin-sign.yml b/build/azure-pipelines/darwin/product-build-darwin-sign.yml
@@ -16,7 +16,7 @@ steps:
   - task: AzureKeyVault@2
     displayName: "Azure Key Vault: Get Secrets"
     inputs:
-      azureSubscription: "vscode-builds-subscription"
+      azureSubscription: vscode
       KeyVaultName: vscode-build-secrets
       SecretsFilter: "ESRP-PKI,esrp-aad-username,esrp-aad-password"
 

diff --git a/build/azure-pipelines/darwin/product-build-darwin-universal.yml b/build/azure-pipelines/darwin/product-build-darwin-universal.yml
@@ -10,7 +10,7 @@ steps:
   - task: AzureKeyVault@2
     displayName: "Azure Key Vault: Get Secrets"
     inputs:
-      azureSubscription: "vscode-builds-subscription"
+      azureSubscription: vscode
       KeyVaultName: vscode-build-secrets
       SecretsFilter: "github-distro-mixin-password,macos-developer-certificate,macos-developer-certificate-key"
 

diff --git a/build/azure-pipelines/darwin/product-build-darwin.yml b/build/azure-pipelines/darwin/product-build-darwin.yml
@@ -28,7 +28,7 @@ steps:
     - task: AzureKeyVault@2
       displayName: "Azure Key Vault: Get Secrets"
       inputs:
-        azureSubscription: "vscode-builds-subscription"
+        azureSubscription: vscode
         KeyVaultName: vscode-build-secrets
         SecretsFilter: "github-distro-mixin-password,macos-developer-certificate,macos-developer-certificate-key"
 

diff --git a/build/azure-pipelines/distro/download-distro.yml b/build/azure-pipelines/distro/download-distro.yml
@@ -2,7 +2,7 @@ steps:
   - task: AzureKeyVault@2
     displayName: "Azure Key Vault: Get Secrets"
     inputs:
-      azureSubscription: "vscode-builds-subscription"
+      azureSubscription: vscode
       KeyVaultName: vscode-build-secrets
       SecretsFilter: "github-distro-mixin-password"
 

diff --git a/build/azure-pipelines/linux/product-build-linux-legacy-server.yml b/build/azure-pipelines/linux/product-build-linux-legacy-server.yml
@@ -18,7 +18,7 @@ steps:
   - task: AzureKeyVault@2
     displayName: "Azure Key Vault: Get Secrets"
     inputs:
-      azureSubscription: "vscode-builds-subscription"
+      azureSubscription: vscode
       KeyVaultName: vscode-build-secrets
       SecretsFilter: "github-distro-mixin-password"
 
@@ -77,7 +77,7 @@ steps:
     - task: Docker@1
       displayName: "Pull Docker image"
       inputs:
-        azureSubscriptionEndpoint: "vscode-builds-subscription"
+        azureSubscriptionEndpoint: vscode
         azureContainerRegistry: vscodehub.azurecr.io
         command: "Run an image"
         imageName: vscode-linux-build-agent:centos7-devtoolset8-$(VSCODE_ARCH)

diff --git a/build/azure-pipelines/linux/product-build-linux.yml b/build/azure-pipelines/linux/product-build-linux.yml
@@ -30,7 +30,7 @@ steps:
     - task: AzureKeyVault@2
       displayName: "Azure Key Vault: Get Secrets"
       inputs:
-        azureSubscription: "vscode-builds-subscription"
+        azureSubscription: vscode
         KeyVaultName: vscode-build-secrets
         SecretsFilter: "github-distro-mixin-password,ESRP-PKI,esrp-aad-username,esrp-aad-password"
 

diff --git a/build/azure-pipelines/product-build.yml b/build/azure-pipelines/product-build.yml
@@ -143,7 +143,7 @@ variables:
   - name: PRSS_PROVISION_TENANT_ID
     value: 72f988bf-86f1-41af-91ab-2d7cd011db47
   - name: AZURE_DOCUMENTDB_ENDPOINT
-    value: https://vscode.documents.azure.com:443/
+    value: https://vscode.documents.azure.com/
   - name: VSCODE_MIXIN_REPO
     value: microsoft/vscode-distro
   - name: skipComponentGovernanceDetection

diff --git a/build/azure-pipelines/product-compile.yml b/build/azure-pipelines/product-compile.yml
@@ -15,7 +15,7 @@ steps:
     - task: AzureKeyVault@2
       displayName: "Azure Key Vault: Get Secrets"
       inputs:
-        azureSubscription: "vscode-builds-subscription"
+        azureSubscription: vscode
         KeyVaultName: vscode-build-secrets
         SecretsFilter: "github-distro-mixin-password"
 
@@ -131,33 +131,24 @@ steps:
     - task: AzureCLI@2
       displayName: Fetch secrets
       inputs:
-        azureSubscription: "vscode-builds-subscription"
+        azureSubscription: vscode
         scriptType: pscore
         scriptLocation: inlineScript
         addSpnToEnvironment: true
         inlineScript: |
           Write-Host "##vso[task.setvariable variable=AZURE_TENANT_ID]$env:tenantId"
           Write-Host "##vso[task.setvariable variable=AZURE_CLIENT_ID]$env:servicePrincipalId"
-          Write-Host "##vso[task.setvariable variable=AZURE_CLIENT_SECRET;issecret=true]$env:servicePrincipalKey"
+          Write-Host "##vso[task.setvariable variable=AZURE_ID_TOKEN;issecret=true]$env:idToken"
 
     - script: |
         set -e
         AZURE_STORAGE_ACCOUNT="vscodeweb" \
         AZURE_TENANT_ID="$(AZURE_TENANT_ID)" \
         AZURE_CLIENT_ID="$(AZURE_CLIENT_ID)" \
-        AZURE_CLIENT_SECRET="$(AZURE_CLIENT_SECRET)" \
+        AZURE_ID_TOKEN="$(AZURE_ID_TOKEN)" \
           node build/azure-pipelines/upload-sourcemaps
       displayName: Upload sourcemaps to Azure
 
-    - script: |
-        set -e
-        AZURE_STORAGE_ACCOUNT="ticino" \
-        AZURE_TENANT_ID="$(AZURE_TENANT_ID)" \
-        AZURE_CLIENT_ID="$(AZURE_CLIENT_ID)" \
-        AZURE_CLIENT_SECRET="$(AZURE_CLIENT_SECRET)" \
-          node build/azure-pipelines/upload-sourcemaps
-      displayName: Upload sourcemaps to Azure (Deprecated)
-
     - script: ./build/azure-pipelines/common/extract-telemetry.sh
       displayName: Generate lists of telemetry events