Skip to content

Commit

Permalink
Migrate custom Key Vault resource to azidentity (#3664)
Browse files Browse the repository at this point in the history
This PR ended up having three distinct but related parts to it.

1. Migrate the custom KV resource to the new `azidentity` backend. We
need to preserve the previous one because it uses a special Autorest
authorizer for Key Vault.
2. Use a Key Vault secret in the azure-in-azure integration test. KV
secrets need a different authentication audience/scope in the access
token and we want to cover this case.
3. A fix that affects master as well: the azure-in-azure test didn't use
the correct environment variable for the new backend toggle.

[Green run of the azcore workflow using the new
backend](https://github.com/pulumi/pulumi-azure-native/actions/runs/11670925535)

Fixes #2432
  • Loading branch information
thomas11 authored Nov 5, 2024
1 parent 7060a50 commit b8ea73e
Show file tree
Hide file tree
Showing 10 changed files with 173 additions and 43 deletions.
18 changes: 10 additions & 8 deletions examples/azure-native-sdk-v2/go-azure-in-azure/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,10 +113,10 @@ func main() {
return err
}

// clientId is required for user-assigned identity to disambiguate between several identities.
var clientId pulumi.StringOutput = pulumi.String("").ToStringOutput()
vmIdentity := &compute.VirtualMachineIdentityArgs{Type: compute.ResourceIdentityTypeSystemAssigned}

var umi *managedidentity.UserAssignedIdentity
var umiClientId pulumi.StringOutput = pulumi.String("").ToStringOutput()
if os.Getenv("PULUMI_TEST_USER_IDENTITY") == "true" {
fmt.Printf("go-azure-in-azure: using user-assigned identity\n")

Expand All @@ -126,7 +126,7 @@ func main() {
if err != nil {
return err
}
umiClientId = umi.ClientId
clientId = umi.ClientId

// Create a second user-assigned identity to test multiple identities. With multiple identities, the one to
// use needs to be specified via clientId.
Expand Down Expand Up @@ -286,8 +286,9 @@ func main() {
}

// Pass feature flags into the VM.
useAutorest := os.Getenv("PULUMI_USE_AUTOREST")
useLegacyAuth := os.Getenv("PULUMI_USE_LEGACY_AUTH")
useAzcore := os.Getenv("PULUMI_ENABLE_AZCORE_BACKEND")

var tenantId pulumi.StringOutput = pulumi.String(os.Getenv("ARM_TENANT_ID")).ToStringOutput()

// We pass the resource group's ID into the inner program via config so the program can
// create a resource in the resource group.
Expand All @@ -297,19 +298,20 @@ export ARM_USE_MSI=true && \
export ARM_SUBSCRIPTION_ID=%s && \
export PATH="$HOME/.pulumi/bin:$PATH" && \
export PULUMI_CONFIG_PASSPHRASE=pass && \
export PULUMI_USE_AUTOREST=%s && \
export PULUMI_USE_LEGACY_AUTH=%s && \
export PULUMI_ENABLE_AZCORE_BACKEND=%s && \
rand=$(openssl rand -hex 4) && \
stackname="%s-$rand" && \
pulumi login --local && \
pulumi stack init $stackname && \
pulumi config set azure-native:clientId "%s" -s $stackname && \
pulumi config set azure-native:tenantId "%s" -s $stackname && \
pulumi config set objectId "%s" -s $stackname && \
pulumi config set rgId "%s" -s $stackname && \
pulumi config -s $stackname && \
pulumi up -s $stackname --skip-preview --logtostderr --logflow -v=9 && \
pulumi down -s $stackname --skip-preview --logtostderr --logflow -v=9 && \
pulumi stack rm --yes $stackname && \
pulumi logout --local`, innerProgram, clientConf.SubscriptionId, useAutorest, useLegacyAuth, innerProgram, umiClientId, rg.ID())
pulumi logout --local`, innerProgram, clientConf.SubscriptionId, useAzcore, innerProgram, clientId, tenantId, principalId, rg.ID())

pulumiPreview, err := remote.NewCommand(ctx, "pulumiUpDown", &remote.CommandArgs{
Connection: sshConn,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,43 @@ config:
azure-native:location: northeurope
rgId:
type: string
objectId:
type: string

resources:
resourceGroup:
type: azure-native:resources:ResourceGroup
get:
id: ${rgId}
storageaccount:
type: azure-native:storage:StorageAccount

keyVault:
type: azure-native:keyvault:Vault
properties:
resourceGroupName: ${resourceGroup.name}
location: ${resourceGroup.location}
properties:
sku:
family: A
name: standard
tenantId: ${azure-native:tenantId}
accessPolicies:
- tenantId: ${azure-native:tenantId}
objectId: ${objectId}
permissions:
secrets:
- get
- list
- set
- delete

secret:
type: azure-native:keyvault:Secret
properties:
resourceGroupName: ${resourceGroup.name}
kind: "StorageV2"
sku: { name: "Standard_LRS" }
vaultName: ${keyVault.name}
properties:
value: "MySecretValue"

outputs:
keyVaultUri: ${keyVault.properties.vaultUri}
secretId: ${secret.id}
4 changes: 4 additions & 0 deletions examples/examples_go_sdk_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,10 @@ func TestServicebusRecreateSdk(t *testing.T) {
}

func TestAzureInAzureWithSystemManagedIdentity(t *testing.T) {
// This test fails on the autorest backend, see #2432
if os.Getenv("PULUMI_ENABLE_AZCORE_BACKEND") != "true" {
t.Skip("Skipping test because azcore backend is not enabled")
}
test := getGoBaseOptionsSdk(t, testDir(t, "go-azure-in-azure"))
integration.ProgramTest(t, &test)
}
Expand Down
5 changes: 4 additions & 1 deletion provider/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.1.0
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.1.0
github.com/Azure/go-autorest/autorest v0.11.29
github.com/blang/semver v3.5.1+incompatible
github.com/brianvoe/gofakeit/v6 v6.16.0
Expand Down Expand Up @@ -47,6 +49,7 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.1 // indirect
github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
github.com/BurntSushi/toml v1.2.1 // indirect
Expand Down Expand Up @@ -155,7 +158,7 @@ require (
github.com/go-git/go-billy/v5 v5.5.0 // indirect
github.com/go-git/go-git/v5 v5.12.0 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/gofrs/uuid v4.2.0+incompatible // indirect
github.com/gofrs/uuid v4.4.0+incompatible // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.5.1 // indirect
github.com/golang/glog v1.2.0 // indirect
Expand Down
10 changes: 8 additions & 2 deletions provider/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,12 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0/go.mod h1:StGsLbuJh06Bd8IBfnAlIFV3fLb+gkczONWf15hpX2E=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE=
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.1.0 h1:DRiANoJTiW6obBQe3SqZizkuV1PEgfiiGivmVocDy64=
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.1.0/go.mod h1:qLIye2hwb/ZouqhpSD9Zn3SJipvpEnz1Ywl3VUk9Y0s=
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.1.0 h1:h4Zxgmi9oyZL2l8jeg1iRTqPloHktywWcu0nlJmo1tA=
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.1.0/go.mod h1:LgLGXawqSreJz135Elog0ywTJDsm0Hz2k+N+6ZK35u8=
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.1 h1:9fXQS/0TtQmKXp8SureKouF+idbQvp7cPUxykiohnBs=
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.1/go.mod h1:f+OaoSg0VQYPMqB0Jp2D54j1VHzITYcJaCNwV+k00ts=
github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
github.com/Azure/go-autorest/autorest v0.11.3/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
Expand Down Expand Up @@ -296,8 +302,8 @@ github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrK
github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0=
github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
Expand Down
18 changes: 4 additions & 14 deletions provider/pkg/provider/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
"github.com/pulumi/pulumi-azure-native/v2/provider/pkg/provider/crud"
"github.com/pulumi/pulumi-azure-native/v2/provider/pkg/resources"
"github.com/pulumi/pulumi-azure-native/v2/provider/pkg/resources/customresources"
"github.com/pulumi/pulumi-azure-native/v2/provider/pkg/util"
"github.com/pulumi/pulumi/pkg/v3/resource/provider"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin"
Expand Down Expand Up @@ -217,7 +218,7 @@ func (k *azureNativeProvider) Configure(ctx context.Context,
userAgent := k.getUserAgent()

var credential azcore.TokenCredential
if enableAzcoreBackend() {
if util.EnableAzcoreBackend() {
credential, err = k.newTokenCredential()
if err != nil {
return nil, fmt.Errorf("creating Pulumi auth credential: %w", err)
Expand Down Expand Up @@ -246,7 +247,7 @@ func (k *azureNativeProvider) Configure(ctx context.Context,
}

func (k *azureNativeProvider) newAzureClient(armAuth autorest.Authorizer, tokenCred azcore.TokenCredential, userAgent string) (azure.AzureClient, error) {
if enableAzcoreBackend() {
if util.EnableAzcoreBackend() {
logging.V(9).Infof("AzureClient: using azcore and azidentity")
return azure.NewAzCoreClient(tokenCred, userAgent, azure.GetCloudByName(k.environment.Name), nil)
}
Expand Down Expand Up @@ -366,7 +367,7 @@ func (k *azureNativeProvider) Invoke(ctx context.Context, req *rpc.InvokeRequest
func (k *azureNativeProvider) getClientToken(ctx context.Context, authConfig *authConfig, endpointArg resource.PropertyValue) (string, error) {
endpoint := k.tokenEndpoint(endpointArg)

if enableAzcoreBackend() {
if util.EnableAzcoreBackend() {
cred, err := k.newTokenCredential()
if err != nil {
return "", err
Expand Down Expand Up @@ -1611,14 +1612,3 @@ func (k *azureNativeProvider) autorestEnvToHamiltonEnv() environments.Environmen
return environments.Global
}
}

// enableAzcoreBackend is a feature toggle that returns true if the newer backend using azcore and
// azidentity for REST and authentication should be used. Otherwise, the previous autorest backend
// is used.
// Tracked in epic #3576, the new backend was added to upgrade from unmaintained libraries that
// don't receive security and other updates. It uses the latest official Azure packages.
// The new backend is gated behind this feature toggle to allow enabling it selectively,
// limiting the blast radius of regressions. It's enabled in the daily CI workflow azcore-scheduled.
func enableAzcoreBackend() bool {
return os.Getenv("PULUMI_ENABLE_AZCORE_BACKEND") == "true"
}
23 changes: 16 additions & 7 deletions provider/pkg/resources/customresources/custom_keyvault.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,15 @@ import (
"fmt"
"net/http"

"github.com/Azure/azure-sdk-for-go/services/keyvault/2016-10-01/keyvault"
"github.com/Azure/go-autorest/autorest"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys"
"github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets"
"github.com/pkg/errors"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
)

// keyVaultSecret creates a custom resource for Azure KeyVault Secret.
func keyVaultSecret(keyVaultDNSSuffix string, kvClient *keyvault.BaseClient) *CustomResource {
func keyVaultSecret(keyVaultDNSSuffix string, tokenCred azcore.TokenCredential) *CustomResource {
return &CustomResource{
path: "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.KeyVault/vaults/{vaultName}/secrets/{secretName}",
Delete: func(ctx context.Context, id string, properties resource.PropertyMap) error {
Expand All @@ -28,14 +29,18 @@ func keyVaultSecret(keyVaultDNSSuffix string, kvClient *keyvault.BaseClient) *Cu
}

vaultUrl := fmt.Sprintf("https://%s.%s", vaultName.StringValue(), keyVaultDNSSuffix)
_, err := kvClient.DeleteSecret(ctx, vaultUrl, secretName.StringValue())
kvClient, err := azsecrets.NewClient(vaultUrl, tokenCred, nil)
if err != nil {
return err
}
_, err = kvClient.DeleteSecret(ctx, secretName.StringValue(), nil)
return reportDeletionError(err)
},
}
}

// keyVaultKey creates a custom resource for Azure KeyVault Key.
func keyVaultKey(keyVaultDNSSuffix string, kvClient *keyvault.BaseClient) *CustomResource {
func keyVaultKey(keyVaultDNSSuffix string, tokenCred azcore.TokenCredential) *CustomResource {
return &CustomResource{
path: "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.KeyVault/vaults/{vaultName}/keys/{keyName}",
Delete: func(ctx context.Context, id string, properties resource.PropertyMap) error {
Expand All @@ -49,14 +54,18 @@ func keyVaultKey(keyVaultDNSSuffix string, kvClient *keyvault.BaseClient) *Custo
}

vaultUrl := fmt.Sprintf("https://%s.%s", vaultName.StringValue(), keyVaultDNSSuffix)
_, err := kvClient.DeleteKey(ctx, vaultUrl, keyName.StringValue())
kvClient, err := azkeys.NewClient(vaultUrl, tokenCred, nil)
if err != nil {
return err
}
_, err = kvClient.DeleteKey(ctx, keyName.StringValue(), nil)
return reportDeletionError(err)
},
}
}

func reportDeletionError(err error) error {
if detailed, ok := err.(autorest.DetailedError); ok && detailed.StatusCode == http.StatusNotFound {
if detailed, ok := err.(*azcore.ResponseError); ok && detailed.StatusCode == http.StatusNotFound {
return nil
}
return err
Expand Down
65 changes: 65 additions & 0 deletions provider/pkg/resources/customresources/custom_keyvault_autorest.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
// Copyright 2021, Pulumi Corporation. All rights reserved.

package customresources

import (
"context"
"fmt"
"net/http"

"github.com/Azure/azure-sdk-for-go/services/keyvault/2016-10-01/keyvault"
"github.com/Azure/go-autorest/autorest"
"github.com/pkg/errors"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
)

// keyVaultSecret_autorest creates a custom resource for Azure KeyVault Secrets, based on the
// deprecated autorest and go-azure-helpers backend.
func keyVaultSecret_autorest(keyVaultDNSSuffix string, kvClient *keyvault.BaseClient) *CustomResource {
return &CustomResource{
path: "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.KeyVault/vaults/{vaultName}/secrets/{secretName}",
Delete: func(ctx context.Context, id string, properties resource.PropertyMap) error {
vaultName := properties["vaultName"]
if !vaultName.HasValue() || !vaultName.IsString() {
return errors.New("vaultName not found in resource state")
}
secretName := properties["secretName"]
if !secretName.HasValue() || !secretName.IsString() {
return errors.New("secretName not found in resource state")
}

vaultUrl := fmt.Sprintf("https://%s.%s", vaultName.StringValue(), keyVaultDNSSuffix)
_, err := kvClient.DeleteSecret(ctx, vaultUrl, secretName.StringValue())
return reportDeletionError_autorest(err)
},
}
}

// keyVaultKey_autorest creates a custom resource for Azure KeyVault Keys, based on the
// deprecated autorest and go-azure-helpers backend.
func keyVaultKey_autorest(keyVaultDNSSuffix string, kvClient *keyvault.BaseClient) *CustomResource {
return &CustomResource{
path: "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.KeyVault/vaults/{vaultName}/keys/{keyName}",
Delete: func(ctx context.Context, id string, properties resource.PropertyMap) error {
vaultName := properties["vaultName"]
if !vaultName.HasValue() || !vaultName.IsString() {
return errors.New("vaultName not found in resource state")
}
keyName := properties["keyName"]
if !keyName.HasValue() || !keyName.IsString() {
return errors.New("keyName not found in resource state")
}

vaultUrl := fmt.Sprintf("https://%s.%s", vaultName.StringValue(), keyVaultDNSSuffix)
_, err := kvClient.DeleteKey(ctx, vaultUrl, keyName.StringValue())
return reportDeletionError_autorest(err)
},
}
}

func reportDeletionError_autorest(err error) error {
if detailed, ok := err.(autorest.DetailedError); ok && detailed.StatusCode == http.StatusNotFound {
return nil
}
return err
}
22 changes: 15 additions & 7 deletions provider/pkg/resources/customresources/customresources.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
azureEnv "github.com/Azure/go-autorest/autorest/azure"
"github.com/pulumi/pulumi-azure-native/v2/provider/pkg/azure"
"github.com/pulumi/pulumi-azure-native/v2/provider/pkg/provider/crud"
"github.com/pulumi/pulumi-azure-native/v2/provider/pkg/util"

. "github.com/pulumi/pulumi-azure-native/v2/provider/pkg/resources"
"github.com/pulumi/pulumi/pkg/v3/codegen"
Expand Down Expand Up @@ -161,10 +162,6 @@ func BuildCustomResources(env *azureEnv.Environment,
userAgent string,
tokenCred azcore.TokenCredential) (map[string]*CustomResource, error) {

kvClient := keyvault.New()
kvClient.Authorizer = kvBearerAuth
kvClient.UserAgent = userAgent

armKVClient, err := armkeyvault.NewVaultsClient(subscriptionID, tokenCred, &arm.ClientOptions{})
if err != nil {
return nil, err
Expand All @@ -184,9 +181,6 @@ func BuildCustomResources(env *azureEnv.Environment,
}

resources := []*CustomResource{
// Azure KeyVault resources.
keyVaultSecret(env.KeyVaultDNSSuffix, &kvClient),
keyVaultKey(env.KeyVaultDNSSuffix, &kvClient),
keyVaultAccessPolicy(armKVClient),
// Storage resources.
newStorageAccountStaticWebsite(env, &storageAccountsClient),
Expand All @@ -198,6 +192,20 @@ func BuildCustomResources(env *azureEnv.Environment,
customWebAppSlot,
}

// For Key Vault, we need to use separate token sources for azidentity and for the legacy auth. The
// `azCoreTokenCredential` adapter that we use elsewhere to translate legacy token sources to azidentity doesn't
// work here because KV needs a different token source for the KV endpoint.
if util.EnableAzcoreBackend() {
resources = append(resources, keyVaultSecret(env.KeyVaultDNSSuffix, tokenCred))
resources = append(resources, keyVaultKey(env.KeyVaultDNSSuffix, tokenCred))
} else {
kvClient := keyvault.New()
kvClient.Authorizer = kvBearerAuth
kvClient.UserAgent = userAgent
resources = append(resources, keyVaultSecret_autorest(env.KeyVaultDNSSuffix, &kvClient))
resources = append(resources, keyVaultKey_autorest(env.KeyVaultDNSSuffix, &kvClient))
}

result := map[string]*CustomResource{}
for _, r := range resources {
result[r.path] = r
Expand Down
Loading

0 comments on commit b8ea73e

Please sign in to comment.