From 9dad0fa312047afcacee265b5a62cda36d34d8a0 Mon Sep 17 00:00:00 2001 From: Karol Date: Mon, 30 Sep 2024 16:47:23 +0200 Subject: [PATCH] Add GCP Cloudrun scala-steward example --- .../gcp-cloudrun-scala-steward/.gitignore | 8 + .../GCPService.scala | 31 +++ .../GcpConfig.scala | 16 ++ .../GitAskPassFile.scala | 13 ++ .../GitConfig.scala | 17 ++ .../gcp-cloudrun-scala-steward/Main.scala | 185 ++++++++++++++++++ .../gcp-cloudrun-scala-steward/Pulumi.yaml | 3 + examples/gcp-cloudrun-scala-steward/README.md | 114 +++++++++++ .../gcp-cloudrun-scala-steward/project.scala | 5 + examples/gcp-cloudrun-scala-steward/repos.md | 8 + 10 files changed, 400 insertions(+) create mode 100644 examples/gcp-cloudrun-scala-steward/.gitignore create mode 100644 examples/gcp-cloudrun-scala-steward/GCPService.scala create mode 100644 examples/gcp-cloudrun-scala-steward/GcpConfig.scala create mode 100644 examples/gcp-cloudrun-scala-steward/GitAskPassFile.scala create mode 100644 examples/gcp-cloudrun-scala-steward/GitConfig.scala create mode 100644 examples/gcp-cloudrun-scala-steward/Main.scala create mode 100644 examples/gcp-cloudrun-scala-steward/Pulumi.yaml create mode 100644 examples/gcp-cloudrun-scala-steward/README.md create mode 100644 examples/gcp-cloudrun-scala-steward/project.scala create mode 100644 examples/gcp-cloudrun-scala-steward/repos.md diff --git a/examples/gcp-cloudrun-scala-steward/.gitignore b/examples/gcp-cloudrun-scala-steward/.gitignore new file mode 100644 index 00000000..4d8d3e40 --- /dev/null +++ b/examples/gcp-cloudrun-scala-steward/.gitignore @@ -0,0 +1,8 @@ +### Scala an JVM +*.class +*.log +.bsp +.scala-build + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* diff --git a/examples/gcp-cloudrun-scala-steward/GCPService.scala b/examples/gcp-cloudrun-scala-steward/GCPService.scala new file mode 100644 index 00000000..578a3cae --- /dev/null +++ b/examples/gcp-cloudrun-scala-steward/GCPService.scala @@ -0,0 +1,31 @@ +import besom.* +import besom.api.gcp +import gcp.projects.{Service, ServiceArgs} + +sealed trait GCPService(private val name: String): + def apply( + disableDependentServices: Input[Boolean] = true, + disableOnDestroy: Input[Boolean] = true + )(using Context): Output[Service] = + GCPService.projectService(s"$name.googleapis.com", disableDependentServices, disableOnDestroy) + +object GCPService: + case object CloudRun extends GCPService("run") + case object Scheduler extends GCPService("cloudscheduler") + case object SecretManager extends GCPService("secretmanager") + + private def projectService( + name: String, + disableDependentServices: Input[Boolean] = true, + disableOnDestroy: Input[Boolean] = true + )(using Context): Output[Service] = + Service( + s"enable-${name.replace(".", "-")}", + ServiceArgs( + service = name, + // if true - at every destroy this will disable the dependent services for the whole project + disableDependentServices = disableDependentServices, + // if true - at every destroy this will disable the service for the whole project + disableOnDestroy = disableOnDestroy + ) + ) diff --git a/examples/gcp-cloudrun-scala-steward/GcpConfig.scala b/examples/gcp-cloudrun-scala-steward/GcpConfig.scala new file mode 100644 index 00000000..30c55bc5 --- /dev/null +++ b/examples/gcp-cloudrun-scala-steward/GcpConfig.scala @@ -0,0 +1,16 @@ +import besom.* + +case class GcpConfig( + project: Output[String], + region: Output[String] +) +object GcpConfig: + extension (o: Output[GcpConfig]) + def project: Output[String] = o.flatMap(_.project) + def region: Output[String] = o.flatMap(_.region) + + def apply(using Context): GcpConfig = + GcpConfig( + project = config.requireString("gcp:project"), + region = config.requireString("gcp:region") + ) diff --git a/examples/gcp-cloudrun-scala-steward/GitAskPassFile.scala b/examples/gcp-cloudrun-scala-steward/GitAskPassFile.scala new file mode 100644 index 00000000..e4c0353f --- /dev/null +++ b/examples/gcp-cloudrun-scala-steward/GitAskPassFile.scala @@ -0,0 +1,13 @@ +object GitAskPassFile: + private val passFileName = "pass.sh" + private val gitPassFolderPath = "/opt/git" + val gitPassEnvName = "GIT_PASSWORD" + val gitPassPath = s"$gitPassFolderPath/$passFileName" + + val gitPassFile: String = + List( + s"mkdir $gitPassFolderPath", + s"echo '#!/bin/sh' >> $gitPassPath", + s"echo 'echo $$$gitPassEnvName' >> $gitPassPath", + s"chmod +x $gitPassPath" + ).mkString(" && ") diff --git a/examples/gcp-cloudrun-scala-steward/GitConfig.scala b/examples/gcp-cloudrun-scala-steward/GitConfig.scala new file mode 100644 index 00000000..54a9f511 --- /dev/null +++ b/examples/gcp-cloudrun-scala-steward/GitConfig.scala @@ -0,0 +1,17 @@ +import besom.* +import besom.json.* + +case class GitConfig( + forgeLogin: String, + forgeApiHost: String, + forgeType: String, + gitAuthorEmail: String, + password: String +) derives JsonFormat +object GitConfig: + extension (o: Output[GitConfig]) + def forgeLogin: Output[String] = o.map(_.forgeLogin) + def forgeApiHost: Output[String] = o.map(_.forgeApiHost) + def forgeType: Output[String] = o.map(_.forgeType) + def gitAuthorEmail: Output[String] = o.map(_.gitAuthorEmail) + def password: Output[String] = o.map(_.password) diff --git a/examples/gcp-cloudrun-scala-steward/Main.scala b/examples/gcp-cloudrun-scala-steward/Main.scala new file mode 100644 index 00000000..ded0cb78 --- /dev/null +++ b/examples/gcp-cloudrun-scala-steward/Main.scala @@ -0,0 +1,185 @@ +import besom.* +import besom.api.gcp +import besom.api.gcp.cloudrunv2.inputs.* +import besom.api.gcp.cloudrunv2.{Job, JobArgs, JobIamMember, JobIamMemberArgs} +import besom.api.gcp.cloudscheduler as csh +import besom.api.gcp.cloudscheduler.inputs.* +import besom.api.gcp.secretmanager.* +import besom.api.gcp.secretmanager.inputs.* +import besom.api.gcp.serviceaccount.{Account, AccountArgs} +import besom.api.gcp.storage.{Bucket, BucketArgs, BucketObject, BucketObjectArgs, BucketIamMember, BucketIamMemberArgs} + +@main def main = Pulumi.run { + val repoFileName = "repos.md" + val volumePath = "/opt/scala-steward" + val appName = "scala-steward" + val scalaStewardVersion = "latest" + val gitConfig = config.requireObject[GitConfig]("git") + val gcpConfig = GcpConfig.apply + + val serviceAccount = Account( + s"$appName-sa", + AccountArgs( + accountId = s"$appName-sa", + displayName = "Service Account for Scala Steward" + ) + ) + val serviceAccountMember = p"serviceAccount:${serviceAccount.email}" + + // Create a Cloud Storage bucket + val bucket = Bucket( + s"$appName-bucket", + BucketArgs( + location = "US", + forceDestroy = true + ) + ) + + // Grant the Cloud Run job service account permissions to access and delete objects in the bucket + val bucketIamMember = BucketIamMember( + s"$appName-bucket-access", + BucketIamMemberArgs( + bucket = bucket.name, + role = "roles/storage.objectAdmin", // This role includes permissions to delete objects + member = serviceAccountMember + ) + ) + + val reposObject = BucketObject( + s"$appName-repos", + BucketObjectArgs( + bucket = bucket.name, + name = repoFileName, + source = besom.Asset.FileAsset(s"./$repoFileName") + ) + ) + + val secret = Secret( + s"$appName-secret", + SecretArgs( + secretId = s"$appName-git-secret", + replication = SecretReplicationArgs( + auto = SecretReplicationAutoArgs() + ) + ), + opts(dependsOn = GCPService.SecretManager()) + ) + + val secretVersion = SecretVersion( + s"$appName-secret-version", + SecretVersionArgs( + secret = secret.name, + secretData = gitConfig.password + ) + ) + + val secretIamMember = SecretIamMember( + s"$appName-secret-access", + SecretIamMemberArgs( + secretId = secret.id, + role = "roles/secretmanager.secretAccessor", + member = serviceAccountMember + ), + opts(dependsOn = secret) + ) + + val scalaStewardRun = + List( + p"/opt/docker/bin/scala-steward", + p"--workspace $volumePath/workspace", + p"--repos-file $volumePath/${reposObject.name}", + p"--git-author-email ${gitConfig.gitAuthorEmail}", + p"--forge-type ${gitConfig.forgeType}", + p"--forge-api-host ${gitConfig.forgeApiHost}", + p"--forge-login ${gitConfig.forgeLogin}", + p"--git-ask-pass ${GitAskPassFile.gitPassPath}", + p"--do-not-fork" + ).sequence.map(_.mkString(" ")) + + // Define the Cloud Run Job + val cloudRunJob = Job( + s"$appName-job", + JobArgs( + location = gcpConfig.region, + template = JobTemplateArgs( + template = JobTemplateTemplateArgs( + serviceAccount = serviceAccount.email, + timeout = "2400s", // 40 min + maxRetries = 2, + containers = JobTemplateTemplateContainerArgs( + // Scala Steward Docker image + image = p"fthomas/scala-steward:$scalaStewardVersion", + envs = JobTemplateTemplateContainerEnvArgs( + name = GitAskPassFile.gitPassEnvName, + valueSource = JobTemplateTemplateContainerEnvValueSourceArgs( + secretKeyRef = JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs( + secret = secret.id, + version = secretVersion.version + ) + ) + ) :: Nil, + commands = List( + "/bin/sh", + "-c", + p"${GitAskPassFile.gitPassFile} && $scalaStewardRun" + ), + resources = JobTemplateTemplateContainerResourcesArgs( + limits = Map("cpu" -> "2", "memory" -> "8Gi") + ), + volumeMounts = JobTemplateTemplateContainerVolumeMountArgs( + name = "gcs-volume", + mountPath = volumePath + ) :: Nil + ) :: Nil, + volumes = JobTemplateTemplateVolumeArgs( + name = "gcs-volume", + gcs = JobTemplateTemplateVolumeGcsArgs( + bucket = bucket.name, + readOnly = false + ) + ) :: Nil + ) + ) + ), + opts(dependsOn = GCPService.CloudRun()) + ) + + // Grant the Cloud Scheduler service account permission to invoke the Cloud Run job + val jobIamMember = JobIamMember( + s"$appName-scheduler-invoker", + JobIamMemberArgs( + name = cloudRunJob.name, + role = "roles/run.invoker", + member = serviceAccountMember + ) + ) + + val schedulerUri = + p"https://${gcpConfig.region}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${gcpConfig.project}/jobs/${cloudRunJob.name}:run" + // Create a Cloud Scheduler job to trigger the Cloud Run job + val schedulerJob = csh.Job( + s"$appName-scheduler", + csh.JobArgs( + schedule = "0 12 * * *", + timeZone = "Etc/UTC", + httpTarget = JobHttpTargetArgs( + httpMethod = "POST", + uri = schedulerUri, + oauthToken = JobHttpTargetOauthTokenArgs( + serviceAccountEmail = serviceAccount.email + ) + ) + ), + opts(dependsOn = GCPService.Scheduler()) + ) + + Stack(bucketIamMember, secretIamMember, jobIamMember) + .exports( + jobName = cloudRunJob.name, + gitPasswordFile = GitAskPassFile.gitPassFile, + scalaStewardRunScript = scalaStewardRun, + schedulerJobName = schedulerJob.name, + secretName = secret.name, + secretVersionName = secretVersion.name + ) +} diff --git a/examples/gcp-cloudrun-scala-steward/Pulumi.yaml b/examples/gcp-cloudrun-scala-steward/Pulumi.yaml new file mode 100644 index 00000000..5bbe9ea0 --- /dev/null +++ b/examples/gcp-cloudrun-scala-steward/Pulumi.yaml @@ -0,0 +1,3 @@ +name: gcp-cloudrun-scala-steward +runtime: scala +description: Scala Steward on Google Cloud Run diff --git a/examples/gcp-cloudrun-scala-steward/README.md b/examples/gcp-cloudrun-scala-steward/README.md new file mode 100644 index 00000000..1021fbc9 --- /dev/null +++ b/examples/gcp-cloudrun-scala-steward/README.md @@ -0,0 +1,114 @@ +# Scala Steward on Google Cloud Run + +A Scala application that uses Google Cloud Run Job to +run [Scala Steward](https://github.com/scala-steward-org/scala-steward/tree/main) job with scheduler. + +## Prerequisites + +1. **Install Pulumi CLI**: + + To install the latest Pulumi release, run the following (see + [installation instructions](https://www.pulumi.com/docs/reference/install/) for additional installation options): + + ```bash + curl -fsSL https://get.pulumi.com/ | sh + ``` + +2. **Install Scala CLI**: + + To install the latest Scala CLI release, run the following (see + [installation instructions](https://scala-cli.virtuslab.org/install) for additional installation options): + + ```bash + curl -sSLf https://scala-cli.virtuslab.org/get | sh + ``` + +3. **Install Scala Language Plugin in Pulumi**: + + To install the latest Scala Language Plugin release, run the following: + + ```bash + pulumi plugin install language scala 0.3.2 --server github://api.github.com/VirtusLab/besom + ``` + If you not do this, you see this error:\ + `error: failed to load language plugin scala: no language plugin 'pulumi-language-scala' found in the workspace or on your $PATH` + + +4. [**Install Google CLI**](https://cloud.google.com/sdk/docs/install) + + +5. **Authenticate and configure GCP**: + + ```bash + gcloud auth application-default login + ``` + +6. **Create access token** and add it to `git:password` (see below) with scope: api, read_repository, write_repository.\ + [Example of creating access token on Gitlab](https://docs.gitlab.com/ee/user/project/settings/project_access_tokens.html) + +## Deploying and running the program + +1. Create a new stack: + + ```bash + pulumi stack init dev + ``` + +2. Set the required configuration variables: + + Set project name: + ```bash + pulumi config set gcp:project + ``` + Set region. Any valid GCP region here but preferred `us-east1, us-west1, or us-central1` because of [**Free Tier + **](https://cloud.google.com/free/docs/free-cloud-features#free-tier): + ```bash + pulumi config set gcp:region us-east1 + ``` + Set password to git repository. Recommended an authentication token: + ```bash + pulumi config set git:password --secret + ``` + Add below lines to `Pulumi.dev.yaml` file. Fill with proper values: + ```yaml + scala-steward:git: + forgeApiHost: + forgeLogin: + forgeType: + gitAuthorEmail: + password: + secure: # secret value from above command + ``` + You may delete config `git:password ` from `Pulumi.dev.yaml` created earlier. + + +3. Run `pulumi up` to preview and deploy changes. After the preview is shown you will be prompted if you want to + continue or not. + + ```bash + pulumi up + ``` + **Warning**: The first launch of the app may take longer, so you may need to change the cloud run job timeout. + Default is set to 40 min (`2400s`) + + +4. To see the output that was created, run: + + ```bash + pulumi stack output + ``` + +5. On GCP console check logs and see if scala-steward is working properly. + +6. To clean up resources, remove docker images, destroy your stack and remove it: + + ```bash + pulumi down + ``` + ```bash + pulumi stack rm dev + ``` + If you want to revoke your Application Default Credentials: + ```bash + gcloud auth application-default revoke + ``` \ No newline at end of file diff --git a/examples/gcp-cloudrun-scala-steward/project.scala b/examples/gcp-cloudrun-scala-steward/project.scala new file mode 100644 index 00000000..9164e5b4 --- /dev/null +++ b/examples/gcp-cloudrun-scala-steward/project.scala @@ -0,0 +1,5 @@ +//> using scala "3.3.1" +//> using options -Werror -Wunused:all -Wvalue-discard -Wnonunit-statement +//> using dep "org.virtuslab::besom-core:0.4.0-SNAPSHOT" +//> using dep "org.virtuslab::besom-gcp:7.26.0-core.0.3" +//> using repository sonatype:snapshots diff --git a/examples/gcp-cloudrun-scala-steward/repos.md b/examples/gcp-cloudrun-scala-steward/repos.md new file mode 100644 index 00000000..8eb63fac --- /dev/null +++ b/examples/gcp-cloudrun-scala-steward/repos.md @@ -0,0 +1,8 @@ +List the repositories here you want your own Scala Steward to keep up-to-date. +The format is "- $owner/$repo". +If you want Scala Steward to keep a non-default branch up-to-date, use "- $owner/$repo:$branch". +All lines that do not start with a hyphen and space are ignored. + +Example: + +- VirtusLab/besom \ No newline at end of file