Add mutli arch build and manifest for the operator repo

Signed-off-by: Mohamed Mahmoud <[email protected]>
bpfman · Jun 1, 2024 · 065706b · 065706b
1 parent 8abeb40
commit 065706b
Show file tree

Hide file tree

Showing 7 changed files with 157 additions and 38 deletions.
diff --git a/.github/workflows/image-build.yaml b/.github/workflows/image-build.yaml
@@ -17,12 +17,15 @@ jobs:
       id-token: write # needed for signing the images with GitHub OIDC Token
 
     runs-on: ubuntu-latest
+    env:
+      WF_BPFMAN_AGENT_IMG: quay.io/bpfman/bpfman-agent
+      WF_BPFMAN_OPERATOR_IMG: quay.io/bpfman/bpfman-operator
+      WF_MULTIARCH_TARGETS: amd64 arm64 ppc64le s390x
     strategy:
       fail-fast: false
       matrix:
         image:
           - registry: quay.io
-            build_language: go
             repository: bpfman
             image: bpfman-agent
             dockerfile: ./Containerfile.bpfman-agent
@@ -36,7 +39,6 @@ jobs:
               type=raw,value=latest,enable={{is_default_branch}}
 
           - registry: quay.io
-            build_language: go
             repository: bpfman
             image: bpfman-operator
             dockerfile: ./Containerfile.bpfman-operator
@@ -50,7 +52,6 @@ jobs:
               type=raw,value=latest,enable={{is_default_branch}}
 
           - registry: quay.io
-            build_language: go
             repository: bpfman
             image: bpfman-operator-bundle
             context: .
@@ -68,10 +69,9 @@ jobs:
       - uses: actions/checkout@v4
 
       - uses: actions/setup-go@v5
-        if: ${{ matrix.image.build_language == 'go' }}
         with:
           # prettier-ignore
-          go-version: '1.21' # yamllint disable-line rule:quoted-strings
+          go-version: '1.22' # yamllint disable-line rule:quoted-strings
 
       - uses: sigstore/[email protected]
 
@@ -109,7 +109,7 @@ jobs:
       - name: Push to registry
         id: push-image
         uses: redhat-actions/push-to-registry@v2
-        if: ${{ github.event_name == 'push' }}
+        if: ${{ github.event_name == 'push' && matrix.image.image != 'mutli-arch-images' }}
         with:
           tags: ${{ steps.meta.outputs.tags }}
 
@@ -120,3 +120,26 @@ jobs:
           for tag in ${tags[@]}; do
             cosign sign -y "${tag}@${{ steps.push-image.outputs.digest }}"
           done
+      - name: get short sha
+        run: |
+          echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+
+      - name: Build operator multi arch images
+        if: ${{ matrix.image.image == 'bpfman-operator' }}
+        run: |
+          BPFMAN_AGENT_IMG="${{ env.WF_BPFMAN_AGENT_IMG }}:${{ env.short_sha }}" BPFMAN_OPERATOR_IMG="${{ env.WF_BPFMAN_OPERATOR_IMG}}:${{ env.short_sha }}" MULTIARCH_TARGETS="${{ env.WF_MULTIARCH_TARGETS }}" make build-operator-images
+
+      - name: Push operator multi arch images and manifest
+        if: ${{ github.event_name == 'push' && matrix.image.image == 'bpfman-operator' }}
+        run: |
+          BPFMAN_AGENT_IMG="${{ env.WF_BPFMAN_AGENT_IMG }}:${{ env.short_sha }}" BPFMAN_OPERATOR_IMG="${{ env.WF_BPFMAN_OPERATOR_IMG}}:${{ env.short_sha }}" MULTIARCH_TARGETS="${{ env.WF_MULTIARCH_TARGETS }}" make operator-images
+
+      - name: Build agent multi arch images
+        if: ${{ matrix.image.image == 'bpfman-agent' }}
+        run: |
+          BPFMAN_AGENT_IMG="${{ env.WF_BPFMAN_AGENT_IMG }}:${{ env.short_sha }}" BPFMAN_OPERATOR_IMG="${{ env.WF_BPFMAN_OPERATOR_IMG}}:${{ env.short_sha }}" MULTIARCH_TARGETS="${{ env.WF_MULTIARCH_TARGETS }}" make build-agent-images
+
+      - name: Push agent multi arch images and manifest
+        if: ${{ github.event_name == 'push' && matrix.image.image == 'bpfman-agent' }}
+        run: |
+          BPFMAN_AGENT_IMG="${{ env.WF_BPFMAN_AGENT_IMG }}:${{ env.short_sha }}" BPFMAN_OPERATOR_IMG="${{ env.WF_BPFMAN_OPERATOR_IMG}}:${{ env.short_sha }}" MULTIARCH_TARGETS="${{ env.WF_MULTIARCH_TARGETS }}" make agent-images
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: ['1.21','1.22']
+        go: ['1.22']
     steps:
     - name: install make
       run: sudo apt-get install make
@@ -27,7 +27,7 @@ jobs:
     - name: check clean vendors
       run: go mod vendor
     - name: Report coverage
-      if: ${{ matrix.go == '1.21' }}
+      if: ${{ matrix.go == '1.22' }}
       uses: codecov/codecov-action@v4
       env:
         CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
@@ -41,10 +41,10 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: ['1.21']
+        go: ['1.22']
     env:
-      BPFMAN_AGENT_IMG: quay.io/bpfman/bpfman-agent:int-test
-      BPFMAN_OPERATOR_IMG: quay.io/bpfman/bpfman-operator:int-test
+      WF_BPFMAN_AGENT_IMG: quay.io/bpfman/bpfman-agent:int-test
+      WF_BPFMAN_OPERATOR_IMG: quay.io/bpfman/bpfman-operator:int-test
       XDP_PASS_PRIVATE_IMAGE_CREDS: ${{ secrets.XDP_PASS_PRIVATE_IMAGE_CREDS }}
     steps:
       - name: Check disk space
@@ -79,10 +79,12 @@ jobs:
           fetch-depth: 0
 
       - name: build images
-        run: make build-images
+        run: |
+          MULTIARCH_TARGETS="amd64" BPFMAN_AGENT_IMG="${{ env.WF_BPFMAN_AGENT_IMG }}" BPFMAN_OPERATOR_IMG="${{ env.WF_BPFMAN_OPERATOR_IMG}}" make build-images
 
       - name: run integration tests
-        run: make test-integration
+        run: |
+          BPFMAN_AGENT_IMG="${{ env.WF_BPFMAN_AGENT_IMG }}-amd64" BPFMAN_OPERATOR_IMG="${{ env.WF_BPFMAN_OPERATOR_IMG}}-amd64" make test-integration
 
       - name: Check disk space
         run: df -h

diff --git a/Containerfile.bpfman-agent b/Containerfile.bpfman-agent
@@ -1,7 +1,10 @@
 # Build the manager binary
-FROM golang:1.22 as bpfman-agent-build
+ARG TARGETPLATFORM=linux/amd64
+ARG BUILDPLATFORM=linux/amd64
+FROM --platform=$BUILDPLATFORM docker.io/library/golang:1.22 as bpfman-agent-build
 ARG TARGETOS
-ARG TARGETARCH
+ARG TARGETPLATFORM
+ARG TARGETARCH=amd64
 
 WORKDIR /usr/src/bpfman-operator
 
@@ -28,7 +31,7 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -mod ven
 
 # Use the fedora minimal image to reduce the size of the final image but still
 # be able to easily install extra packages.
-FROM quay.io/fedora/fedora-minimal
+FROM --platform=$TARGETPLATFORM quay.io/fedora/fedora-minimal
 ARG DNF_CMD="microdnf"
 
 # The full fedora image can be used for debugging purposes.  To use it, comment

diff --git a/Containerfile.bpfman-operator b/Containerfile.bpfman-operator
@@ -1,7 +1,10 @@
 # Build the manager binary
-FROM golang:1.22 as bpfman-operator-build
+ARG TARGETPLATFORM=linux/amd64
+ARG BUILDPLATFORM=linux/amd64
+FROM --platform=$BUILDPLATFORM docker.io/library/golang:1.22 as bpfman-operator-build
 ARG TARGETOS
-ARG TARGETARCH
+ARG TARGETPLATFORM
+ARG TARGETARCH=amd64
 
 WORKDIR /usr/src/bpfman-operator
 
@@ -32,7 +35,7 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -mod ven
 # The full fedora image can be used for debugging purposes, but you may need to
 # change "microdnf" to "dnf" below to install extra packages.
 # FROM fedora:38
-FROM quay.io/fedora/fedora-minimal
+FROM --platform=$TARGETPLATFORM quay.io/fedora/fedora-minimal
 ARG TARGETARCH
 WORKDIR /
 COPY --from=bpfman-operator-build /usr/src/bpfman-operator/config/bpfman-deployment/daemonset.yaml ./config/bpfman-deployment/daemonset.yaml

diff --git a/Makefile b/Makefile
@@ -4,6 +4,7 @@
 # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2)
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
 VERSION ?= 0.4.1
+MULTIARCH_TARGETS ?= amd64
 
 # CHANNELS define the bundle channels used in the bundle.
 # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")
@@ -45,12 +46,12 @@ USE_IMAGE_DIGESTS ?= false
 ifeq ($(USE_IMAGE_DIGESTS), true)
 	BUNDLE_GEN_FLAGS += --use-image-digests
 endif
-
+IMAGE_TAG ?= latest
 # Image URL to use all building/pushing image targets
-BPFMAN_IMG ?= quay.io/bpfman/bpfman:latest
-BPFMAN_AGENT_IMG ?= quay.io/bpfman/bpfman-agent:latest
-BPFMAN_OPERATOR_IMG ?= quay.io/bpfman/bpfman-operator:latest
-BPFMAN_OPERATOR_BUNDLE_IMG ?= quay.io/bpfman/bpfman-operator-bundle:latest
+BPFMAN_IMG ?= quay.io/bpfman/bpfman:$(IMAGE_TAG)
+BPFMAN_AGENT_IMG ?= quay.io/bpfman/bpfman-agent:$(IMAGE_TAG)
+BPFMAN_OPERATOR_IMG ?= quay.io/bpfman/bpfman-operator:$(IMAGE_TAG)
+BPFMAN_OPERATOR_BUNDLE_IMG ?= quay.io/bpfman/bpfman-operator-bundle:$(IMAGE_TAG)
 KIND_CLUSTER_NAME ?= bpfman-deployment
 
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
@@ -71,6 +72,23 @@ endif
 SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
 
+# Image building tool (docker / podman) - docker is preferred in CI
+OCI_BIN_PATH := $(shell which docker 2>/dev/null || which podman)
+OCI_BIN ?= $(shell basename ${OCI_BIN_PATH})
+
+# build a single arch target provided as argument
+define build_target
+	echo 'building $(1) for arch $(2)'; \
+	DOCKER_BUILDKIT=1 $(OCI_BIN) buildx build --load --build-arg TARGETPLATFORM=linux/$(2)\
+	 --build-arg TARGETARCH=$(2) --build-arg BUILDPLATFORM=linux/amd64 -t  $(1)-$(2) -f $(3) .;
+endef
+
+# push a single arch target image
+define push_target
+	echo 'pushing image $(1)-$(2)'; \
+	DOCKER_BUILDKIT=1 $(OCI_BIN) push $(1)-$(2);
+endef
+
 .PHONY: all
 all: build
 
@@ -126,7 +144,8 @@ OPERATOR_SDK_DL_URL=https://github.com/operator-framework/operator-sdk/releases/
 .PHONY: operator-sdk
 operator-sdk: $(OPERATOR_SDK)
 $(OPERATOR_SDK): $(LOCALBIN)
-	test -s $(LOCALBIN)/operator_sdk || { curl -LO ${OPERATOR_SDK_DL_URL} && chmod +x ${OPERATOR_SDK_DL_NAME} && mv ${OPERATOR_SDK_DL_NAME} $(LOCALBIN)/operator-sdk; }
+	test -s $(LOCALBIN)/operator_sdk || { curl -LO ${OPERATOR_SDK_DL_URL} && chmod +x ${OPERATOR_SDK_DL_NAME} &&\
+	 mv ${OPERATOR_SDK_DL_NAME} $(LOCALBIN)/operator-sdk; }
 
 .PHONY: controller-gen
 controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary.
@@ -219,15 +238,17 @@ generate-typed-clients: client-gen ## Generate typed client code
 
 .PHONY: generate-typed-listers
 generate-typed-listers: lister-gen ## Generate typed listers code
-	$(LISTER_GEN) "${PKG}/apis/v1alpha1" \
+	$(LISTER_GEN) \
+		"${PKG}/apis/v1alpha1" \
 		--output-pkg "${PKG}/pkg/client" \
 		--output-dir "./pkg/client" \
 		${COMMON_FLAGS}
 
 
 .PHONY: generate-typed-informers
 generate-typed-informers: informer-gen ## Generate typed informers code
-	$(INFORMER_GEN) "${PKG}/apis/v1alpha1" \
+	$(INFORMER_GEN) \
+		"${PKG}/apis/v1alpha1" \
 		--versioned-clientset-package "${PKG}/pkg/client/clientset" \
 		--listers-package "${PKG}/pkg/client" \
 		--output-pkg "${PKG}/pkg/client" \
@@ -279,7 +300,8 @@ test-integration: ## Run Integration tests.
 .PHONY: bundle
 bundle: operator-sdk generate kustomize manifests ## Generate bundle manifests and metadata, then validate generated files.
 	cd config/bpfman-operator-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-operator=${BPFMAN_OPERATOR_IMG}
-	cd config/bpfman-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman=${BPFMAN_IMG} && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-agent=${BPFMAN_AGENT_IMG}
+	cd config/bpfman-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman=${BPFMAN_IMG} &&\
+	 $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-agent=${BPFMAN_AGENT_IMG}
 	$(KUSTOMIZE) build config/manifests | $(OPERATOR_SDK) generate bundle $(BUNDLE_GEN_FLAGS)
 	cp config/manifests/dependencies.yaml bundle/metadata/
 	$(OPERATOR_SDK) bundle validate ./bundle
@@ -298,15 +320,79 @@ build: fmt ## Build bpfman-operator and bpfman-agent binaries.
 # If you wish built the manager image targeting other platforms you can use the --platform flag.
 # (i.e. docker build --platform linux/arm64 ). However, you must enable docker buildKit for it.
 # More info: https://docs.docker.com/develop/develop-images/build_enhancements/
+.PHONY: build-operator-images
+build-operator-images: ## Build bpfman-operator images.
+	trap 'exit' INT; \
+	$(foreach target,$(MULTIARCH_TARGETS),$(call build_target,${BPFMAN_OPERATOR_IMG},$(target),Containerfile.bpfman-operator))
+
+.PHONY: build-agent-images
+build-agent-images: ## Build bpfman-agent images.
+	trap 'exit' INT; \
+	$(foreach target,$(MULTIARCH_TARGETS),$(call build_target,${BPFMAN_AGENT_IMG},$(target),Containerfile.bpfman-agent))
+
 .PHONY: build-images
-build-images: ## Build bpfman, bpfman-agent, and bpfman-operator images.
-	docker build -t ${BPFMAN_OPERATOR_IMG} -f Containerfile.bpfman-operator ./
-	docker build -t ${BPFMAN_AGENT_IMG} -f Containerfile.bpfman-agent ./
+build-images: build-operator-images build-agent-images ## Build bpfman-agent, and bpfman-operator images.
+
+.PHONY: push-operator-images
+push-operator-images: ## Push bpfman-operator images.
+	trap 'exit' INT; \
+	$(foreach target,$(MULTIARCH_TARGETS),$(call push_target,${BPFMAN_OPERATOR_IMG},$(target)))
+
+.PHONY: push-agent-images
+push-agent-images: ## Push bpfman-agent images.
+	trap 'exit' INT; \
+	$(foreach target,$(MULTIARCH_TARGETS),$(call push_target,${BPFMAN_AGENT_IMG},$(target)))
+
 
 .PHONY: push-images
-push-images: ## Push bpfman-agent, bpfman-operator images.
-	docker push ${BPFMAN_OPERATOR_IMG}
-	docker push ${BPFMAN_AGENT_IMG}
+push-images: push-operator-images push-agent-images ## Push bpfman-agent, bpfman-operator images.
+
+.PHONY: manifest-operator-build
+manifest-operator-build: ## Build MULTIARCH_TARGETS manifest for bpfman-operator.
+	echo 'building manifest for $(BPFMAN_OPERATOR_IMG)'
+	DOCKER_BUILDKIT=1 $(OCI_BIN) rmi ${BPFMAN_OPERATOR_IMG} -f
+	DOCKER_BUILDKIT=1 $(OCI_BIN) manifest create ${BPFMAN_OPERATOR_IMG} \
+		$(foreach target,$(MULTIARCH_TARGETS), --amend ${BPFMAN_OPERATOR_IMG}-$(target));
+
+.PHONY: manifest-agent-build
+manifest-agent-build: ## Build MULTIARCH_TARGETS manifest for bpfman-agent.
+	echo 'building manifest for $(BPFMAN_AGENT_IMG)'
+	DOCKER_BUILDKIT=1 $(OCI_BIN) rmi ${BPFMAN_AGENT_IMG} -f
+	DOCKER_BUILDKIT=1 $(OCI_BIN) manifest create ${BPFMAN_AGENT_IMG} \
+		$(foreach target,$(MULTIARCH_TARGETS), --amend ${BPFMAN_AGENT_IMG}-$(target));
+
+.PHONY: manifest-build
+manifest-build: manifest-operator-build manifest-agent-build ## Build MULTIARCH_TARGETS manifest for bpfman-operator and bpfman-agent.
+
+.PHONY: manifest-operator-push
+manifest-operator-push: ## Push MULTIARCH_TARGETS manifest for bpfman-operator.
+	@echo 'publish manifest for $(BPFMAN_OPERATOR_IMG)'
+ifeq (${OCI_BIN}, docker)
+	DOCKER_BUILDKIT=1 $(OCI_BIN) manifest push ${BPFMAN_OPERATOR_IMG};
+else
+	DOCKER_BUILDKIT=1 $(OCI_BIN) manifest push ${BPFMAN_OPERATOR_IMG} docker://${BPFMAN_OPERATOR_IMG};
+endif
+
+.PHONY: manifest-agent-push
+manifest-agent-push: ## Push MULTIARCH_TARGETS manifest for bpfman-agent.
+	@echo 'publish manifest for $(BPFMAN_AGENT_IMG)'
+ifeq (${OCI_BIN}, docker)
+	DOCKER_BUILDKIT=1 $(OCI_BIN) manifest push ${BPFMAN_AGENT_IMG};
+else
+	DOCKER_BUILDKIT=1 $(OCI_BIN) manifest push ${BPFMAN_AGENT_IMG} docker://${BPFMAN_AGENT_IMG};
+endif
+
+.PHONY: manifest-push
+manifest-push: manifest-operator-push manifest-agent-push ## Push MULTIARCH_TARGETS manifest for bpfman-operator and bpfman-agent.
+
+.PHONY: operator-images
+operator-images: build-operator-images push-operator-images manifest-operator-build manifest-operator-push ## Build and push bpfman-operator images and manifest
+
+.PHONY: agent-images
+agent-images: build-agent-images push-agent-images manifest-agent-build manifest-agent-push ## Build and push bpfman-agent images and manifest
+
+.PHONY: images
+images: operator-images agent-images ## Build and push bpfman-agent, and bpfman-operator images and manifest
 
 .PHONY: load-images-kind
 load-images-kind: ## Load bpfman-agent, and bpfman-operator images into the running local kind devel cluster.
@@ -370,7 +456,8 @@ destroy-kind: ## Destroy Kind cluster
 .PHONY: deploy
 deploy: manifests kustomize ## Deploy bpfman-operator to the K8s cluster specified in ~/.kube/config with the csi driver initialized.
 	cd config/bpfman-operator-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-operator=${BPFMAN_OPERATOR_IMG}
-	cd config/bpfman-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman=${BPFMAN_IMG} && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-agent=${BPFMAN_AGENT_IMG}
+	cd config/bpfman-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman=${BPFMAN_IMG} && \
+	$(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-agent=${BPFMAN_AGENT_IMG}
 	$(KUSTOMIZE) build config/default | kubectl apply -f -
 
 .PHONY: undeploy
@@ -392,7 +479,8 @@ run-on-kind: kustomize setup-kind build-images load-images-kind deploy ## Kind D
 .PHONY: deploy-openshift
 deploy-openshift: manifests kustomize ## Deploy bpfman-operator to the Openshift cluster specified in ~/.kube/config.
 	cd config/bpfman-operator-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-operator=${BPFMAN_OPERATOR_IMG}
-	cd config/bpfman-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman=${BPFMAN_IMG} && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-agent=${BPFMAN_AGENT_IMG}
+	cd config/bpfman-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman=${BPFMAN_IMG} \
+	&& $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-agent=${BPFMAN_AGENT_IMG}
 	$(KUSTOMIZE) build config/openshift | kubectl apply -f -
 
 .PHONY: undeploy-openshift

diff --git a/config/test/kustomization.yaml b/config/test/kustomization.yaml
@@ -46,4 +46,4 @@ resources:
 images:
   - name: quay.io/bpfman/bpfman-operator
     newName: quay.io/bpfman/bpfman-operator
-    newTag: int-test
+    newTag: int-test-amd64
diff --git a/config/test/patch.yaml b/config/test/patch.yaml
@@ -4,7 +4,7 @@ metadata:
   name: config
   namespace: kube-system
 data:
-  bpfman.agent.image: quay.io/bpfman/bpfman-agent:int-test
+  bpfman.agent.image: quay.io/bpfman/bpfman-agent:int-test-amd64
   bpfman.image: quay.io/bpfman/bpfman:latest
   bpfman.log.level: bpfman=debug
   bpfman.agent.log.level: debug