Add mutli arch build and manifest for the operator repo

Signed-off-by: Mohamed Mahmoud <[email protected]>

.github/workflows/image-build.yaml

@@ -17,6 +17,10 @@ jobs:
       id-token: write # needed for signing the images with GitHub OIDC Token
 
     runs-on: ubuntu-latest
+    env:
+      WF_BPFMAN_AGENT_IMG: quay.io/bpfman/bpfman-agent
+      WF_BPFMAN_OPERATOR_IMG: quay.io/bpfman/bpfman-operator
+      WF_MULTIARCH_TARGETS: amd64 arm64 ppc64le s390x
     strategy:
       fail-fast: false
       matrix:
@@ -71,7 +75,7 @@ jobs:
         if: ${{ matrix.image.build_language == 'go' }}
         with:
           # prettier-ignore
-          go-version: '1.21' # yamllint disable-line rule:quoted-strings
+          go-version: '1.22' # yamllint disable-line rule:quoted-strings
 
       - uses: sigstore/[email protected]
 
@@ -120,3 +124,18 @@ jobs:
           for tag in ${tags[@]}; do
             cosign sign -y "${tag}@${{ steps.push-image.outputs.digest }}"
           done
+      - name: get short sha
+        run: |
+          echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+      - name: Build and Push multi arch images
+        run: |
+          IMAGE_TAG=${{ env.short_sha }}
+          BPFMAN_AGENT_IMG=${{ env.WF_BPFMAN_AGENT_IMG }}
+          BPFMAN_OPERATOR_IMG=${{ env.WF_BPFMAN_OPERATOR_IMG}}
+          MULTIARCH_TARGETS="${{ env.WF_MULTIARCH_TARGETS }}" make build-images push-images
+      - name: Build and Push manifest
+        run: |
+          IMAGE_TAG=${{ env.short_sha }}
+          BPFMAN_AGENT_IMG=${{ env.WF_BPFMAN_AGENT_IMG }}
+          BPFMAN_OPERATOR_IMG=${{ env.WF_BPFMAN_OPERATOR_IMG}}
+          MULTIARCH_TARGETS="${{ env.WF_MULTIARCH_TARGETS }}" make manifest-build manifest-push

.github/workflows/pull_request.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: ['1.21','1.22']
+        go: ['1.22']
     steps:
     - name: install make
       run: sudo apt-get install make
@@ -27,7 +27,7 @@ jobs:
     - name: check clean vendors
       run: go mod vendor
     - name: Report coverage
-      if: ${{ matrix.go == '1.21' }}
+      if: ${{ matrix.go == '1.22' }}
       uses: codecov/codecov-action@v4
       env:
         CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
@@ -41,10 +41,10 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: ['1.21']
+        go: ['1.22']
     env:
-      BPFMAN_AGENT_IMG: quay.io/bpfman/bpfman-agent:int-test
-      BPFMAN_OPERATOR_IMG: quay.io/bpfman/bpfman-operator:int-test
+      WF_BPFMAN_AGENT_IMG: quay.io/bpfman/bpfman-agent
+      WF_BPFMAN_OPERATOR_IMG: quay.io/bpfman/bpfman-operator
       XDP_PASS_PRIVATE_IMAGE_CREDS: ${{ secrets.XDP_PASS_PRIVATE_IMAGE_CREDS }}
     steps:
       - name: Check disk space
@@ -79,10 +79,20 @@ jobs:
           fetch-depth: 0
 
       - name: build images
-        run: make build-images
+        run: |
+          IMAGE_TAG="int-test"
+          MULTIARCH_TARGETS="amd64"
+          BPFMAN_AGENT_IMG="${{ env.WF_BPFMAN_AGENT_IMG }}"
+          BPFMAN_OPERATOR_IMG="${{ env.WF_BPFMAN_OPERATOR_IMG}}"
+          make build-images
 
       - name: run integration tests
-        run: make test-integration
+        run: |
+          IMAGE_TAG="int-test"
+          MULTIARCH_TARGETS="amd64"
+          BPFMAN_AGENT_IMG="${{ env.WF_BPFMAN_AGENT_IMG }}"
+          BPFMAN_OPERATOR_IMG="${{ env.WF_BPFMAN_OPERATOR_IMG}}"
+          make test-integration
 
       - name: Check disk space
         run: df -h

Containerfile.bpfman-agent

@@ -1,7 +1,10 @@
 # Build the manager binary
-FROM golang:1.22 as bpfman-agent-build
+ARG TARGETPLATFORM=linux/amd64
+ARG BUILDPLATFORM=linux/amd64
+FROM --platform=$BUILDPLATFORM docker.io/library/golang:1.22 as bpfman-agent-build
 ARG TARGETOS
-ARG TARGETARCH
+ARG TARGETPLATFORM
+ARG TARGETARCH=amd64
 
 WORKDIR /usr/src/bpfman-operator
 
@@ -28,7 +31,7 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -mod ven
 
 # Use the fedora minimal image to reduce the size of the final image but still
 # be able to easily install extra packages.
-FROM quay.io/fedora/fedora-minimal
+FROM --platform=$TARGETPLATFORM quay.io/fedora/fedora-minimal
 ARG DNF_CMD="microdnf"
 
 # The full fedora image can be used for debugging purposes.  To use it, comment

Containerfile.bpfman-operator

@@ -1,7 +1,10 @@
 # Build the manager binary
-FROM golang:1.22 as bpfman-operator-build
+ARG TARGETPLATFORM=linux/amd64
+ARG BUILDPLATFORM=linux/amd64
+FROM --platform=$BUILDPLATFORM docker.io/library/golang:1.22 as bpfman-operator-build
 ARG TARGETOS
-ARG TARGETARCH
+ARG TARGETPLATFORM
+ARG TARGETARCH=amd64
 
 WORKDIR /usr/src/bpfman-operator
 
@@ -32,7 +35,7 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -mod ven
 # The full fedora image can be used for debugging purposes, but you may need to
 # change "microdnf" to "dnf" below to install extra packages.
 # FROM fedora:38
-FROM quay.io/fedora/fedora-minimal
+FROM --platform=$TARGETPLATFORM quay.io/fedora/fedora-minimal
 ARG TARGETARCH
 WORKDIR /
 COPY --from=bpfman-operator-build /usr/src/bpfman-operator/config/bpfman-deployment/daemonset.yaml ./config/bpfman-deployment/daemonset.yaml

Makefile

@@ -4,6 +4,7 @@
 # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2)
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
 VERSION ?= 0.4.1
+MULTIARCH_TARGETS ?= amd64
 
 # CHANNELS define the bundle channels used in the bundle.
 # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")
@@ -45,12 +46,12 @@ USE_IMAGE_DIGESTS ?= false
 ifeq ($(USE_IMAGE_DIGESTS), true)
 	BUNDLE_GEN_FLAGS += --use-image-digests
 endif
-
+IMAGE_TAG ?= latest
 # Image URL to use all building/pushing image targets
-BPFMAN_IMG ?= quay.io/bpfman/bpfman:latest
-BPFMAN_AGENT_IMG ?= quay.io/bpfman/bpfman-agent:latest
-BPFMAN_OPERATOR_IMG ?= quay.io/bpfman/bpfman-operator:latest
-BPFMAN_OPERATOR_BUNDLE_IMG ?= quay.io/bpfman/bpfman-operator-bundle:latest
+BPFMAN_IMG ?= quay.io/bpfman/bpfman:$(IMAGE_TAG)
+BPFMAN_AGENT_IMG ?= quay.io/bpfman/bpfman-agent:$(IMAGE_TAG)
+BPFMAN_OPERATOR_IMG ?= quay.io/bpfman/bpfman-operator:$(IMAGE_TAG)
+BPFMAN_OPERATOR_BUNDLE_IMG ?= quay.io/bpfman/bpfman-operator-bundle:$(IMAGE_TAG)
 KIND_CLUSTER_NAME ?= bpfman-deployment
 
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
@@ -71,6 +72,23 @@ endif
 SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
 
+# Image building tool (docker / podman) - docker is preferred in CI
+OCI_BIN_PATH := $(shell which docker 2>/dev/null || which podman)
+OCI_BIN ?= $(shell basename ${OCI_BIN_PATH})
+
+# build a single arch target provided as argument
+define build_target
+	echo 'building $(1) for arch $(2)'; \
+	DOCKER_BUILDKIT=1 $(OCI_BIN) buildx build --load --build-arg TARGETPLATFORM=linux/$(2)\
+	 --build-arg TARGETARCH=$(2) --build-arg BUILDPLATFORM=linux/amd64 -t  $(1)-$(2) -f $(3) .;
+endef
+
+# push a single arch target image
+define push_target
+	echo 'pushing image $(1)-$(2)'; \
+	DOCKER_BUILDKIT=1 $(OCI_BIN) push $(1)-$(2);
+endef
+
 .PHONY: all
 all: build
 
@@ -126,7 +144,8 @@ OPERATOR_SDK_DL_URL=https://github.com/operator-framework/operator-sdk/releases/
 .PHONY: operator-sdk
 operator-sdk: $(OPERATOR_SDK)
 $(OPERATOR_SDK): $(LOCALBIN)
-	test -s $(LOCALBIN)/operator_sdk || { curl -LO ${OPERATOR_SDK_DL_URL} && chmod +x ${OPERATOR_SDK_DL_NAME} && mv ${OPERATOR_SDK_DL_NAME} $(LOCALBIN)/operator-sdk; }
+	test -s $(LOCALBIN)/operator_sdk || { curl -LO ${OPERATOR_SDK_DL_URL} && chmod +x ${OPERATOR_SDK_DL_NAME} &&\
+	 mv ${OPERATOR_SDK_DL_NAME} $(LOCALBIN)/operator-sdk; }
 
 .PHONY: controller-gen
 controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary.
@@ -219,15 +238,17 @@ generate-typed-clients: client-gen ## Generate typed client code
 
 .PHONY: generate-typed-listers
 generate-typed-listers: lister-gen ## Generate typed listers code
-	$(LISTER_GEN) "${PKG}/apis/v1alpha1" \
+	$(LISTER_GEN) \
+		"${PKG}/apis/v1alpha1" \
 		--output-pkg "${PKG}/pkg/client" \
 		--output-dir "./pkg/client" \
 		${COMMON_FLAGS}
 
 
 .PHONY: generate-typed-informers
 generate-typed-informers: informer-gen ## Generate typed informers code
-	$(INFORMER_GEN) "${PKG}/apis/v1alpha1" \
+	$(INFORMER_GEN) \
+		"${PKG}/apis/v1alpha1" \
 		--versioned-clientset-package "${PKG}/pkg/client/clientset" \
 		--listers-package "${PKG}/pkg/client" \
 		--output-pkg "${PKG}/pkg/client" \
@@ -279,7 +300,8 @@ test-integration: ## Run Integration tests.
 .PHONY: bundle
 bundle: operator-sdk generate kustomize manifests ## Generate bundle manifests and metadata, then validate generated files.
 	cd config/bpfman-operator-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-operator=${BPFMAN_OPERATOR_IMG}
-	cd config/bpfman-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman=${BPFMAN_IMG} && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-agent=${BPFMAN_AGENT_IMG}
+	cd config/bpfman-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman=${BPFMAN_IMG} &&\
+	 $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-agent=${BPFMAN_AGENT_IMG}
 	$(KUSTOMIZE) build config/manifests | $(OPERATOR_SDK) generate bundle $(BUNDLE_GEN_FLAGS)
 	cp config/manifests/dependencies.yaml bundle/metadata/
 	$(OPERATOR_SDK) bundle validate ./bundle
@@ -299,14 +321,35 @@ build: fmt ## Build bpfman-operator and bpfman-agent binaries.
 # (i.e. docker build --platform linux/arm64 ). However, you must enable docker buildKit for it.
 # More info: https://docs.docker.com/develop/develop-images/build_enhancements/
 .PHONY: build-images
-build-images: ## Build bpfman, bpfman-agent, and bpfman-operator images.
-	docker build -t ${BPFMAN_OPERATOR_IMG} -f Containerfile.bpfman-operator ./
-	docker build -t ${BPFMAN_AGENT_IMG} -f Containerfile.bpfman-agent ./
+build-images: ## Build bpfman-agent, and bpfman-operator images.
+	trap 'exit' INT; \
+	$(foreach target,$(MULTIARCH_TARGETS),$(call build_target,${BPFMAN_OPERATOR_IMG}:${IMAGE_TAG},$(target),Containerfile.bpfman-operator)) \
+	$(foreach target,$(MULTIARCH_TARGETS),$(call build_target,${BPFMAN_AGENT_IMG}:${IMAGE_TAG},$(target),Containerfile.bpfman-agent))
 
 .PHONY: push-images
 push-images: ## Push bpfman-agent, bpfman-operator images.
-	docker push ${BPFMAN_OPERATOR_IMG}
-	docker push ${BPFMAN_AGENT_IMG}
+	trap 'exit' INT; \
+	$(foreach target,$(MULTIARCH_TARGETS),$(call push_target,${BPFMAN_OPERATOR_IMG}:${IMAGE_TAG},$(target))) \
+	$(foreach target,$(MULTIARCH_TARGETS),$(call push_target,${BPFMAN_AGENT_IMG}:$(IMAGE_TAG),$(target)))
+
+.PHONY: manifest-build
+manifest-build: ## Build MULTIARCH_TARGETS manifest for bpfman-operator and bpfman-agent.
+	echo 'building manifest for $(BPFMAN_OPERATOR_IMG) and $(BPFMAN_AGENT_IMG)'
+	DOCKER_BUILDKIT=1 $(OCI_BIN) rmi ${BPFMAN_OPERATOR_IMG} -f
+	DOCKER_BUILDKIT=1 $(OCI_BIN) rmi ${BPFMAN_AGENT_IMG} -f
+	DOCKER_BUILDKIT=1 $(OCI_BIN) manifest create ${BPFMAN_OPERATOR_IMG}:${IMAGE_TAG} $(foreach target,$(MULTIARCH_TARGETS), --amend ${BPFMAN_OPERATOR_IMG}:{IMAGE_TAG}-$(target));
+	DOCKER_BUILDKIT=1 $(OCI_BIN) manifest create ${BPFMAN_AGENT_IMG}:{IMAGE_TAG} $(foreach target,$(MULTIARCH_TARGETS), --amend ${BPFMAN_AGENT_IMG}:{IMAGE_TAG}-$(target));
+
+.PHONY: manifest-push
+manifest-push: ## Push MULTIARCH_TARGETS manifest for bpfman-operator and bpfman-agent.
+	@echo 'publish manifest for $(BPFMAN_OPERATOR_IMG) and $(BPFMAN_AGENT_IMG)'
+ifeq (${OCI_BIN}, docker)
+	DOCKER_BUILDKIT=1 $(OCI_BIN) manifest push ${BPFMAN_OPERATOR_IMG};
+	DOCKER_BUILDKIT=1 $(OCI_BIN) manifest push ${BPFMAN_AGENT_IMG};
+else
+	DOCKER_BUILDKIT=1 $(OCI_BIN) manifest push ${BPFMAN_OPERATOR_IMG} docker://${BPFMAN_OPERATOR_IMG};
+	DOCKER_BUILDKIT=1 $(OCI_BIN) manifest push ${BPFMAN_AGENT_IMG} docker://${BPFMAN_AGENT_IMG};
+endif
 
 .PHONY: load-images-kind
 load-images-kind: ## Load bpfman-agent, and bpfman-operator images into the running local kind devel cluster.
@@ -370,7 +413,8 @@ destroy-kind: ## Destroy Kind cluster
 .PHONY: deploy
 deploy: manifests kustomize ## Deploy bpfman-operator to the K8s cluster specified in ~/.kube/config with the csi driver initialized.
 	cd config/bpfman-operator-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-operator=${BPFMAN_OPERATOR_IMG}
-	cd config/bpfman-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman=${BPFMAN_IMG} && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-agent=${BPFMAN_AGENT_IMG}
+	cd config/bpfman-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman=${BPFMAN_IMG} && \
+	$(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-agent=${BPFMAN_AGENT_IMG}
 	$(KUSTOMIZE) build config/default | kubectl apply -f -
 
 .PHONY: undeploy
@@ -392,7 +436,8 @@ run-on-kind: kustomize setup-kind build-images load-images-kind deploy ## Kind D
 .PHONY: deploy-openshift
 deploy-openshift: manifests kustomize ## Deploy bpfman-operator to the Openshift cluster specified in ~/.kube/config.
 	cd config/bpfman-operator-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-operator=${BPFMAN_OPERATOR_IMG}
-	cd config/bpfman-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman=${BPFMAN_IMG} && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-agent=${BPFMAN_AGENT_IMG}
+	cd config/bpfman-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman=${BPFMAN_IMG} \
+	&& $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-agent=${BPFMAN_AGENT_IMG}
 	$(KUSTOMIZE) build config/openshift | kubectl apply -f -
 
 .PHONY: undeploy-openshift

config/test/kustomization.yaml

@@ -46,4 +46,4 @@ resources:
 images:
   - name: quay.io/bpfman/bpfman-operator
     newName: quay.io/bpfman/bpfman-operator
-    newTag: int-test
+    newTag: int-test-amd64

test/integration/suite_test.go

@@ -24,6 +24,8 @@ import (
 	bpfmanHelpers "github.com/bpfman/bpfman-operator/pkg/helpers"
 )
 
+const intTestImageTag = ":int-test-amd64"
+
 var (
 	ctx          context.Context
 	cancel       context.CancelFunc
@@ -32,8 +34,8 @@ var (
 
 	// These images should already be built on the node so they can
 	// be loaded into kind.
-	bpfmanAgentImage    = os.Getenv("BPFMAN_AGENT_IMG")
-	bpfmanOperatorImage = os.Getenv("BPFMAN_OPERATOR_IMG")
+	bpfmanAgentImage    = os.Getenv("BPFMAN_AGENT_IMG") + intTestImageTag
+	bpfmanOperatorImage = os.Getenv("BPFMAN_OPERATOR_IMG") + intTestImageTag
 
 	existingCluster      = os.Getenv("USE_EXISTING_KIND_CLUSTER")
 	keepTestCluster      = func() bool { return os.Getenv("TEST_KEEP_CLUSTER") == "true" || existingCluster != "" }()