From fe9c08776a53c073c7e798a1b7b8caced2f771cc Mon Sep 17 00:00:00 2001 From: Rahul Sharma Date: Tue, 26 Nov 2024 15:30:13 +0000 Subject: [PATCH] update GHA to run e2e tests --- .github/filters.yml | 3 ++ .github/workflows/ci.yml | 61 ++++++++++++++++++++++++ Makefile | 92 ++++++++++++++++++++++++++++++++---- devbox.json | 9 +++- e2e/setup/ctlptl-config.yaml | 9 ++++ 5 files changed, 165 insertions(+), 9 deletions(-) create mode 100644 .github/filters.yml create mode 100644 e2e/setup/ctlptl-config.yaml diff --git a/.github/filters.yml b/.github/filters.yml new file mode 100644 index 00000000..e307b278 --- /dev/null +++ b/.github/filters.yml @@ -0,0 +1,3 @@ +# Any file that is not a doc *.md file +src: + - "!**/**.md" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9cbc9be7..71425b16 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,25 @@ on: pull_request: null jobs: + changes: + runs-on: ubuntu-latest + outputs: + paths: ${{ steps.filter.outputs.changes }} + steps: + - uses: actions/checkout@v4 + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + - uses: dorny/paths-filter@v3 + id: filter + with: + base: ${{ github.ref }} + filters: .github/filters.yml ci: runs-on: ubuntu-latest strategy: @@ -61,3 +80,45 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: | REV=${{ github.ref_name }} + + e2e-tests: + runs-on: ubuntu-latest + needs: changes + if: ${{ contains(fromJSON(needs.changes.outputs.paths), 'src') }} + env: + GITHUB_TOKEN: ${{ secrets.github_token }} + LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }} + IMG: linode/linode-cloud-controller-manager:${{ github.ref == 'refs/heads/main' && 'latest' || format('pr-{0}', github.event.number) || github.ref_name }} + LINODE_REGION: us-lax + LINODE_CONTROL_PLANE_MACHINE_TYPE: g6-standard-2 + LINODE_MACHINE_TYPE: g6-standard-2 + WORKER_NODES: '2' + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + check-latest: true + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Install devbox + uses: jetify-com/devbox-install-action@v0.11.0 + + - name: Setup CAPL Management Kind Cluster and CAPL Child Cluster For Testing + run: devbox run mgmt-and-capl-cluster + + - name: Run E2E Tests + run: devbox run e2e-test + + - name: Cleanup Resources + if: always() + run: devbox run cleanup-cluster diff --git a/Makefile b/Makefile index a64209ab..f4d84a68 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,26 @@ -IMG ?= linode/linode-cloud-controller-manager:canary -RELEASE_DIR ?= release -PLATFORM ?= linux/amd64 +IMG ?= linode/linode-cloud-controller-manager:canary +RELEASE_DIR ?= release +PLATFORM ?= linux/amd64 # Use CACHE_BIN for tools that cannot use devbox and LOCALBIN for tools that can use either method -CACHE_BIN ?= $(CURDIR)/bin -LOCALBIN ?= $(CACHE_BIN) - -DEVBOX_BIN ?= $(DEVBOX_PACKAGES_DIR)/bin +CACHE_BIN ?= $(CURDIR)/bin +LOCALBIN ?= $(CACHE_BIN) + +DEVBOX_BIN ?= $(DEVBOX_PACKAGES_DIR)/bin + +##################################################################### +# Dev Setup +##################################################################### +CLUSTER_NAME ?= ccm-$(shell git rev-parse --short HEAD) +K8S_VERSION ?= "v1.29.1" +CAPI_VERSION ?= "v1.6.3" +HELM_VERSION ?= "v0.2.1" +CAPL_VERSION ?= "v0.7.1" +CONTROLPLANE_NODES ?= 1 +WORKER_NODES ?= 1 +LINODE_FIREWALL_ENABLED ?= true +LINODE_REGION ?= us-lax +KUBECONFIG_PATH ?= $(CURDIR)/test-cluster-kubeconfig.yaml # if the $DEVBOX_PACKAGES_DIR env variable exists that means we are within a devbox shell and can safely # use devbox's bin for our tools @@ -88,9 +102,11 @@ docker-build: build-linux .PHONY: docker-push # must run the docker build before pushing the image docker-push: - echo "[reminder] Did you run `make docker-build`?" docker push ${IMG} +.PHONY: docker-setup +docker-setup: docker-build docker-push + .PHONY: run # run the ccm locally, really only makes sense on linux anyway run: build @@ -108,6 +124,66 @@ run-debug: build --kubeconfig=${KUBECONFIG} \ --linodego-debug +##################################################################### +# E2E Test Setup +##################################################################### + +.PHONY: mgmt-and-capl-cluster +mgmt-and-capl-cluster: docker-setup mgmt-cluster capl-cluster + +.PHONY: capl-cluster +capl-cluster: generate-capl-cluster-manifests create-capl-cluster patch-linode-ccm + +.PHONY: generate-capl-cluster-manifests +generate-capl-cluster-manifests: + # Create the CAPL cluster manifests without any CSI driver stuff + LINODE_FIREWALL_ENABLED=$(LINODE_FIREWALL_ENABLED) clusterctl generate cluster $(CLUSTER_NAME) \ + --kubernetes-version $(K8S_VERSION) \ + --infrastructure linode-linode:$(CAPL_VERSION) \ + --control-plane-machine-count $(CONTROLPLANE_NODES) --worker-machine-count $(WORKER_NODES) > capl-cluster-manifests.yaml + +.PHONY: create-capl-cluster +create-capl-cluster: + # Create a CAPL cluster with updated CCM and wait for it to be ready + kubectl apply -f capl-cluster-manifests.yaml + kubectl wait --for=condition=ControlPlaneReady cluster/$(CLUSTER_NAME) --timeout=600s || (kubectl get cluster -o yaml; kubectl get linodecluster -o yaml; kubectl get linodemachines -o yaml) + kubectl wait --for=condition=NodeHealthy=true machines -l cluster.x-k8s.io/cluster-name=$(CLUSTER_NAME) --timeout=900s + clusterctl get kubeconfig $(CLUSTER_NAME) > $(KUBECONFIG_PATH) + KUBECONFIG=$(KUBECONFIG_PATH) kubectl wait --for=condition=Ready nodes --all --timeout=600s + # Remove all taints so that pods can be scheduled anywhere (without this, some tests fail) + KUBECONFIG=$(KUBECONFIG_PATH) kubectl taint nodes -l node-role.kubernetes.io/control-plane node-role.kubernetes.io/control-plane- + +.PHONY: patch-linode-ccm +patch-linode-ccm: + KUBECONFIG=$(KUBECONFIG_PATH) kubectl patch -n kube-system daemonset ccm-linode --type='json' -p="[{'op': 'replace', 'path': '/spec/template/spec/containers/0/image', 'value': '${IMG}'}]" + KUBECONFIG=$(KUBECONFIG_PATH) kubectl rollout status -n kube-system daemonset/ccm-linode --timeout=600s + +.PHONY: mgmt-cluster +mgmt-cluster: + # Create a mgmt cluster + ctlptl apply -f e2e/setup/ctlptl-config.yaml + clusterctl init \ + --wait-providers \ + --wait-provider-timeout 600 \ + --core cluster-api:$(CAPI_VERSION) \ + --addon helm:$(HELM_VERSION) \ + --infrastructure linode-linode:$(CAPL_VERSION) + +.PHONY: cleanup-cluster +cleanup-cluster: + kubectl delete cluster --all + kubectl delete linodefirewalls --all + kubectl delete lvpc --all + kind delete cluster -n caplccm + +.PHONY: e2e-test +e2e-test: + $(MAKE) -C e2e test LINODE_API_TOKEN=$(LINODE_TOKEN) SUITE_ARGS="--region=$(LINODE_REGION) --use-existing --timeout=5m --kubeconfig=$(KUBECONFIG_PATH) --image=$(IMG) --linode-url https://api.linode.com/" + +##################################################################### +# OS / ARCH +##################################################################### + # Set the host's OS. Only linux and darwin supported for now HOSTOS := $(shell uname -s | tr '[:upper:]' '[:lower:]') ifeq ($(filter darwin linux,$(HOSTOS)),) diff --git a/devbox.json b/devbox.json index 481ecf54..e918e800 100644 --- a/devbox.json +++ b/devbox.json @@ -18,6 +18,13 @@ "init_hook": [ "export \"GOROOT=$(go env GOROOT)\"" ], - "scripts": {} + "scripts": { + "mgmt-and-capl-cluster": "make mgmt-and-capl-cluster", + "e2e-test": "make e2e-test", + "cleanup-cluster": "make cleanup-cluster" + } + }, + "env": { + "EXP_CLUSTER_RESOURCE_SET": "true" } } diff --git a/e2e/setup/ctlptl-config.yaml b/e2e/setup/ctlptl-config.yaml new file mode 100644 index 00000000..ed3bee8f --- /dev/null +++ b/e2e/setup/ctlptl-config.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: ctlptl.dev/v1alpha1 +kind: Cluster +product: kind +kindV1Alpha4Cluster: + name: caplccm + nodes: + - role: control-plane + image: kindest/node:v1.29.2