From b4ee2af3f54e7b1ecbfbebd49fa97d56cffb2dd1 Mon Sep 17 00:00:00 2001 From: Rahul Sharma Date: Wed, 27 Nov 2024 18:14:06 -0500 Subject: [PATCH] [repo-ci-improvement] : update GHA to run e2e tests (#248) * update GHA to run e2e tests * simplify go versions used for ci tests * update helm version, update Makefile * update k8s version * fix linting * address review comments --- .github/filters.yml | 3 + .github/workflows/ci.yml | 115 ++++++++++++++++++++++++++++++++--- Makefile | 111 ++++++++++++++++++++++++++++----- cloud/linode/cloud.go | 2 +- devbox.json | 9 ++- e2e/setup/ctlptl-config.yaml | 9 +++ main.go | 4 +- 7 files changed, 225 insertions(+), 28 deletions(-) create mode 100644 .github/filters.yml create mode 100644 e2e/setup/ctlptl-config.yaml diff --git a/.github/filters.yml b/.github/filters.yml new file mode 100644 index 00000000..e307b278 --- /dev/null +++ b/.github/filters.yml @@ -0,0 +1,3 @@ +# Any file that is not a doc *.md file +src: + - "!**/**.md" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9cbc9be7..60751e4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,37 +6,92 @@ on: - main pull_request: null +permissions: + contents: read + pull-requests: read + actions: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + jobs: - ci: + changes: + runs-on: ubuntu-latest + outputs: + paths: ${{ steps.filter.outputs.changes }} + steps: + - uses: actions/checkout@v4 + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + - uses: dorny/paths-filter@v3 + id: filter + with: + base: ${{ github.ref }} + filters: .github/filters.yml + + build-test: runs-on: ubuntu-latest - strategy: - matrix: - go-version: [ 'stable', '1.22' ] + needs: changes + if: ${{ contains(fromJSON(needs.changes.outputs.paths), 'src') }} steps: + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + golang.org:443 + proxy.golang.org:443 + sum.golang.org:443 + objects.githubusercontent.com:443 + storage.googleapis.com:443 + cli.codecov.io:443 + api.codecov.io:443 + raw.githubusercontent.com:443 + get.helm.sh:443 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: - go-version: ${{ matrix.go-version }} + go-version-file: go.mod + check-latest: true + - name: Vet run: make vet - - name: Lint - run: make lint + + - name: lint + uses: golangci/golangci-lint-action@v6 + with: + version: latest + - name: Helm Lint run: make helm-lint + - name: Test run: make test + - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v4 with: files: ./coverage.out - fail_ci_if_error: true verbose: true token: ${{ secrets.CODECOV_TOKEN }} slug: linode/linode-cloud-controller-manager + - name: Build run: make build + docker-build: runs-on: ubuntu-latest steps: @@ -61,3 +116,45 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: | REV=${{ github.ref_name }} + + e2e-tests: + runs-on: ubuntu-latest + needs: changes + if: ${{ contains(fromJSON(needs.changes.outputs.paths), 'src') }} + env: + GITHUB_TOKEN: ${{ secrets.github_token }} + LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }} + IMG: linode/linode-cloud-controller-manager:${{ github.ref == 'refs/heads/main' && 'latest' || format('pr-{0}', github.event.number) || github.ref_name }} + LINODE_REGION: us-lax + LINODE_CONTROL_PLANE_MACHINE_TYPE: g6-standard-2 + LINODE_MACHINE_TYPE: g6-standard-2 + WORKER_NODES: '2' + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + check-latest: true + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Install devbox + uses: jetify-com/devbox-install-action@v0.11.0 + + - name: Setup CAPL Management Kind Cluster and CAPL Child Cluster For Testing + run: devbox run mgmt-and-capl-cluster + + - name: Run E2E Tests + run: devbox run e2e-test + + - name: Cleanup Resources + if: always() + run: devbox run cleanup-cluster diff --git a/Makefile b/Makefile index a64209ab..17c5f76c 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,29 @@ -IMG ?= linode/linode-cloud-controller-manager:canary -RELEASE_DIR ?= release -PLATFORM ?= linux/amd64 +IMG ?= linode/linode-cloud-controller-manager:canary +RELEASE_DIR ?= release +PLATFORM ?= linux/amd64 # Use CACHE_BIN for tools that cannot use devbox and LOCALBIN for tools that can use either method -CACHE_BIN ?= $(CURDIR)/bin -LOCALBIN ?= $(CACHE_BIN) - -DEVBOX_BIN ?= $(DEVBOX_PACKAGES_DIR)/bin +CACHE_BIN ?= $(CURDIR)/bin +LOCALBIN ?= $(CACHE_BIN) + +DEVBOX_BIN ?= $(DEVBOX_PACKAGES_DIR)/bin +HELM ?= $(LOCALBIN)/helm +HELM_VERSION ?= v3.16.3 + +##################################################################### +# Dev Setup +##################################################################### +CLUSTER_NAME ?= ccm-$(shell git rev-parse --short HEAD) +K8S_VERSION ?= "v1.31.2" +CAPI_VERSION ?= "v1.6.3" +CAAPH_VERSION ?= "v0.2.1" +CAPL_VERSION ?= "v0.7.1" +CONTROLPLANE_NODES ?= 1 +WORKER_NODES ?= 1 +LINODE_FIREWALL_ENABLED ?= true +LINODE_REGION ?= us-lax +LINODE_OS ?= linode/ubuntu22.04 +KUBECONFIG_PATH ?= $(CURDIR)/test-cluster-kubeconfig.yaml # if the $DEVBOX_PACKAGES_DIR env variable exists that means we are within a devbox shell and can safely # use devbox's bin for our tools @@ -41,10 +58,15 @@ vet: fmt .PHONY: lint lint: - docker run --rm -v "$(shell pwd):/var/work:ro" -w /var/work \ - golangci/golangci-lint:v1.57.2 golangci-lint run -v --timeout=5m - docker run --rm -v "$(shell pwd):/var/work:ro" -w /var/work/e2e \ - golangci/golangci-lint:v1.57.2 golangci-lint run -v --timeout=5m + docker run --rm -v "$(PWD):/var/work:ro" -w /var/work \ + golangci/golangci-lint:latest golangci-lint run -v --timeout=5m + docker run --rm -v "$(PWD):/var/work:ro" -w /var/work/e2e \ + golangci/golangci-lint:latest golangci-lint run -v --timeout=5m + +.PHONY: gosec +gosec: ## Run gosec against code. + docker run --rm -v "$(PWD):/var/work:ro" -w /var/work securego/gosec:2.19.0 \ + -exclude-dir=bin -exclude-generated ./... .PHONY: fmt fmt: @@ -88,9 +110,11 @@ docker-build: build-linux .PHONY: docker-push # must run the docker build before pushing the image docker-push: - echo "[reminder] Did you run `make docker-build`?" docker push ${IMG} +.PHONY: docker-setup +docker-setup: docker-build docker-push + .PHONY: run # run the ccm locally, really only makes sense on linux anyway run: build @@ -108,6 +132,66 @@ run-debug: build --kubeconfig=${KUBECONFIG} \ --linodego-debug +##################################################################### +# E2E Test Setup +##################################################################### + +.PHONY: mgmt-and-capl-cluster +mgmt-and-capl-cluster: docker-setup mgmt-cluster capl-cluster + +.PHONY: capl-cluster +capl-cluster: generate-capl-cluster-manifests create-capl-cluster patch-linode-ccm + +.PHONY: generate-capl-cluster-manifests +generate-capl-cluster-manifests: + # Create the CAPL cluster manifests without any CSI driver stuff + LINODE_FIREWALL_ENABLED=$(LINODE_FIREWALL_ENABLED) LINODE_OS=$(LINODE_OS) clusterctl generate cluster $(CLUSTER_NAME) \ + --kubernetes-version $(K8S_VERSION) --infrastructure linode-linode:$(CAPL_VERSION) \ + --control-plane-machine-count $(CONTROLPLANE_NODES) --worker-machine-count $(WORKER_NODES) > capl-cluster-manifests.yaml + +.PHONY: create-capl-cluster +create-capl-cluster: + # Create a CAPL cluster with updated CCM and wait for it to be ready + kubectl apply -f capl-cluster-manifests.yaml + kubectl wait --for=condition=ControlPlaneReady cluster/$(CLUSTER_NAME) --timeout=600s || (kubectl get cluster -o yaml; kubectl get linodecluster -o yaml; kubectl get linodemachines -o yaml) + kubectl wait --for=condition=NodeHealthy=true machines -l cluster.x-k8s.io/cluster-name=$(CLUSTER_NAME) --timeout=900s + clusterctl get kubeconfig $(CLUSTER_NAME) > $(KUBECONFIG_PATH) + KUBECONFIG=$(KUBECONFIG_PATH) kubectl wait --for=condition=Ready nodes --all --timeout=600s + # Remove all taints from control plane node so that pods scheduled on it by tests can run (without this, some tests fail) + KUBECONFIG=$(KUBECONFIG_PATH) kubectl taint nodes -l node-role.kubernetes.io/control-plane node-role.kubernetes.io/control-plane- + +.PHONY: patch-linode-ccm +patch-linode-ccm: + KUBECONFIG=$(KUBECONFIG_PATH) kubectl patch -n kube-system daemonset ccm-linode --type='json' -p="[{'op': 'replace', 'path': '/spec/template/spec/containers/0/image', 'value': '${IMG}'}]" + KUBECONFIG=$(KUBECONFIG_PATH) kubectl rollout status -n kube-system daemonset/ccm-linode --timeout=600s + KUBECONFIG=$(KUBECONFIG_PATH) kubectl -n kube-system get daemonset/ccm-linode -o yaml + +.PHONY: mgmt-cluster +mgmt-cluster: + # Create a mgmt cluster + ctlptl apply -f e2e/setup/ctlptl-config.yaml + clusterctl init \ + --wait-providers \ + --wait-provider-timeout 600 \ + --core cluster-api:$(CAPI_VERSION) \ + --addon helm:$(CAAPH_VERSION) \ + --infrastructure linode-linode:$(CAPL_VERSION) + +.PHONY: cleanup-cluster +cleanup-cluster: + kubectl delete cluster -A --all --timeout=180s + kubectl delete linodefirewalls -A --all --timeout=60s + kubectl delete lvpc -A --all --timeout=60s + kind delete cluster -n caplccm + +.PHONY: e2e-test +e2e-test: + $(MAKE) -C e2e test LINODE_API_TOKEN=$(LINODE_TOKEN) SUITE_ARGS="--region=$(LINODE_REGION) --use-existing --timeout=5m --kubeconfig=$(KUBECONFIG_PATH) --image=$(IMG) --linode-url https://api.linode.com/" + +##################################################################### +# OS / ARCH +##################################################################### + # Set the host's OS. Only linux and darwin supported for now HOSTOS := $(shell uname -s | tr '[:upper:]' '[:lower:]') ifeq ($(filter darwin linux,$(HOSTOS)),) @@ -121,9 +205,6 @@ else ifeq ($(ARCH_SHORT),aarch64) ARCH_SHORT := arm64 endif -HELM ?= $(LOCALBIN)/helm -HELM_VERSION ?= v3.9.1 - .PHONY: helm helm: $(HELM) ## Download helm locally if necessary $(HELM): $(LOCALBIN) diff --git a/cloud/linode/cloud.go b/cloud/linode/cloud.go index b85f6191..e9ac261b 100644 --- a/cloud/linode/cloud.go +++ b/cloud/linode/cloud.go @@ -117,7 +117,7 @@ func newCloud() (cloudprovider.Interface, error) { if len(Options.IpHolderSuffix) > 23 { msg := fmt.Sprintf("ip-holder-suffix must be 23 characters or less: %s is %d characters\n", Options.IpHolderSuffix, len(Options.IpHolderSuffix)) klog.Error(msg) - return nil, fmt.Errorf(msg) + return nil, fmt.Errorf("%s", msg) } // create struct that satisfies cloudprovider.Interface diff --git a/devbox.json b/devbox.json index 481ecf54..e918e800 100644 --- a/devbox.json +++ b/devbox.json @@ -18,6 +18,13 @@ "init_hook": [ "export \"GOROOT=$(go env GOROOT)\"" ], - "scripts": {} + "scripts": { + "mgmt-and-capl-cluster": "make mgmt-and-capl-cluster", + "e2e-test": "make e2e-test", + "cleanup-cluster": "make cleanup-cluster" + } + }, + "env": { + "EXP_CLUSTER_RESOURCE_SET": "true" } } diff --git a/e2e/setup/ctlptl-config.yaml b/e2e/setup/ctlptl-config.yaml new file mode 100644 index 00000000..6d9570a2 --- /dev/null +++ b/e2e/setup/ctlptl-config.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: ctlptl.dev/v1alpha1 +kind: Cluster +product: kind +kindV1Alpha4Cluster: + name: caplccm + nodes: + - role: control-plane + image: kindest/node:v1.31.2 diff --git a/main.go b/main.go index dde9feb8..53b5b2ab 100644 --- a/main.go +++ b/main.go @@ -113,7 +113,7 @@ func main() { linode.Options.KubeconfigFlag = command.Flags().Lookup("kubeconfig") if linode.Options.KubeconfigFlag == nil { msg := "kubeconfig missing from CCM flag set" - sentry.CaptureError(ctx, fmt.Errorf(msg)) + sentry.CaptureError(ctx, fmt.Errorf("%s", msg)) fmt.Fprintf(os.Stderr, "kubeconfig missing from CCM flag set"+"\n") os.Exit(1) } @@ -122,7 +122,7 @@ func main() { _, network, err := net.ParseCIDR(externalSubnet) if err != nil { msg := fmt.Sprintf("Unable to parse %s as network subnet: %v", externalSubnet, err) - sentry.CaptureError(ctx, fmt.Errorf(msg)) + sentry.CaptureError(ctx, fmt.Errorf("%s", msg)) fmt.Fprintf(os.Stderr, "%v\n", msg) os.Exit(1) }