Replace Terroform to CAPL provider for E2E testing

.gitignore

@@ -9,6 +9,7 @@ linode-cloud-controller-manager
 # Folders
 _obj
 _test
+bin
 
 # Architecture specific extensions/prefixes
 *.[568vq]

deploy/ccm-linode-template.yaml

@@ -22,7 +22,7 @@ metadata:
 rules:
 - apiGroups: [""]
   resources: ["endpoints"]
-  verbs: ["get", "watch", "list", "update", "create"]
+  verbs: ["get"]
 - apiGroups: [""]
   resources: ["nodes"]
   verbs: ["get", "watch", "list", "update", "delete", "patch"]
@@ -44,6 +44,9 @@ rules:
 - apiGroups: [""]
   resources: ["services/status"]
   verbs: ["get", "watch", "list", "update", "patch"]
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "update", "create"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -101,9 +104,8 @@ spec:
           imagePullPolicy: Always
           name: ccm-linode
           args:
-          - --leader-elect-resource-lock=endpoints
+          - --leader-elect-resource-lock=leases
           - --v=3
-          - --port=0
           - --secure-port=10253
           volumeMounts:
           - mountPath: /etc/kubernetes

deploy/chart/templates/clusterrole-rbac.yaml

@@ -5,7 +5,7 @@ metadata:
 rules:
   - apiGroups: [""]
     resources: ["endpoints"]
-    verbs: ["get", "watch", "list", "update", "create"]
+    verbs: ["get"]
   - apiGroups: ["coordination.k8s.io"]
     resources: ["leases"]
     verbs: ["get", "watch", "list", "update", "create"]
@@ -30,3 +30,6 @@ rules:
   - apiGroups: [""]
     resources: ["services/status"]
     verbs: ["get", "watch", "list", "update", "patch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "update", "create"]

deploy/generate-manifest.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+: ${IMG:="linode/linode-cloud-controller-manager:latest"}
+
 set -o pipefail -o noclobber -o nounset
 
 die() { echo "$*" 1>&2; exit 1; }
@@ -13,6 +15,8 @@ Second argument must be a Linode region.
 Example:
 $ ./generate-manifest.sh \$LINODE_API_TOKEN us-east"
 
+rm -rf ccm-linode.yaml
+
 BASE64FLAGS=""
 longstring="1234567890123456789012345678901234567890123456789012345678901234567890"
 if [ $(echo "$longstring" | base64 | wc -l) -gt 1 ]; then
@@ -26,4 +30,5 @@ ENCODED_REGION=$(echo -n $2 | base64 $BASE64FLAGS)
 
 cat "$(dirname "$0")/ccm-linode-template.yaml" |
 sed -e "s|{{ .Values.apiTokenB64 }}|$ENCODED_TOKEN|" |
-sed -e "s|{{ .Values.linodeRegionB64 }}|$ENCODED_REGION|" > ccm-linode.yaml
+sed -e "s|{{ .Values.linodeRegionB64 }}|$ENCODED_REGION|" |
+sed "s|linode/linode-cloud-controller-manager:latest|$IMG|" > ccm-linode.yaml

devbox.json

@@ -0,0 +1,18 @@
+{
+  "packages": [
+    "go_1_22@latest",
+    "[email protected]",
+    "[email protected]",
+    "[email protected]",
+    "[email protected]",
+    "[email protected]"
+  ],
+  "shell": {
+    "init_hook": [
+      "export GOROOT=$(go env GOROOT)",
+      "export KUBECONFIG=${PWD}/e2e/test/kind-management.conf",
+      "export K8S_VERSION=v1.28.7"
+    ],
+    "scripts": {}
+  }
+}

devbox.lock

@@ -0,0 +1,125 @@
+{
+  "lockfile_version": "1",
+  "packages": {
+    "[email protected]": {
+      "last_modified": "2024-03-22T11:26:23Z",
+      "resolved": "github:NixOS/nixpkgs/a3ed7406349a9335cb4c2a71369b697cecd9d351#clusterctl",
+      "source": "devbox-search",
+      "version": "1.6.3",
+      "systems": {
+        "aarch64-darwin": {
+          "store_path": "/nix/store/s5i9i38fvpgf69x994n8v1sb82lpyl5f-clusterctl-1.6.3"
+        },
+        "aarch64-linux": {
+          "store_path": "/nix/store/pj9x3qd0yz5bpk19snjwz25cm7bxnz5m-clusterctl-1.6.3"
+        },
+        "x86_64-darwin": {
+          "store_path": "/nix/store/0vhfm1fkqx3wyrjiyam70zbakg9qmyra-clusterctl-1.6.3"
+        },
+        "x86_64-linux": {
+          "store_path": "/nix/store/mjc4miffskqdby7ah5f66crxgydlns82-clusterctl-1.6.3"
+        }
+      }
+    },
+    "[email protected]": {
+      "last_modified": "2024-03-22T11:26:23Z",
+      "resolved": "github:NixOS/nixpkgs/a3ed7406349a9335cb4c2a71369b697cecd9d351#ctlptl",
+      "source": "devbox-search",
+      "version": "0.8.28",
+      "systems": {
+        "aarch64-darwin": {
+          "store_path": "/nix/store/3arviljap22bf4j6cwlks443kvmyaa9g-ctlptl-0.8.28"
+        },
+        "aarch64-linux": {
+          "store_path": "/nix/store/35xjh902dsvq8n584g6jqk971gpsbl40-ctlptl-0.8.28"
+        },
+        "x86_64-darwin": {
+          "store_path": "/nix/store/724aylc9p18xnj1i26qx86ql49vvpj2d-ctlptl-0.8.28"
+        },
+        "x86_64-linux": {
+          "store_path": "/nix/store/9451r473b34va2ap5i1k830wdy3nbc9l-ctlptl-0.8.28"
+        }
+      }
+    },
+    "[email protected]": {
+      "last_modified": "2024-03-22T11:26:23Z",
+      "resolved": "github:NixOS/nixpkgs/a3ed7406349a9335cb4c2a71369b697cecd9d351#ginkgo",
+      "source": "devbox-search",
+      "version": "2.17.0",
+      "systems": {
+        "aarch64-darwin": {
+          "store_path": "/nix/store/3xjbbhq85x3iix4db5bky7x9h92l8mr9-ginkgo-2.17.0"
+        },
+        "aarch64-linux": {
+          "store_path": "/nix/store/9fgwgs6q82q5r3z4d6sqh9nmpkkllz7i-ginkgo-2.17.0"
+        },
+        "x86_64-darwin": {
+          "store_path": "/nix/store/3kkdi5gvy9gwm6ncc8ggvxk68wn0nfcp-ginkgo-2.17.0"
+        },
+        "x86_64-linux": {
+          "store_path": "/nix/store/ifbjn9vcrplax832613bc0vz08qymwm9-ginkgo-2.17.0"
+        }
+      }
+    },
+    "go_1_22@latest": {
+      "last_modified": "2024-03-22T11:26:23Z",
+      "resolved": "github:NixOS/nixpkgs/a3ed7406349a9335cb4c2a71369b697cecd9d351#go",
+      "source": "devbox-search",
+      "version": "1.22.1",
+      "systems": {
+        "aarch64-darwin": {
+          "store_path": "/nix/store/n1k6wf8q10q7k0863gb78b1rf0j07r7r-go-1.22.1"
+        },
+        "aarch64-linux": {
+          "store_path": "/nix/store/fl6cjlp5bvykfz1kxrw687zxzld25pn7-go-1.22.1"
+        },
+        "x86_64-darwin": {
+          "store_path": "/nix/store/xgdp7gnf6vzr2ick2ip1lhq4cww65p7w-go-1.22.1"
+        },
+        "x86_64-linux": {
+          "store_path": "/nix/store/bp39dh48cdqp89hk5mpdi1lxdf0mjl7x-go-1.22.1"
+        }
+      }
+    },
+    "[email protected]": {
+      "last_modified": "2024-03-22T11:26:23Z",
+      "resolved": "github:NixOS/nixpkgs/a3ed7406349a9335cb4c2a71369b697cecd9d351#kind",
+      "source": "devbox-search",
+      "version": "0.22.0",
+      "systems": {
+        "aarch64-darwin": {
+          "store_path": "/nix/store/cff6f3a0av9pmak5w1wm9w2n1niq223b-kind-0.22.0"
+        },
+        "aarch64-linux": {
+          "store_path": "/nix/store/xkkw8di5pmlgj7mw7mhq12mjigvh033l-kind-0.22.0"
+        },
+        "x86_64-darwin": {
+          "store_path": "/nix/store/fxh74m2g045xwqv90lbvdg57bvk99pf6-kind-0.22.0"
+        },
+        "x86_64-linux": {
+          "store_path": "/nix/store/m362mrwv0419cj176nwg31zcs4af6wrs-kind-0.22.0"
+        }
+      }
+    },
+    "[email protected]": {
+      "last_modified": "2024-02-10T18:15:24Z",
+      "resolved": "github:NixOS/nixpkgs/10b813040df67c4039086db0f6eaf65c536886c6#kubectl",
+      "source": "devbox-search",
+      "version": "1.28.4",
+      "systems": {
+        "aarch64-darwin": {
+          "store_path": "/nix/store/9awcbcg079c6v9crrxv1s9r5lrxcjjyi-kubectl-1.28.4"
+        },
+        "aarch64-linux": {
+          "store_path": "/nix/store/vvrzf6y3xn6ynfykkimjqrgirdhb9c8i-kubectl-1.28.4"
+        },
+        "x86_64-darwin": {
+          "store_path": "/nix/store/3z0fannmka27ggzaqhfxangy11c80f1f-kubectl-1.28.4"
+        },
+        "x86_64-linux": {
+          "store_path": "/nix/store/77vmp37rm1vqmb37km04fwzp8rpcqys4-kubectl-1.28.4"
+        }
+      }
+    }
+  }
+}

e2e/Makefile

@@ -1,28 +1,36 @@
+ROOT_DIR="$(shell git rev-parse --show-toplevel)"
+
 IMG ?= linode/linode-cloud-controller-manager:latest
-GINKGO_PATH := $(shell go env GOPATH)/bin/ginkgo
+K8S_VERSION?="v1.28.7"
 
 REUSED_KUBECONFIG := test/ccm-linode-for-reuse.conf
+MANAGEMENT_KUBECONFIG := test/kind-management.conf
 
 ifneq ("$(wildcard $(REUSED_KUBECONFIG))","")
     CONTROL_PLANE_IP := $(shell grep -E -o "([0-9]{1,3}[\.]){3}[0-9]{1,3}" $(REUSED_KUBECONFIG))
 endif
 
 .PHONY: test reuse-and-test clean
 
-${GINKGO_PATH}:
-	go install github.com/onsi/ginkgo/v2/[email protected]
+test-cluster: kind ctlptl kind clean-cluster
+	./test/scripts/create_management_cluster.sh $(K8S_VERSION)
+
+clean-cluster: ctlptl
+	$(CTLPTL) delete cluster kind-ccm-management --ignore-not-found
+	rm -rf $(MANAGEMENT_KUBECONFIG)
 
 reuse-and-test: SUITE_ARGS='--reuse'
 
-test reuse-and-test: ${GINKGO_PATH} check-token
+test reuse-and-test: clusterctl check-token test-cluster _test
+	$(MAKE) clean-cluster
+
+_test: ginkgo
 	go list -m; \
-	ginkgo -r --vv --trace $(TEST_ARGS) -- --image=${IMG} $(SUITE_ARGS)
+	MANAGEMENT_KUBECONFIG=$(ROOT_DIR)/e2e/$(MANAGEMENT_KUBECONFIG) $(GINKGO) -r --vv --trace $(TEST_ARGS) -- --image=${IMG} --k8s_version=${K8S_VERSION} $(SUITE_ARGS)
 
 clean: check-token
 	cd test; \
-	./scripts/delete_cluster.sh ccm-linode-for-reuse; \
-	rm terraform.tfstate; \
-    rm -rf terraform.tfstate.d
+	./scripts/delete_cluster.sh ccm-linode-for-reuse
 
 check-token:
 	@if test "$(LINODE_API_TOKEN)" = "" ; then \
@@ -32,3 +40,78 @@ check-token:
 
 control-plane-ssh: $(REUSED_KUBECONFIG)
 	ssh root@$(CONTROL_PLANE_IP)
+
+#####################################################################
+# OS / ARCH
+#####################################################################
+OS=$(shell uname -s | tr '[:upper:]' '[:lower:]')
+ARCH=$(shell uname -m)
+ARCH_SHORT=$(ARCH)
+ifeq ($(ARCH_SHORT),x86_64)
+ARCH_SHORT := amd64
+else ifeq ($(ARCH_SHORT),aarch64)
+ARCH_SHORT := arm64
+endif
+
+## --------------------------------------
+## Build Dependencies
+## --------------------------------------
+
+##@ Build Dependencies:
+
+## Location to install dependencies to
+
+# Use CACHE_BIN for tools that cannot use devbox and LOCALBIN for tools that can use either method
+CACHE_BIN ?= $(CURDIR)/../bin
+LOCALBIN ?= $(CACHE_BIN)
+
+DEVBOX_BIN ?= $(DEVBOX_PACKAGES_DIR)/bin
+
+# if the $DEVBOX_PACKAGES_DIR env variable exists that means we are within a devbox shell and can safely
+# use devbox's bin for our tools
+ifdef DEVBOX_PACKAGES_DIR
+	LOCALBIN = $(DEVBOX_BIN)
+endif
+
+export PATH := $(CACHE_BIN):$(PATH)
+$(CACHE_BIN):
+	mkdir -p $(CACHE_BIN)
+$(LOCALBIN):
+	mkdir -p $(LOCALBIN)
+
+##@ Tooling Binaries:
+GINKGO         ?= $(LOCALBIN)/ginkgo
+CTLPTL         ?= $(LOCALBIN)/ctlptl
+CLUSTERCTL     ?= $(LOCALBIN)/clusterctl
+KIND           ?= $(LOCALBIN)/kind
+
+## Tool Versions
+GINKGO_VERSION           ?= v2.17.0
+CTLPTL_VERSION           ?= v0.8.28
+CLUSTERCTL_VERSION       ?= v1.6.3
+KIND_VERSION             ?= v0.22.0
+
+.PHONY: tools
+tools: ginkgo ctlptl clusterctl
+
+.PHONY: ginkgo
+ginkgo: $(GINKGO)
+$(GINKGO): $(LOCALBIN) ## Download ginkgo locally if necessary.
+	GOBIN=$(LOCALBIN) go install github.com/onsi/ginkgo/v2/ginkgo@$(GINKGO_VERSION)
+
+.PHONY: ctlptl
+ctlptl: $(CTLPTL) ## Download ctlptl locally if necessary.
+$(CTLPTL): $(LOCALBIN)
+	GOBIN=$(LOCALBIN) go install github.com/tilt-dev/ctlptl/cmd/ctlptl@$(CTLPTL_VERSION)
+
+.PHONY: clusterctl
+clusterctl: $(CLUSTERCTL) ## Download clusterctl locally if necessary.
+$(CLUSTERCTL): $(LOCALBIN)
+	curl -fsSL https://github.com/kubernetes-sigs/cluster-api/releases/download/$(CLUSTERCTL_VERSION)/clusterctl-$(OS)-$(ARCH_SHORT) -o $(CLUSTERCTL)
+	chmod +x $(CLUSTERCTL)
+
+.PHONY: kind
+kind: $(KIND) ## Download kind locally if necessary.
+$(KIND): $(LOCALBIN)
+	curl -fsSL https://github.com/kubernetes-sigs/kind/releases/download/$(KIND_VERSION)/kind-$(OS)-$(ARCH_SHORT) -o $(KIND)
+	chmod +x $(KIND)