12/build(glooctl): removed ginkgo and gomega dependencies (#7937)

.gitignore

@@ -37,3 +37,6 @@ junit.xml
 *.test
 # swp files
 *.swp
+
+# python pickle files
+*.p

changelog/v1.12.48/reduce-dependency-footprint.yaml

@@ -0,0 +1,5 @@
+changelog:
+  - type: NON_USER_FACING
+    issueLink: https://github.com/solo-io/solo-projects/issues/4626
+    resolvesIssue: false
+    description: Remove ginkgo and gomega dependencies from "make glooctl-*"

hack/dependency-hunter.py

@@ -0,0 +1,105 @@
+#!/usr/bin/python3
+"""
+Analysis script geared towards hunting and removing unwanted dependencies from a project.
+
+Prior runs have yielded results resembling:
+```
+    PACKAGE="github.com/solo-io/gloo/projects/gloo/cli/cmd"
+    TARGET_BASE="github.com/onsi/ginkgo/v2"
+    REFRESH_GO_LIST=True
+        [github.com/solo-io/gloo/projects/gloo/cli/cmd, github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd, github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/federation, github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/federation/register, github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/install, github.com/solo-io/solo-kit/test/setup]
+        [github.com/solo-io/gloo/projects/gloo/cli/cmd, github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd, github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/debug, github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/install, github.com/solo-io/solo-kit/test/setup]
+        [github.com/solo-io/gloo/projects/gloo/cli/cmd, github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd, github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/install, github.com/solo-io/solo-kit/test/setup]
+        [github.com/solo-io/gloo/projects/gloo/cli/cmd, github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd, github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/check-crds, github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/install, github.com/solo-io/solo-kit/test/setup]
+```
+
+which were used to remove a dependency on ginkgo by avoiding a solo-kit import
+"""
+
+import subprocess
+import json
+import pickle
+
+PACKAGE         = "github.com/solo-io/gloo/projects/gloo/cli/cmd"   # go module to analyze
+TARGET_BASE     = "github.com/onsi/gomega"                          # string match for dependency to hunt
+REFRESH_GO_LIST = False                                             # recompute "go list" operation, overwriting saved pickle files
+
+# simple access to bash shell
+def shell(cmd):
+    process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE)
+    output, _ = process.communicate()
+    return output
+
+
+# run "go list -json {package}" for all dependencies of package and collect data
+def run_go_lists(package):
+    dependency_map, import_map = {}, {}
+
+    go_list = json.loads(shell(f"go list -json {package}"))
+
+    dependency_map[package] = go_list["Deps"]
+    import_map[package] = go_list["Imports"]
+
+    for i, dep in enumerate(go_list["Deps"]):
+        go_list = json.loads(shell(f"go list -json {dep}"))
+
+        if "Deps" not in go_list:       # some packages have no dependencies
+            dependency_map[dep] = []
+        else:                           # ...but most _do_ have dependencies
+            dependency_map[dep] = go_list["Deps"]
+
+        if "Imports" not in go_list:    # some packages have no imports
+            import_map[dep] = []
+        else:                           # ...but most _do_ have imports
+            import_map[dep] = go_list["Imports"]
+
+        print(f"go list {i}/{len(dependency_map[package])}")
+
+    return dependency_map, import_map
+
+
+# load "go list" data collectors from disk or run them if they don't exist
+def load_or_run(package, fresh_run=False):
+    if fresh_run:
+        dependency_map, import_map = run_go_lists(package)
+        pickle.dump(dependency_map, open("dependency_map.p", 'wb'))
+        pickle.dump(import_map, open("import_map.p", 'wb'))
+        print(f"computed {len(dependency_map)} fresh items")
+    else:
+        dependency_map = pickle.load(open("dependency_map.p", "rb"))
+        import_map = pickle.load(open("import_map.p", "rb"))
+        print(f"loaded {len(dependency_map)} items from disk")
+
+    return dependency_map, import_map
+
+if __name__ == "__main__":
+    # populate "go list" data collectors for all dependencies
+    dependency_map, import_map = load_or_run(PACKAGE, REFRESH_GO_LIST)
+
+    results = set()
+    package_imports = [(
+        import_map[PACKAGE], f"[{PACKAGE}"              # to begin with, we consider PACKAGE
+        )]
+    while len(package_imports) > 0:                     # stop iterating when we are out of packages
+        imports, path = package_imports.pop(0)
+
+        imports_target = any([i.startswith(TARGET_BASE) for i in imports])
+        if imports_target:                              # if we directly import target: we've found a culprit
+            results.add(f"{path}]")
+
+        for i in imports:                               # if not, some sub-dependency (or several) are the culprits...
+            depends_target = any([d.startswith(TARGET_BASE) for d in dependency_map[i]])
+            is_target = i.startswith(TARGET_BASE)
+
+            if depends_target and not is_target:        # ...and we keep recursing
+                new_imports = import_map[i]
+                new_path = f"{path}, {i}"
+                package_imports.append((new_imports, new_path))
+
+    # at present, the results _do not consider_ TestImports or XTestImports.  While I don't know for *sure*
+    # I've assumed that go only downloads production dependencies on standard compile operations
+    # 
+    # If I'm wrong, we can add a similar loop for TestImports/XTestImports, but I'm not sure it's worth the effort
+    print(f"{len(results)} matches found for {TARGET_BASE} in {PACKAGE}")
+    for r in results:
+        print(r)

pkg/cliutil/install/kubernetes.go

@@ -6,9 +6,6 @@ import (
 	"io"
 	"os"
 	"os/exec"
-	"strings"
-
-	. "github.com/onsi/gomega"
 
 	"github.com/solo-io/gloo/pkg/cliutil"
 )
@@ -42,45 +39,6 @@ func (k *CmdKubectl) KubectlOut(stdin io.Reader, args ...string) ([]byte, error)
 	return KubectlOut(stdin, args...)
 }
 
-type MockKubectl struct {
-	Expected        []string
-	Next            int
-	StdoutLines     []string
-	StdoutLineIndex int
-}
-
-var _ KubeCli = &MockKubectl{}
-
-func NewMockKubectl(cmds []string, stdoutLines []string) *MockKubectl {
-	return &MockKubectl{
-		Expected:    cmds,
-		Next:        0,
-		StdoutLines: stdoutLines,
-	}
-}
-
-func (k *MockKubectl) Kubectl(stdin io.Reader, args ...string) error {
-	// If this fails then the CLI tried to run commands we didn't account for in the mock
-	Expect(k.Next < len(k.Expected)).To(BeTrue())
-	Expect(stdin).To(BeNil())
-	cmd := strings.Join(args, " ")
-	Expect(cmd).To(BeEquivalentTo(k.Expected[k.Next]))
-	k.Next = k.Next + 1
-	return nil
-}
-
-func (k *MockKubectl) KubectlOut(stdin io.Reader, args ...string) ([]byte, error) {
-	Expect(k.Next < len(k.Expected)).To(BeTrue(), "MockKubectl did not have a next command for KubectlOut")
-	Expect(stdin).To(BeNil(), "Should have passed nil to MockKubectl.KubectlOut")
-	cmd := strings.Join(args, " ")
-	Expect(cmd).To(BeEquivalentTo(k.Expected[k.Next]), "Wrong next command for MockKubectl.KubectlOut")
-	k.Next = k.Next + 1
-	Expect(k.StdoutLineIndex < len(k.StdoutLines)).To(BeTrue(), "Mock kubectl has run out of stdout lines on command "+cmd)
-	stdOutLine := k.StdoutLines[k.StdoutLineIndex]
-	k.StdoutLineIndex = k.StdoutLineIndex + 1
-	return []byte(stdOutLine), nil
-}
-
 var verbose bool
 
 func SetVerbose(b bool) {

pkg/cliutil/testutil/testutil.go

@@ -1,6 +1,8 @@
 package testutil
 
 import (
+	"io"
+	"strings"
 	"time"
 
 	expect "github.com/Netflix/go-expect"
@@ -90,3 +92,40 @@ func (c *Console) ExpectEOF() string {
 	Expect(err).NotTo(HaveOccurred())
 	return ret
 }
+
+type MockKubectl struct {
+	Expected        []string
+	Next            int
+	StdoutLines     []string
+	StdoutLineIndex int
+}
+
+func NewMockKubectl(cmds []string, stdoutLines []string) *MockKubectl {
+	return &MockKubectl{
+		Expected:    cmds,
+		Next:        0,
+		StdoutLines: stdoutLines,
+	}
+}
+
+func (k *MockKubectl) Kubectl(stdin io.Reader, args ...string) error {
+	// If this fails then the CLI tried to run commands we didn't account for in the mock
+	Expect(k.Next < len(k.Expected)).To(BeTrue())
+	Expect(stdin).To(BeNil())
+	cmd := strings.Join(args, " ")
+	Expect(cmd).To(BeEquivalentTo(k.Expected[k.Next]))
+	k.Next = k.Next + 1
+	return nil
+}
+
+func (k *MockKubectl) KubectlOut(stdin io.Reader, args ...string) ([]byte, error) {
+	Expect(k.Next < len(k.Expected)).To(BeTrue(), "MockKubectl did not have a next command for KubectlOut")
+	Expect(stdin).To(BeNil(), "Should have passed nil to MockKubectl.KubectlOut")
+	cmd := strings.Join(args, " ")
+	Expect(cmd).To(BeEquivalentTo(k.Expected[k.Next]), "Wrong next command for MockKubectl.KubectlOut")
+	k.Next = k.Next + 1
+	Expect(k.StdoutLineIndex < len(k.StdoutLines)).To(BeTrue(), "Mock kubectl has run out of stdout lines on command "+cmd)
+	stdOutLine := k.StdoutLines[k.StdoutLineIndex]
+	k.StdoutLineIndex = k.StdoutLineIndex + 1
+	return []byte(stdOutLine), nil
+}

projects/gloo/cli/pkg/cmd/debug/debug_test.go

@@ -9,10 +9,9 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/solo-io/gloo/pkg/cliutil/testutil"
 	installcmd "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/install"
 
-	"github.com/solo-io/gloo/pkg/cliutil/install"
-
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/options"
@@ -80,7 +79,7 @@ var _ = Describe("Debug", func() {
 
 	Context("yaml dumper", func() {
 		var (
-			kubeCli        *install.MockKubectl
+			kubeCli        *testutil.MockKubectl
 			expectedOutput []string
 			importantKinds = append(append([]string{}, installcmd.GlooNamespacedKinds...), installcmd.GlooCrdNames...)
 		)
@@ -106,7 +105,7 @@ var _ = Describe("Debug", func() {
 			// don't really care what the returned data is, just want len(cmd) lines returned
 			expectedOutput = strings.Split(strings.Repeat("dummy-data-ignore_", len(cmds)), "_")
 
-			kubeCli = install.NewMockKubectl(cmds, expectedOutput)
+			kubeCli = testutil.NewMockKubectl(cmds, expectedOutput)
 
 			err = DumpYaml(tempFile.Name(), "test-namespace", kubeCli)
 			Expect(err).NotTo(HaveOccurred(), "Should be able to dump yaml without returning an error")

projects/gloo/cli/pkg/cmd/install/knative.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
+	"os/exec"
 	"regexp"
 	"strings"
 	"time"
@@ -15,7 +16,6 @@ import (
 
 	"github.com/solo-io/go-utils/contextutils"
 	"github.com/solo-io/k8s-utils/kubeutils"
-	"github.com/solo-io/solo-kit/test/setup"
 	"go.uber.org/zap"
 	"helm.sh/helm/v3/pkg/chartutil"
 
@@ -48,11 +48,29 @@ const (
 	yamlJoiner = "\n---\n"
 )
 
+// copied over from solo-kit to avoid a dependency on ginkgo
+func kubectlOut(args ...string) (string, error) {
+	cmd := exec.Command("kubectl", args...)
+	cmd.Env = os.Environ()
+	// disable DEBUG=1 from getting through to kube
+	for i, pair := range cmd.Env {
+		if strings.HasPrefix(pair, "DEBUG") {
+			cmd.Env = append(cmd.Env[:i], cmd.Env[i+1:]...)
+			break
+		}
+	}
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		err = fmt.Errorf("%s (%v)", out, err)
+	}
+	return string(out), err
+}
+
 func waitKnativeApiserviceReady() error {
 	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()
 	for {
-		stdout, err := setup.KubectlOut("get", "apiservice", "-ojsonpath='{.items[*].status.conditions[*].status}'")
+		stdout, err := kubectlOut("get", "apiservice", "-ojsonpath='{.items[*].status.conditions[*].status}'")
 		if err != nil {
 			contextutils.CliLogErrorw(ctx, "error getting apiserverice", "err", err)
 		}

projects/gloo/cli/pkg/cmd/install/uninstall_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/golang/mock/gomock"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	installutil "github.com/solo-io/gloo/pkg/cliutil/install"
+	"github.com/solo-io/gloo/pkg/cliutil/testutil"
 	"github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/install"
 	"github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/install/mocks"
 	"github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/options"
@@ -88,7 +88,7 @@ spec:
 		})
 
 		It("can uninstall", func() {
-			uninstaller := install.NewUninstallerWithOutput(mockHelmClient, installutil.NewMockKubectl([]string{}, []string{}), new(bytes.Buffer))
+			uninstaller := install.NewUninstallerWithOutput(mockHelmClient, testutil.NewMockKubectl([]string{}, []string{}), new(bytes.Buffer))
 			err := uninstaller.Uninstall(ctx, &options.HelmUninstall{
 				Namespace:       defaults.GlooSystem,
 				HelmReleaseName: constants.GlooReleaseName,
@@ -98,7 +98,7 @@ spec:
 		})
 
 		It("can uninstall CRDs when requested", func() {
-			mockKubectl := installutil.NewMockKubectl([]string{"delete crd " + crdName}, []string{})
+			mockKubectl := testutil.NewMockKubectl([]string{"delete crd " + crdName}, []string{})
 
 			uninstaller := install.NewUninstallerWithOutput(mockHelmClient, mockKubectl, new(bytes.Buffer))
 			err := uninstaller.Uninstall(ctx, &options.HelmUninstall{
@@ -111,7 +111,7 @@ spec:
 		})
 
 		It("can remove namespace when requested", func() {
-			mockKubectl := installutil.NewMockKubectl([]string{
+			mockKubectl := testutil.NewMockKubectl([]string{
 				"delete namespace " + defaults.GlooSystem,
 			}, []string{})
 
@@ -126,7 +126,7 @@ spec:
 		})
 
 		It("--all flag behaves as expected", func() {
-			mockKubectl := installutil.NewMockKubectl([]string{
+			mockKubectl := testutil.NewMockKubectl([]string{
 				"delete crd " + crdName,
 				"delete namespace " + defaults.GlooSystem,
 			}, []string{})
@@ -166,7 +166,7 @@ spec:
 		})
 
 		It("deletes all resources with the app=gloo label in the given namespace", func() {
-			mockKubectl := installutil.NewMockKubectl(namespacedDeleteCmds, []string{})
+			mockKubectl := testutil.NewMockKubectl(namespacedDeleteCmds, []string{})
 
 			uninstaller := install.NewUninstallerWithOutput(mockHelmClient, mockKubectl, new(bytes.Buffer))
 			err := uninstaller.Uninstall(ctx, &options.HelmUninstall{
@@ -178,7 +178,7 @@ spec:
 		})
 
 		It("removes the Gloo CRDs when the appropriate flag is provided", func() {
-			mockKubectl := installutil.NewMockKubectl(append(namespacedDeleteCmds, crdDeleteCmd), []string{})
+			mockKubectl := testutil.NewMockKubectl(append(namespacedDeleteCmds, crdDeleteCmd), []string{})
 
 			uninstaller := install.NewUninstallerWithOutput(mockHelmClient, mockKubectl, new(bytes.Buffer))
 			err := uninstaller.Uninstall(ctx, &options.HelmUninstall{
@@ -191,7 +191,7 @@ spec:
 		})
 
 		It("removes namespace when the appropriate flag is provided", func() {
-			mockKubectl := installutil.NewMockKubectl(append(namespacedDeleteCmds, "delete namespace "+defaults.GlooSystem), []string{})
+			mockKubectl := testutil.NewMockKubectl(append(namespacedDeleteCmds, "delete namespace "+defaults.GlooSystem), []string{})
 
 			uninstaller := install.NewUninstallerWithOutput(mockHelmClient, mockKubectl, new(bytes.Buffer))
 			err := uninstaller.Uninstall(ctx, &options.HelmUninstall{
@@ -207,7 +207,7 @@ spec:
 			commands := append(namespacedDeleteCmds, clusterScopedDeleteCmds...)
 			commands = append(commands, crdDeleteCmd)
 			commands = append(commands, "delete namespace "+defaults.GlooSystem)
-			mockKubectl := installutil.NewMockKubectl(commands, []string{})
+			mockKubectl := testutil.NewMockKubectl(commands, []string{})
 
 			uninstaller := install.NewUninstallerWithOutput(mockHelmClient, mockKubectl, new(bytes.Buffer))
 			err := uninstaller.Uninstall(ctx, &options.HelmUninstall{