✨ Running builtin provider externally from a GRPC Provider

Makefile

@@ -82,7 +82,7 @@ run-external-providers-pod:
 	podman build -f demo-local.Dockerfile -t localhost/testing:latest
 
 run-demo-image:
-	podman run --entrypoint /usr/local/bin/konveyor-analyzer --pod=analyzer -v $(PWD)/demo-dep-output.yaml:/analyzer-lsp/demo-dep-output.yaml:Z -v $(PWD)/demo-output.yaml:/analyzer-lsp/output.yaml:Z localhost/testing:latest --output-file=/analyzer-lsp/output.yaml --dep-output-file=/analyzer-lsp/demo-dep-output.yaml
+	podman run --entrypoint /usr/local/bin/konveyor-analyzer --pod=analyzer -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) -v $(PWD)/demo-dep-output.yaml:/analyzer-lsp/demo-dep-output.yaml:Z -v $(PWD)/demo-output.yaml:/analyzer-lsp/output.yaml:Z localhost/testing:latest --output-file=/analyzer-lsp/output.yaml --dep-output-file=/analyzer-lsp/demo-dep-output.yaml
 
 stop-external-providers-pod: stop-external-providers
 	podman pod kill analyzer

cmd/analyzer/main.go

@@ -128,6 +128,7 @@ func AnalysisCmd() *cobra.Command {
 			}
 
 			providers := map[string]provider.InternalProviderClient{}
+			builtinConfigs := []provider.InitConfig{}
 			providerLocations := []string{}
 			for _, config := range configs {
 				config.ContextLines = contextLines
@@ -140,6 +141,9 @@ func AnalysisCmd() *cobra.Command {
 					for _, i := range config.InitConfig {
 						i.AnalysisMode = provider.AnalysisMode(analysisMode)
 						inits = append(inits, i)
+						// builtinConfigs = append(builtinConfigs, provider.InitConfig{
+						// 	Location: i.Location,
+						// })
 					}
 					config.InitConfig = inits
 				}
@@ -204,15 +208,52 @@ func AnalysisCmd() *cobra.Command {
 				}
 			}
 			// Now that we have all the providers, we need to start them.
-			for name, provider := range needProviders {
-				initCtx, initSpan := tracing.StartNewSpan(ctx, "init",
-					attribute.Key("provider").String(name))
-				err := provider.ProviderInit(initCtx)
-				if err != nil {
-					errLog.Error(err, "unable to init the providers", "provider", name)
+			for name, prov := range needProviders {
+				switch name {
+				// other providers can return additional configs for the builtin provider
+				// therefore, we initiate builtin provider separately at the end
+				case "builtin":
+					continue
+				default:
+					initCtx, initSpan := tracing.StartNewSpan(ctx, "init",
+						attribute.Key("provider").String(name))
+					additionalBuiltinConfs, err := prov.ProviderInit(initCtx, nil)
+
+					if err != nil {
+						errLog.Error(err, "unable to init the providers", "provider", name)
+						os.Exit(1)
+					}
+
+					if additionalBuiltinConfs != nil {
+						config := prov.GetConfig()
+						newConfigs := []provider.InitConfig{}
+						if config.BuiltinAddres != "" {
+							// Builtin provider must understand these specific config items
+							// it will tell to to not use the local builtin provider, but a
+							// GRPC profvider
+							for _, c := range additionalBuiltinConfs {
+								c.Proxy = config.Proxy
+								c.ProviderSpecificConfig = map[string]interface{}{
+									"address":  config.BuiltinAddres,
+									"JWTToken": config.JWTToken,
+									"CertPath": config.CertPath,
+								}
+								newConfigs = append(newConfigs, c)
+							}
+							builtinConfigs = append(builtinConfigs, newConfigs...)
+						} else {
+							builtinConfigs = append(builtinConfigs, additionalBuiltinConfs...)
+						}
+					}
+					initSpan.End()
+				}
+			}
+
+			if builtinClient, ok := needProviders["builtin"]; ok {
+				if _, err = builtinClient.ProviderInit(ctx, builtinConfigs); err != nil {
+					errLog.Error(err, "unable to init builtin provider")
 					os.Exit(1)
 				}
-				initSpan.End()
 			}
 
 			wg := &sync.WaitGroup{}

cmd/dep/main.go

@@ -90,7 +90,7 @@ func DependencyCmd() *cobra.Command {
 
 				time.Sleep(5 * time.Second)
 
-				err = prov.ProviderInit(ctx)
+				_, err = prov.ProviderInit(ctx, nil)
 				b, _ := json.Marshal(config)
 				if err != nil {
 					errLog.Error(err, "unable to init the providers", "provider", config.Name, "the-error-is", err, "config", string(b))

demo-output.yaml

@@ -201,6 +201,14 @@
           data: dependency
           innerText: "\n\t\t\torg.springframework\n\t\t\tspring-webmvc\n\t\t\t${spring-framework.version}\n\t\t"
           matchingXML: <groupId>org.springframework</groupId><artifactId>spring-webmvc</artifactId><version>${spring-framework.version}</version>
+      - uri: file:///examples/java-project/pom.xml
+        message: <groupId>io.javaoperatorsdk.operator</groupId><artifactId>sample</artifactId><version>0.0.0</version>
+        codeSnip: "11    <url>http://www.konveyor.io</url>\n12  \n13    <properties>\n14      <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>\n15    </properties>\n16  \n17    <dependencies>\n18  \n19      <dependency>\n20        <groupId>io.javaoperatorsdk.operator</groupId>\n21        <artifactId>sample</artifactId>\n22        <version>0.0.0</version>\n23      </dependency>\n24  \n25    </dependencies>\n26  \n27    <build>\n28    </build>\n29  </project>\n"
+        lineNumber: 20
+        variables:
+          data: dependency
+          innerText: "\n      io.javaoperatorsdk.operator\n      sample\n      0.0.0\n    "
+          matchingXML: <groupId>io.javaoperatorsdk.operator</groupId><artifactId>sample</artifactId><version>0.0.0</version>
       - uri: file:///examples/java/dummy/pom.xml
         message: |-
           <groupId>javax</groupId><artifactId>javaee-api</artifactId><!-- This leads to https://github.com/konveyor/analyzer-lsp/issues/390
@@ -781,6 +789,14 @@
           data: dependency
           innerText: "\n\t\t\torg.springframework\n\t\t\tspring-webmvc\n\t\t\t${spring-framework.version}\n\t\t"
           matchingXML: <groupId>org.springframework</groupId><artifactId>spring-webmvc</artifactId><version>${spring-framework.version}</version>
+      - uri: file:///examples/java-project/pom.xml
+        message: POM XML dependencies - '<groupId>io.javaoperatorsdk.operator</groupId><artifactId>sample</artifactId><version>0.0.0</version>'
+        codeSnip: "11    <url>http://www.konveyor.io</url>\n12  \n13    <properties>\n14      <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>\n15    </properties>\n16  \n17    <dependencies>\n18  \n19      <dependency>\n20        <groupId>io.javaoperatorsdk.operator</groupId>\n21        <artifactId>sample</artifactId>\n22        <version>0.0.0</version>\n23      </dependency>\n24  \n25    </dependencies>\n26  \n27    <build>\n28    </build>\n29  </project>\n"
+        lineNumber: 20
+        variables:
+          data: dependency
+          innerText: "\n      io.javaoperatorsdk.operator\n      sample\n      0.0.0\n    "
+          matchingXML: <groupId>io.javaoperatorsdk.operator</groupId><artifactId>sample</artifactId><version>0.0.0</version>
       - uri: file:///examples/java/dummy/pom.xml
         message: |-
           POM XML dependencies - '<groupId>javax</groupId><artifactId>javaee-api</artifactId><!-- This leads to https://github.com/konveyor/analyzer-lsp/issues/390
@@ -918,21 +934,21 @@
       labels:
       - tag=Java Operator SDK
       incidents:
-      - uri: file:///java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
+      - uri: file:///examples/java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
         message: ""
         codeSnip: " 1  package io.javaoperatorsdk.operator.sample;\n 2  \n 3  import io.fabric8.kubernetes.client.KubernetesClient;\n 4  import io.javaoperatorsdk.operator.Operator;\n 5  import io.javaoperatorsdk.operator.api.config.ConfigurationService;\n 6  import io.javaoperatorsdk.operator.api.config.ControllerConfiguration;\n 7  import io.quarkus.runtime.Quarkus;\n 8  import io.quarkus.runtime.QuarkusApplication;\n 9  import io.quarkus.runtime.annotations.QuarkusMain;\n10  import javax.inject.Inject;\n11  \n12  @QuarkusMain\n13  public class QuarkusOperator implements QuarkusApplication {\n14     @Inject"
         lineNumber: 4
         variables:
-          file: file:///java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
+          file: file:///examples/java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
           kind: Module
           name: io.javaoperatorsdk.operator.Operator
           package: io.javaoperatorsdk.operator.sample
-      - uri: file:///java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
+      - uri: file:///examples/java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
         message: ""
         codeSnip: " 7  import io.quarkus.runtime.Quarkus;\n 8  import io.quarkus.runtime.QuarkusApplication;\n 9  import io.quarkus.runtime.annotations.QuarkusMain;\n10  import javax.inject.Inject;\n11  \n12  @QuarkusMain\n13  public class QuarkusOperator implements QuarkusApplication {\n14     @Inject\n15     KubernetesClient client;\n16     @Inject\n17     Operator operator;\n18     @Inject\n19     ConfigurationService configuration;\n20     @Inject\n21     CustomServiceController controller;\n22  \n23     public static void main(String... args) {\n24        Quarkus.run(QuarkusOperator.class, args);\n25     }\n26  \n27     public int run(String... args) throws Exception {"
         lineNumber: 17
         variables:
-          file: file:///java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
+          file: file:///examples/java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
           kind: Field
           name: operator
           package: io.javaoperatorsdk.operator.sample
@@ -960,6 +976,12 @@
       incidents:
       - uri: file:///examples/customers-tomcat-legacy/pom.xml
         message: ""
+      - uri: file:///examples/gradle-multi-project-example/gradle/wrapper/gradle-wrapper.jar
+        message: ""
+      - uri: file:///examples/inclusion-tests/pom.xml
+        message: ""
+      - uri: file:///examples/java-project/pom.xml
+        message: ""
       - uri: file:///examples/java/dummy/pom.xml
         message: ""
       - uri: file:///examples/java/example/pom.xml

external-providers/dotnet-external-provider/main.go

@@ -7,7 +7,7 @@ import (
 	"os"
 
 	"github.com/bombsimon/logrusr/v3"
-	"github.com/konveyor/analyzer-lsp/provider"
+	server "github.com/konveyor/analyzer-lsp/provider/server"
 	"github.com/konveyor/dotnet-external-provider/pkg/dotnet"
 	"github.com/sirupsen/logrus"
 )
@@ -55,7 +55,7 @@ func main() {
 		secret = *secretKey
 	}
 
-	s := provider.NewServer(client, *port, c, k, secret, log)
+	s := server.NewServer(client, *port, c, k, secret, 0, log)
 	ctx := context.TODO()
 	s.Start(ctx)
 }

external-providers/dotnet-external-provider/pkg/dotnet/provider.go

@@ -65,10 +65,10 @@ func (p *dotnetProvider) Capabilities() []provider.Capability {
 	return caps
 }
 
-func (p *dotnetProvider) Init(ctx context.Context, log logr.Logger, config provider.InitConfig) (provider.ServiceClient, error) {
+func (p *dotnetProvider) Init(ctx context.Context, log logr.Logger, config provider.InitConfig) (provider.ServiceClient, provider.InitConfig, error) {
 	var mode provider.AnalysisMode = provider.AnalysisMode(config.AnalysisMode)
 	if mode != provider.SourceOnlyAnalysisMode {
-		return nil, fmt.Errorf("only full analysis is supported")
+		return nil, provider.InitConfig{}, fmt.Errorf("only full analysis is supported")
 	}
 
 	// handle proxy settings
@@ -79,7 +79,7 @@ func (p *dotnetProvider) Init(ctx context.Context, log logr.Logger, config provi
 	codePath, err := filepath.Abs(config.Location)
 	if err != nil {
 		log.Error(err, "unable to get path to analyze")
-		return nil, err
+		return nil, provider.InitConfig{}, err
 	}
 
 	ctx, cancelFunc := context.WithCancel(ctx)
@@ -91,27 +91,27 @@ func (p *dotnetProvider) Init(ctx context.Context, log logr.Logger, config provi
 	lspServerPath, ok := config.ProviderSpecificConfig[provider.LspServerPathConfigKey].(string)
 	if !ok || lspServerPath == "" {
 		cancelFunc()
-		return nil, fmt.Errorf("invalid lspServerPath provided, unable to init dotnet provider")
+		return nil, provider.InitConfig{}, fmt.Errorf("invalid lspServerPath provided, unable to init dotnet provider")
 	}
 
 	cmd := exec.CommandContext(ctx, lspServerPath)
 	cmd.Dir = codePath // At a minimum, 'csharp-ls' doesn't respect URI @initialization
 	stdin, err := cmd.StdinPipe()
 	if err != nil {
 		cancelFunc()
-		return nil, err
+		return nil, provider.InitConfig{}, err
 	}
 	clientWriter := io.MultiWriter(stdin, sentLog)
 	stdout, err := cmd.StdoutPipe()
 	if err != nil {
 		cancelFunc()
-		return nil, err
+		return nil, provider.InitConfig{}, err
 	}
 	clientReader := io.TeeReader(stdout, recvLog)
 	if err := cmd.Start(); err != nil {
 		log.Error(err, "failed to start language server process")
 		cancelFunc()
-		return nil, err
+		return nil, provider.InitConfig{}, err
 	}
 	log.V(2).Info("language server started")
 
@@ -175,7 +175,7 @@ func (p *dotnetProvider) Init(ctx context.Context, log logr.Logger, config provi
 	if err := conn.Notify(ctx, protocol.MethodInitialized, &protocol.InitializedParams{}); err != nil {
 		log.Error(err, "initialized notification failed")
 		cancelFunc()
-		return nil, err
+		return nil, provider.InitConfig{}, err
 	}
 
 	log.Info("waiting for language server to load the project")
@@ -189,5 +189,5 @@ func (p *dotnetProvider) Init(ctx context.Context, log logr.Logger, config provi
 		cmd:        cmd,
 		log:        log,
 		config:     config,
-	}, nil
+	}, provider.InitConfig{}, nil
 }

external-providers/generic-external-provider/main.go

@@ -7,7 +7,7 @@ import (
 	"os"
 
 	"github.com/bombsimon/logrusr/v3"
-	"github.com/konveyor/analyzer-lsp/provider"
+	"github.com/konveyor/analyzer-lsp/provider/server"
 	"github.com/konveyor/generic-external-provider/pkg/generic_external_provider"
 	"github.com/sirupsen/logrus"
 )
@@ -72,7 +72,7 @@ func main() {
 		secret = *secretKey
 	}
 
-	s := provider.NewServer(client, *port, c, k, secret, log)
+	s := server.NewServer(client, *port, c, k, secret, 0, log)
 	ctx := context.TODO()
 	s.Start(ctx)
 }

external-providers/generic-external-provider/pkg/generic_external_provider/provider.go

@@ -64,7 +64,7 @@ func (p *genericProvider) Capabilities() []provider.Capability {
 // with the rest of the analyzer-lsp. For example, this Init() function returns
 // a provider.ServiceClient to the original analyzer process. genericProvider
 // here just sort of... doesn't matter at all
-func (p *genericProvider) Init(ctx context.Context, log logr.Logger, c provider.InitConfig) (provider.ServiceClient, error) {
+func (p *genericProvider) Init(ctx context.Context, log logr.Logger, c provider.InitConfig) (provider.ServiceClient, provider.InitConfig, error) {
 	// return nil, fmt.Errorf("nothing")
 
 	log.Error(fmt.Errorf("Nothing"), "Started generic provider init")
@@ -78,7 +78,7 @@ func (p *genericProvider) Init(ctx context.Context, log logr.Logger, c provider.
 		log.Error(fmt.Errorf("lspServerName must be the same for each instantiation of the generic-external-provider (%s != %s)", p.lspServerName, lspServerName), "Inside genericProvider init")
 		fmt.Fprintf(os.Stderr, "lspservername blah")
 
-		return nil, fmt.Errorf("lspServerName must be the same for each instantiation of the generic-external-provider (%s != %s)", p.lspServerName, lspServerName)
+		return nil, provider.InitConfig{}, fmt.Errorf("lspServerName must be the same for each instantiation of the generic-external-provider (%s != %s)", p.lspServerName, lspServerName)
 	}
 
 	// Simple matter of calling the constructor that we set earlier to get the
@@ -87,8 +87,8 @@ func (p *genericProvider) Init(ctx context.Context, log logr.Logger, c provider.
 	if err != nil {
 		log.Error(err, "ctor error")
 		fmt.Fprintf(os.Stderr, "ctor blah")
-		return nil, fmt.Errorf("ctor error: %w", err)
+		return nil, provider.InitConfig{}, fmt.Errorf("ctor error: %w", err)
 	}
 
-	return sc, nil
+	return sc, provider.InitConfig{}, nil
 }

external-providers/generic-external-provider/pkg/server_configurations/generic/service_client_test.go

@@ -67,7 +67,7 @@ func TestHopefullyNothingBroke(t *testing.T) {
 
 	goplsExamples := "file://" + filepath.Join(examplesDir, "/golang")
 
-	goplsSC, err := prov.Init(ctx, log, provider.InitConfig{
+	goplsSC, _, err := prov.Init(ctx, log, provider.InitConfig{
 		ProviderSpecificConfig: map[string]interface{}{
 			"lspServerName":    "generic",
 			"lspServerPath":    "gopls",

external-providers/generic-external-provider/pkg/server_configurations/pylsp/service_client_test.go

@@ -37,7 +37,7 @@ func TestHopefullyNothingBroke(t *testing.T) {
 
 	prov := generic_external_provider.NewGenericProvider("generic", log)
 
-	pylspSC, err := prov.Init(ctx, log, provider.InitConfig{
+	pylspSC, _, err := prov.Init(ctx, log, provider.InitConfig{
 		ProviderSpecificConfig: map[string]interface{}{
 			"lspServerName":     "generic",
 			"lspServerPath":     "pylsp",

external-providers/generic-external-provider/pkg/server_configurations/yaml_language_server/service_client_test.go

@@ -33,7 +33,7 @@ func TestHopefullyNothingBroke(t *testing.T) {
 
 	prov := generic_external_provider.NewGenericProvider("yaml_language_server", log)
 
-	yamlSC, err := prov.Init(ctx, log, provider.InitConfig{
+	yamlSC, _, err := prov.Init(ctx, log, provider.InitConfig{
 		ProviderSpecificConfig: map[string]interface{}{
 			"lspServerName":    "yaml_language_server",
 			"lspServerPath":    "node",

external-providers/java-external-provider/go.mod

@@ -18,7 +18,18 @@ require (
 )
 
 require (
+	cloud.google.com/go/compute v1.23.3 // indirect
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	github.com/antchfx/jsonquery v1.3.0 // indirect
+	github.com/antchfx/xmlquery v1.3.12 // indirect
+	github.com/antchfx/xpath v1.2.4 // indirect
+	github.com/bufbuild/protocompile v0.10.0 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/jhump/protoreflect v1.16.0 // indirect
+	github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 // indirect
+	golang.org/x/oauth2 v0.16.0 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect
 )
 
@@ -41,3 +52,5 @@ require (
 	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/protobuf v1.33.1-0.20240408130810-98873a205002 // indirect
 )
+
+replace github.com/konveyor/analyzer-lsp => ../../