Optimize CodeGen for the executable JAR

.github/workflows/pull_request_full_build.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - master
+      - feature-codegen-optimizer
 
 jobs:
   build-lang:

.github/workflows/pull_request_ubuntu_build.yml

@@ -15,6 +15,7 @@ on:
       - native-build
       - revert-client-decl-master
       - query-grouping-aggregation
+      - feature-codegen-optimizer
 
 jobs:
   ubuntu_build:

.github/workflows/pull_request_windows_build.yml

@@ -15,6 +15,8 @@ on:
       - native-build
       - revert-client-decl-master
       - query-grouping-aggregation
+      - feature-codegen-optimizer
+
 jobs:
   windows_build:
     name: Build with some tests on Windows

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/BuildCommand.java

@@ -209,6 +209,14 @@ public BuildCommand() {
     @CommandLine.Option(names = "--optimize-dependency-compilation", hidden = true,
             description = "experimental memory optimization for large projects")
     private Boolean optimizeDependencyCompilation;
+    @CommandLine.Option(names = "--eliminate-dead-code", description = "eliminate dead code in generated executable",
+            defaultValue = "false")
+    private Boolean deadCodeElimination;
+
+    @CommandLine.Option(names = "--dead-code-elimination-report",
+            description = "generate a report containing which sections were removed as part of dead code elimination",
+            defaultValue = "false")
+    private Boolean deadCodeEliminationReport;
 
     @Override
     public void execute() {
@@ -291,8 +299,8 @@ public void execute() {
                 // resolve maven dependencies in Ballerina.toml
                 .addTask(new ResolveMavenDependenciesTask(outStream))
                 // compile the modules
-                .addTask(new CompileTask(outStream, errStream, false, true,
-                        isPackageModified, buildOptions.enableCache()))
+                .addTask(new CompileTask(outStream, errStream, false, true, false, isPackageModified,
+                        buildOptions.enableCache()))
                 .addTask(new CreateExecutableTask(outStream, this.output, null, false))
                 .addTask(new DumpBuildTimeTask(outStream), !project.buildOptions().dumpBuildTime())
                 .build();
@@ -326,7 +334,10 @@ private BuildOptions constructBuildOptions() {
                 .disableSyntaxTreeCaching(disableSyntaxTreeCaching)
                 .setGraalVMBuildOptions(graalVMBuildOptions)
                 .setShowDependencyDiagnostics(showDependencyDiagnostics)
-                .setOptimizeDependencyCompilation(optimizeDependencyCompilation);
+                .setOptimizeDependencyCompilation(optimizeDependencyCompilation)
+                .setShowDependencyDiagnostics(showDependencyDiagnostics)
+                .setEliminateDeadCode(deadCodeElimination)
+                .setDeadCodeEliminationReport(deadCodeEliminationReport);
 
         if (targetDir != null) {
             buildOptionsBuilder.targetDir(targetDir.toString());

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PackCommand.java

@@ -263,8 +263,8 @@ public void execute() {
                 .addTask(new CleanTargetDirTask(isPackageModified, buildOptions.enableCache()), isSingleFileBuild)
                 .addTask(new RunBuildToolsTask(outStream), isSingleFileBuild)
                 .addTask(new ResolveMavenDependenciesTask(outStream))
-                .addTask(new CompileTask(outStream, errStream, true, false,
-                        isPackageModified, buildOptions.enableCache()))
+                .addTask(new CompileTask(outStream, errStream, true, false, false, isPackageModified,
+                        buildOptions.enableCache()))
                 .addTask(new CreateBalaTask(outStream))
                 .addTask(new DumpBuildTimeTask(outStream), !project.buildOptions().dumpBuildTime())
                 .build();

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/ProfileCommand.java

@@ -198,7 +198,7 @@ private TaskExecutor createTaskExecutor(boolean isPackageModified, String[] args
                 .addTask(new CleanTargetDirTask(isPackageModified, buildOptions.enableCache()), isSingleFileBuild)
                 .addTask(new RunBuildToolsTask(outStream), isSingleFileBuild)
                 .addTask(new ResolveMavenDependenciesTask(outStream))
-                .addTask(new CompileTask(outStream, errStream, false, false, isPackageModified,
+                .addTask(new CompileTask(outStream, errStream, false, false, false, isPackageModified,
                         buildOptions.enableCache()))
                 .addTask(new CreateExecutableTask(outStream, null, null, false), false)
                 .addTask(new DumpBuildTimeTask(outStream), false)

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/RunCommand.java

@@ -293,8 +293,8 @@ public void execute() {
                 // resolve maven dependencies in Ballerina.toml
                 .addTask(new ResolveMavenDependenciesTask(outStream))
                 // compile the modules
-                .addTask(new CompileTask(outStream, errStream, false, false,
-                        isPackageModified, buildOptions.enableCache()))
+                .addTask(new CompileTask(outStream, errStream, false, false, false, isPackageModified,
+                        buildOptions.enableCache()))
 //                .addTask(new CopyResourcesTask(), isSingleFileBuild)
                 .addTask(new CreateExecutableTask(outStream, null, target, true))
                 .addTask(runExecutableTask = new RunExecutableTask(args, outStream, errStream, target))

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/TestCommand.java

@@ -219,9 +219,17 @@ public TestCommand() {
     @CommandLine.Option(names = "--cloud", description = "Enable cloud artifact generation")
     private String cloud;
 
+    @CommandLine.Option(names = "--eliminate-dead-code", description = "eliminate dead code in generated executable",
+            defaultValue = "false")
+    private Boolean eliminateDeadCode;
+
     @CommandLine.Option(names = "--optimize-dependency-compilation", hidden = true,
             description = "experimental memory optimization for large projects")
     private Boolean optimizeDependencyCompilation;
+    @CommandLine.Option(names = "--dead-code-elimination-report",
+            description = "generate a report containing which sections were removed as part of dead code elimination",
+            defaultValue = "false")
+    private Boolean deadCodeEliminationReport;
 
     private static final String testCmd = "bal test [--OPTIONS]\n" +
             "                   [<ballerina-file> | <package-path>] [(-Ckey=value)...]";
@@ -387,8 +395,8 @@ public void execute() {
         TaskExecutor taskExecutor = new TaskExecutor.TaskBuilder()
                 .addTask(new ResolveMavenDependenciesTask(outStream)) // resolve maven dependencies in Ballerina.toml
                 // compile the modules
-                .addTask(new CompileTask(outStream, errStream, false, false,
-                        isPackageModified, buildOptions.enableCache()))
+                .addTask(new CompileTask(outStream, errStream, false, false, true, isPackageModified,
+                        buildOptions.enableCache()))
 //                .addTask(new CopyResourcesTask(), listGroups) // merged with CreateJarTask
                 .addTask(new CreateTestExecutableTask(outStream, groupList, disableGroupList, testList, listGroups,
                                 cliArgs, isParallelExecution),
@@ -429,7 +437,9 @@ private BuildOptions constructBuildOptions() {
                 .disableSyntaxTreeCaching(disableSyntaxTreeCaching)
                 .setGraalVMBuildOptions(graalVMBuildOptions)
                 .setShowDependencyDiagnostics(showDependencyDiagnostics)
-                .setOptimizeDependencyCompilation(optimizeDependencyCompilation);
+                .setOptimizeDependencyCompilation(optimizeDependencyCompilation)
+                .setEliminateDeadCode(eliminateDeadCode)
+                .setDeadCodeEliminationReport(deadCodeEliminationReport);
 
         if (targetDir != null) {
             buildOptionsBuilder.targetDir(targetDir.toString());

cli/ballerina-cli/src/main/java/io/ballerina/cli/task/CompileTask.java

@@ -19,6 +19,7 @@
 package io.ballerina.cli.task;
 
 import io.ballerina.cli.utils.BuildTime;
+import io.ballerina.projects.BuildOptions;
 import io.ballerina.projects.CodeGeneratorResult;
 import io.ballerina.projects.CodeModifierResult;
 import io.ballerina.projects.JBallerinaBackend;
@@ -65,23 +66,26 @@ public class CompileTask implements Task {
     private final transient PrintStream err;
     private final boolean compileForBalPack;
     private final boolean compileForBalBuild;
+    private final boolean compileForBalTest;
     private final boolean isPackageModified;
     private final boolean cachesEnabled;
 
     public CompileTask(PrintStream out, PrintStream err) {
-        this(out, err, false, false, true, false);
+        this(out, err, false, false, false, true, false);
     }
 
     public CompileTask(PrintStream out,
                        PrintStream err,
                        boolean compileForBalPack,
                        boolean compileForBalBuild,
+                       boolean compileForBalTest,
                        boolean isPackageModified,
                        boolean cachesEnabled) {
         this.out = out;
         this.err = err;
         this.compileForBalPack = compileForBalPack;
         this.compileForBalBuild = compileForBalBuild;
+        this.compileForBalTest = compileForBalTest;
         this.isPackageModified = isPackageModified;
         this.cachesEnabled = cachesEnabled;
     }
@@ -113,6 +117,10 @@ public void execute(Project project) {
             if (this.compileForBalBuild) {
                 addDiagnosticForProvidedPlatformLibs(project, diagnostics);
             }
+            if (this.compileForBalBuild || this.compileForBalTest) {
+                addDiagnosticForInvalidDeadCodeEliminationReportFlagUsage(project, diagnostics);
+            }
+
             long start = 0;
 
             if (project.currentPackage().compilationOptions().dumpGraph()
@@ -336,6 +344,19 @@ private void addDiagnosticForProvidedPlatformLibs(Project project, List<Diagnost
         }
     }
 
+    private void addDiagnosticForInvalidDeadCodeEliminationReportFlagUsage(Project project,
+                                                                           List<Diagnostic> diagnostics) {
+        BuildOptions buildOptions = project.buildOptions();
+        if (buildOptions.deadCodeEliminationReport() && !buildOptions.eliminateDeadCode()) {
+            DiagnosticInfo diagnosticInfo = new DiagnosticInfo(
+                    ProjectDiagnosticErrorCode.INVALID_VERBOSE_FLAG_USAGE.diagnosticId(),
+                    "--dead-code-elimination-report flag can only be used with --eliminate-dead-code flag",
+                    DiagnosticSeverity.ERROR);
+            diagnostics.add(new PackageDiagnostic(diagnosticInfo,
+                    project.currentPackage().descriptor().name().toString()));
+        }
+    }
+
     /**
      * If CompileTask is triggered by `bal pack` command, and project does not have CompilerPlugin.toml or BalTool.toml,
      * skip the compilation if project is empty. The project should be evaluated for emptiness before calling this.

cli/ballerina-cli/src/main/java/io/ballerina/cli/task/CreateExecutableTask.java

@@ -40,6 +40,7 @@
 import static io.ballerina.cli.launcher.LauncherUtils.createLauncherException;
 import static io.ballerina.cli.utils.FileUtils.getFileNameWithoutExtension;
 import static io.ballerina.projects.util.ProjectConstants.BLANG_COMPILED_JAR_EXT;
+import static io.ballerina.projects.util.ProjectConstants.BYTECODE_OPTIMIZED_JAR_SUFFIX;
 import static io.ballerina.projects.util.ProjectConstants.USER_DIR;
 
 /**
@@ -92,6 +93,8 @@ public void execute(Project project) {
                     out.println(warnings);
                 }
                 emitResult = jBallerinaBackend.emit(JBallerinaBackend.OutputType.GRAAL_EXEC, executablePath);
+            } else if (project.buildOptions().eliminateDeadCode()) {
+                emitResult = jBallerinaBackend.emit(JBallerinaBackend.OutputType.OPTIMIZE_CODEGEN, executablePath);
             } else {
                 emitResult = jBallerinaBackend.emit(JBallerinaBackend.OutputType.EXEC, executablePath);
             }
@@ -118,7 +121,7 @@ public void execute(Project project) {
             throw createLauncherException(e.getMessage());
         }
 
-        if (!project.buildOptions().nativeImage() && !isHideTaskOutput) {
+        if (!project.buildOptions().nativeImage() && !isHideTaskOutput && !project.buildOptions().eliminateDeadCode()) {
             Path relativePathToExecutable = currentDir.relativize(executablePath);
 
             if (project.buildOptions().getTargetPath() != null) {
@@ -133,6 +136,24 @@ public void execute(Project project) {
             }
         }
 
+        // TODO: this has lot of common code with default case, refactor to common method
+        if (project.buildOptions().eliminateDeadCode()) {
+            Path relativePathToExecutable = currentDir.relativize(executablePath);
+            String relativePathToExecutableString =
+                    relativePathToExecutable.toString().replace(BLANG_COMPILED_JAR_EXT, BYTECODE_OPTIMIZED_JAR_SUFFIX);
+            String executablePathString =
+                    executablePath.toString().replace(BLANG_COMPILED_JAR_EXT, BYTECODE_OPTIMIZED_JAR_SUFFIX);
+
+            if (project.buildOptions().getTargetPath() != null) {
+                this.out.println("\t" + relativePathToExecutableString);
+            } else if (relativePathToExecutableString.contains("..")
+                    || relativePathToExecutableString.contains("." + File.separator)) {
+                this.out.println("\t" + executablePathString);
+            } else {
+                this.out.println("\t" + relativePathToExecutableString);
+            }
+        }
+
         // notify plugin
         // todo following call has to be refactored after introducing new plugin architecture
         BuildUtils.notifyPlugins(project, target);

cli/ballerina-cli/src/main/java/io/ballerina/cli/utils/TestUtils.java

@@ -67,6 +67,7 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.StringJoiner;
+import java.util.function.Predicate;
 
 import static io.ballerina.cli.launcher.LauncherUtils.createLauncherException;
 import static org.ballerinalang.test.runtime.util.TesterinaConstants.COVERAGE_DIR;
@@ -483,8 +484,10 @@ public static String getClassPath(JBallerinaBackend jBallerinaBackend, Package c
         JarResolver jarResolver = jBallerinaBackend.jarResolver();
         Set<Path> jars = new HashSet<>(getModuleJarPaths(jBallerinaBackend, currentPackage));
 
-        List<Path> dependencies = getTestDependencyPaths(currentPackage, jarResolver)
-                .stream().filter(dependency -> !jars.contains(dependency)).toList();
+        List<Path> dependencies = getTestDependencyPaths(currentPackage, jarResolver).stream()
+                .filter(Predicate.not(jars::contains))
+                .filter(Predicate.not(jarResolver.getOptimizedJarLibraryPaths()::contains))
+                .toList();
 
         StringJoiner classPath = joinClassPaths(dependencies);
         return classPath.toString();

compiler/ballerina-lang/build.gradle

@@ -34,6 +34,7 @@ dependencies {
     }
     implementation libs.netty.buffer
     implementation libs.ow2.asm
+    implementation libs.ow2.asm.tree
     implementation libs.commons.io
     implementation libs.zafarkhaja.jsemver
     testImplementation(libs.guru.nidi.graphviz) {
@@ -76,6 +77,7 @@ processResources {
     include 'ballerina-toml-schema.json'
     include 'compiler.properties'
     include 'dependencies-toml-schema.json'
+    include 'interop-dependencies.properties'
     include 'old-dependencies-toml-schema.json'
     include 'settings-toml-schema.json'
     include 'bal-tools-toml-schema.json'

compiler/ballerina-lang/src/main/java/io/ballerina/projects/BuildOptions.java

@@ -138,6 +138,14 @@ public boolean showDependencyDiagnostics() {
         return toBooleanDefaultIfNull(this.showDependencyDiagnostics);
     }
 
+    public boolean eliminateDeadCode() {
+        return this.compilationOptions.eliminateDeadCode();
+    }
+
+    public boolean deadCodeEliminationReport() {
+        return this.compilationOptions.deadCodeEliminationReport();
+    }
+
     /**
      * Merge the given build options by favoring theirs if there are conflicts.
      *
@@ -211,6 +219,8 @@ public BuildOptions acceptTheirs(BuildOptions theirOptions) {
         buildOptionsBuilder.setExportOpenAPI(compilationOptions.exportOpenAPI);
         buildOptionsBuilder.setExportComponentModel(compilationOptions.exportComponentModel);
         buildOptionsBuilder.setEnableCache(compilationOptions.enableCache);
+        buildOptionsBuilder.setEliminateDeadCode(compilationOptions.eliminateDeadCode);
+        buildOptionsBuilder.setDeadCodeEliminationReport(compilationOptions.deadCodeEliminationReport);
         buildOptionsBuilder.setRemoteManagement(compilationOptions.remoteManagement);
         buildOptionsBuilder.setOptimizeDependencyCompilation(compilationOptions.optimizeDependencyCompilation);
 
@@ -252,7 +262,9 @@ public enum OptionName {
         EXPORT_COMPONENT_MODEL("exportComponentModel"),
         GRAAL_VM_BUILD_OPTIONS("graalvmBuildOptions"),
         SHOW_DEPENDENCY_DIAGNOSTICS("showDependencyDiagnostics"),
-        OPTIMIZE_DEPENDENCY_COMPILATION("optimizeDependencyCompilation");
+        OPTIMIZE_DEPENDENCY_COMPILATION("optimizeDependencyCompilation"),
+        ELIMINATE_DEAD_CODE("eliminateDeadCode"),
+        DEAD_CODE_ELIMINATION_REPORT("deadCodeEliminationReport");
 
         private final String name;
 
@@ -407,6 +419,11 @@ public BuildOptionsBuilder setNativeImage(Boolean value) {
             return this;
         }
 
+        public BuildOptionsBuilder setEliminateDeadCode(Boolean value) {
+            compilationOptionsBuilder.setEliminateDeadCode(value);
+            return this;
+        }
+
         public BuildOptionsBuilder setRemoteManagement(Boolean value) {
             compilationOptionsBuilder.setRemoteManagement(value);
             return this;
@@ -428,6 +445,11 @@ public BuildOptionsBuilder setOptimizeDependencyCompilation(Boolean value) {
             return this;
         }
 
+        public BuildOptionsBuilder setDeadCodeEliminationReport(Boolean value) {
+            compilationOptionsBuilder.setDeadCodeEliminationReport(value);
+            return this;
+        }
+
         public BuildOptions build() {
             CompilationOptions compilationOptions = compilationOptionsBuilder.build();
             return new BuildOptions(testReport, codeCoverage, dumpBuildTime, skipTests, compilationOptions,