Add custom repository support for selective repositories

cli/ballerina-cli/build.gradle

@@ -58,6 +58,7 @@ dependencies {
     implementation group: 'org.ow2.asm', name: 'asm', version: "${project.ow2AsmVersion}"
     implementation group: 'org.ow2.asm', name: 'asm-commons', version: "${project.ow2AsmCommonsVersion}"
     implementation group: 'org.ow2.asm', name: 'asm-tree', version: "${project.ow2AsmTreeVersion}"
+    implementation 'commons-io:commons-io'
 
     testImplementation "org.testng:testng:${project.testngVersion}"
     testImplementation "org.mockito:mockito-core:${project.mockitoCoreVersion}"

cli/ballerina-cli/spotbugs-exclude.xml

@@ -17,6 +17,14 @@
   -->
 
 <FindBugsFilter>
+    <Match>
+        <Class name="io.ballerina.cli.cmd.PullCommand"/>
+        <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"/>
+    </Match>
+    <Match>
+        <Class name="io.ballerina.cli.cmd.PullCommand"/>
+        <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE"/>
+    </Match>
     <Match>
         <Class name="io.ballerina.cli.cmd.BuildCommand"/>
         <Bug pattern="URF_UNREAD_FIELD"/>

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PullCommand.java

@@ -18,28 +18,39 @@
 
 package io.ballerina.cli.cmd;
 
+import com.google.gson.Gson;
+import com.google.gson.JsonObject;
 import io.ballerina.cli.BLauncherCmd;
 import io.ballerina.projects.ProjectException;
 import io.ballerina.projects.SemanticVersion;
 import io.ballerina.projects.Settings;
+import io.ballerina.projects.internal.model.Proxy;
+import io.ballerina.projects.internal.model.Repository;
 import io.ballerina.projects.util.ProjectConstants;
 import io.ballerina.projects.util.ProjectUtils;
 import org.ballerinalang.central.client.CentralAPIClient;
 import org.ballerinalang.central.client.CentralClientConstants;
 import org.ballerinalang.central.client.exceptions.CentralClientException;
 import org.ballerinalang.central.client.exceptions.PackageAlreadyExistsException;
+import org.ballerinalang.maven.bala.client.MavenResolverClient;
+import org.ballerinalang.maven.bala.client.MavenResolverClientException;
 import org.ballerinalang.toml.exceptions.SettingsTomlException;
 import org.wso2.ballerinalang.compiler.util.Names;
 import org.wso2.ballerinalang.util.RepoUtils;
 import picocli.CommandLine;
 
+import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;
 
 import static io.ballerina.cli.cmd.Constants.PULL_COMMAND;
 import static io.ballerina.cli.launcher.LauncherUtils.createLauncherException;
+import static io.ballerina.projects.util.ProjectConstants.BALA_EXTENSION;
+import static io.ballerina.projects.util.ProjectConstants.PLATFORM;
 import static io.ballerina.projects.util.ProjectUtils.getAccessTokenOfCLI;
 import static io.ballerina.projects.util.ProjectUtils.initializeProxy;
 import static io.ballerina.projects.util.ProjectUtils.validateOrgName;
@@ -71,6 +82,9 @@
     @CommandLine.Option(names = "--debug", hidden = true)
     private String debugPort;
 
+    @CommandLine.Option(names = "--repository")
+    private String repositoryName;
+
     public PullCommand() {
         this.errStream = System.err;
         this.exitWhenFinish = true;
@@ -163,6 +177,77 @@
             }
         }
 
+        Settings settings;
+        try {
+            settings = RepoUtils.readSettings();
+        } catch (SettingsTomlException e) {
+            settings = Settings.from();
+        }
+
+        Repository targetRepository = null;
+        if (repositoryName != null) {
+            for (Repository repository : settings.getRepositories()) {
+                if (repositoryName.equals(repository.id())) {
+                    targetRepository = repository;
+                    break;
+                }
+            }
+        }
+
+        if (targetRepository == null && repositoryName != null) {
+            String errMsg = "unsupported repository '" + repositoryName + "' found. Only " +
+                    "repositories mentioned in the Settings.toml are supported.";
+            CommandUtil.printError(this.errStream, errMsg, null, false);
+            CommandUtil.exitError(this.exitWhenFinish);
+            return;
+        }
+
+        if (targetRepository != null) {
+            MavenResolverClient mavenResolverClient = new MavenResolverClient();
+            if (!targetRepository.username().isEmpty() && !targetRepository.password().isEmpty()) {
+                mavenResolverClient.addRepository(targetRepository.id(), targetRepository.url(),
+                        targetRepository.username(), targetRepository.password());
+            } else {
+                mavenResolverClient.addRepository(targetRepository.id(), targetRepository.url());
+            }
+            Proxy proxy = settings.getProxy();
+            mavenResolverClient.setProxy(proxy.host(), proxy.port(), proxy.username(), proxy.password());
+
+            Path mavenBalaCachePath = RepoUtils.createAndGetHomeReposPath()
+                    .resolve(ProjectConstants.REPOSITORIES_DIR)
+                    .resolve(targetRepository.id())
+                    .resolve(ProjectConstants.BALA_DIR_NAME);
+
+            try {
+                Path tmpDownloadDirectory = Files.createTempDirectory("ballerina-" + System.nanoTime());
+                mavenResolverClient.pullPackage(orgName, packageName, version,
+                        String.valueOf(tmpDownloadDirectory.toAbsolutePath()));
+                Path balaDownloadPath = tmpDownloadDirectory.resolve(orgName).resolve(packageName).resolve(version)
+                        .resolve(packageName + "-" + version + BALA_EXTENSION);
+                Path temporaryExtractionPath = tmpDownloadDirectory.resolve(orgName).resolve(packageName)
+                        .resolve(version).resolve(PLATFORM);
+                ProjectUtils.extractBala(balaDownloadPath, temporaryExtractionPath);
+                Path packageJsonPath = temporaryExtractionPath.resolve("package.json");
+                try (BufferedReader bufferedReader = Files.newBufferedReader(packageJsonPath, StandardCharsets.UTF_8)) {
+                    JsonObject resultObj = new Gson().fromJson(bufferedReader, JsonObject.class);
+                    String platform = resultObj.get(PLATFORM).getAsString();
+                    Path actualBalaPath = mavenBalaCachePath.resolve(orgName).resolve(packageName)
+                            .resolve(version).resolve(platform);
+                    org.apache.commons.io.FileUtils.copyDirectory(temporaryExtractionPath.toFile(),
+                            actualBalaPath.toFile());
+                }
+            } catch (MavenResolverClientException e) {
+                errStream.println("unexpected error occurred while pulling package:" + e.getMessage());
+                CommandUtil.exitError(this.exitWhenFinish);
+            } catch (IOException e) {
+                throw createLauncherException(
+                        "unexpected error occurred while creating package repository in bala cache: " + e.getMessage());
+            }
+            PrintStream out = System.out;
+            out.println("Successfully pulled the package from the custom repository.");
+            return;
+        }
+
         Path packagePathInBalaCache = ProjectUtils.createAndGetHomeReposPath()
                 .resolve(ProjectConstants.REPOSITORIES_DIR).resolve(ProjectConstants.CENTRAL_REPOSITORY_CACHE_NAME)
                 .resolve(ProjectConstants.BALA_DIR_NAME)
@@ -180,14 +265,6 @@
         for (int i = 0; i < SUPPORTED_PLATFORMS.length; i++) {
             String supportedPlatform = SUPPORTED_PLATFORMS[i];
             try {
-                Settings settings;
-                try {
-                    settings = RepoUtils.readSettings();
-                    // Ignore Settings.toml diagnostics in the pull command
-                } catch (SettingsTomlException e) {
-                    // Ignore 'Settings.toml' parsing errors and return empty Settings object
-                    settings = Settings.from();
-                }
                 CentralAPIClient client = new CentralAPIClient(RepoUtils.getRemoteRepoURL(),
                         initializeProxy(settings.getProxy()), settings.getProxy().username(),
                         settings.getProxy().password(), getAccessTokenOfCLI(settings));

cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java

@@ -29,13 +29,17 @@
 import io.ballerina.projects.Settings;
 import io.ballerina.projects.bala.BalaProject;
 import io.ballerina.projects.directory.BuildProject;
+import io.ballerina.projects.internal.model.Proxy;
+import io.ballerina.projects.internal.model.Repository;
 import io.ballerina.projects.repos.TempDirCompilationCache;
 import io.ballerina.projects.util.ProjectConstants;
 import io.ballerina.projects.util.ProjectUtils;
 import org.ballerinalang.central.client.CentralAPIClient;
 import org.ballerinalang.central.client.CentralClientConstants;
 import org.ballerinalang.central.client.exceptions.CentralClientException;
 import org.ballerinalang.central.client.exceptions.NoPackageException;
+import org.ballerinalang.maven.bala.client.MavenResolverClient;
+import org.ballerinalang.maven.bala.client.MavenResolverClientException;
 import org.ballerinalang.toml.exceptions.SettingsTomlException;
 import org.wso2.ballerinalang.util.RepoUtils;
 import picocli.CommandLine;
@@ -145,19 +149,33 @@
         System.setProperty(CentralClientConstants.ENABLE_OUTPUT_STREAM, "true");
 
         try {
+            Settings settings = RepoUtils.readSettings();
+
             // If the repository flag is specified, validate and push to the provided repo
             if (repositoryName != null) {
-                if (!repositoryName.equals(ProjectConstants.LOCAL_REPOSITORY_NAME)) {
+                boolean isCustomRepository = false;
+                Repository targetRepository = null;
+                for (Repository repository : settings.getRepositories()) {
+                    if (repositoryName.equals(repository.id())) {
+                        isCustomRepository = true;
+                        targetRepository = repository;
+                        break;
+                    }
+                }
+
+                if (!repositoryName.equals(ProjectConstants.LOCAL_REPOSITORY_NAME) && !isCustomRepository) {
                     String errMsg = "unsupported repository '" + repositoryName + "' found. Only '"
-                            + ProjectConstants.LOCAL_REPOSITORY_NAME + "' repository is supported.";
+                            + ProjectConstants.LOCAL_REPOSITORY_NAME +
+                            "' repository and repositories mentioned in the Settings.toml are supported.";
                     CommandUtil.printError(this.errStream, errMsg, null, false);
                     CommandUtil.exitError(this.exitWhenFinish);
                     return;
                 }
 
-                if (balaPath == null) {
+                if (balaPath == null && repositoryName.equals(ProjectConstants.LOCAL_REPOSITORY_NAME)) {
                     pushPackage(project);
-                } else {
+                    return;
+                } else if (repositoryName.equals(ProjectConstants.LOCAL_REPOSITORY_NAME)) {
                     if (!balaPath.toFile().exists()) {
                         throw new ProjectException("path provided for the bala file does not exist: " + balaPath + ".");
                     }
@@ -166,9 +184,34 @@
                     }
                     validatePackageMdAndBalToml(balaPath);
                     pushBalaToCustomRepo(balaPath);
+                    return;
+                }
+
+                MavenResolverClient mvnClient = new MavenResolverClient();
+                if (!targetRepository.username().isEmpty() && !targetRepository.password().isEmpty()) {
+                    mvnClient.addRepository(targetRepository.id(), targetRepository.url(), targetRepository.username(),
+                                            targetRepository.password());
+                } else {
+                    mvnClient.addRepository(targetRepository.id(), targetRepository.url());
                 }
+                Proxy proxy = settings.getProxy();
+                mvnClient.setProxy(proxy.host(), proxy.port(), proxy.username(), proxy.password());
+
+                if (balaPath == null && isCustomRepository) {
+                    pushPackage(project, mvnClient);
+                } else if (isCustomRepository) {
+                    if (!balaPath.toFile().exists()) {
+                        throw new ProjectException("path provided for the bala file does not exist: " + balaPath + ".");
+                    }
+                    if (!FileUtils.getExtension(balaPath).equals("bala")) {
+                        throw new ProjectException("file provided is not a bala file: " + balaPath + ".");
+                    }
+                    validatePackageMdAndBalToml(balaPath);
+                    pushBalaToCustomRepo(balaPath, mvnClient);
+                }
+
+
             } else {
-                Settings settings = RepoUtils.readSettings();
                 if (settings.diagnostics().hasErrors()) {
                     CommandUtil.printError(this.errStream, settings.getErrorMessage(), null, false);
                     CommandUtil.exitError(this.exitWhenFinish);
@@ -226,6 +269,11 @@
         pushBalaToCustomRepo(balaFilePath);
     }
 
+    private void pushPackage(BuildProject project, MavenResolverClient client) {
+        Path balaFilePath = validateBalaFile(project, this.balaPath);
+        pushBalaToCustomRepo(balaFilePath, client);
+    }
+
     private void pushPackage(BuildProject project, CentralAPIClient client)
             throws CentralClientException {
         Path balaFilePath = validateBala(project, client, this.balaPath);
@@ -402,6 +450,35 @@
         }
     }
 
+    private void pushBalaToCustomRepo(Path balaPath, MavenResolverClient client) {
+        Path balaFileName = balaPath.getFileName();
+        if (null != balaFileName) {
+            ProjectEnvironmentBuilder defaultBuilder = ProjectEnvironmentBuilder.getDefaultBuilder();
+            defaultBuilder.addCompilationCacheFactory(TempDirCompilationCache::from);
+            BalaProject balaProject = BalaProject.loadProject(defaultBuilder, balaPath);
+
+            String org = balaProject.currentPackage().manifest().org().toString();
+            String name = balaProject.currentPackage().manifest().name().toString();
+            String version = balaProject.currentPackage().manifest().version().toString();
+
+            try {
+                Path customRepoPath = Files.createTempDirectory("ballerina-" + System.nanoTime());
+                client.pushPackage(balaPath, org, name, version, customRepoPath);
+            } catch (MavenResolverClientException | IOException e) {
+                throw new ProjectException(e.getMessage());
+            }
+
+            Path relativePathToBalaFile;
+            if (this.balaPath != null) {
+                relativePathToBalaFile = balaPath;
+            } else {
+                relativePathToBalaFile = userDir.relativize(balaPath);
+            }
+            outStream.println("Successfully pushed " + relativePathToBalaFile
+                    + " to '" + repositoryName + "' repository.");
+        }
+    }
+
     /**
      * Check if package already available in the remote.
      *

cli/ballerina-cli/src/main/java/module-info.java

@@ -21,4 +21,5 @@
     requires io.ballerina.toml;
     requires io.ballerina.identifier;
     requires org.objectweb.asm;
+    requires org.apache.commons.io;
 }

cli/ballerina-cli/src/main/resources/cli-help/ballerina-pull.help

@@ -1,5 +1,6 @@
 NAME
-       ballerina-pull - Fetch packages from Ballerina Central
+       ballerina-pull - Fetch packages from Ballerina Central or a custom
+       package repository
 
 SYNOPSIS
        bal pull <org-name>/<package-name>[:<version>]
@@ -15,6 +16,14 @@ DESCRIPTION
        Organizations are unique within a repository and can be mapped to an
        individual user or organization registered with the repository.
 
+       To download a package from a custom repository, configure it in the Settings.toml
+       file and pass the given id with the --repository flag.
+
+OPTIONS
+       --repository
+           Pull a package from a custom repository.
+           The repository must be configured in the <USER_HOME>/.ballerina/Settings.toml file.
+
 
 EXAMPLES
        Pull the latest version of the 'gmail' connector in the 'wso2'
@@ -24,3 +33,7 @@ EXAMPLES
        Pull the '1.1.0' version of the 'gmail' connector in the 'wso2'
        organization from Ballerina Central.
            $ bal pull wso2/gmail:1.1.0
+
+       Pull the '1.1.0' version of the 'gmail' connector in the 'wso2'
+       organization from the github package repository defined in the Settings.toml file.
+           $ bal pull wso2/gmail:1.1.0 --repository=wso2

cli/ballerina-cli/src/main/resources/cli-help/ballerina-push.help

@@ -1,24 +1,27 @@
 NAME
        ballerina-push - Push the Ballerina Archive (BALA) of the current package
-       to Ballerina Central or to the local repository
+       to a package repository
 
 SYNOPSIS
        bal push [OPTIONS] <bala-path>
 
 
 DESCRIPTION
        Push the Ballerina archive (.bala) of the current package to Ballerina
-       Central or to a local repository. Once the package is pushed to Ballerina
+       Central, local or a custom remote repository. Once the package is pushed to Ballerina
        Central, it becomes public and sharable and will be permanent.
 
        To be able to publish a package to Ballerina Central, you should sign in
        to Ballerina Central and obtain an access token.
 
+       To be able to publish a package to a custom remote repository, it must be defined
+       in the <USER_HOME>/.ballerina/Settings.toml file.
+
 
 OPTIONS
        --repository
            Push the BALA of the current package to a custom repository.
-           Only 'local' is allowed.
+           Only 'local' and repositories specified in the Settings.toml are allowed.
 
 EXAMPLES
        Push the BALA of the current package to Ballerina Central.