Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add custom repository support for selective repositories #41177

Merged
merged 18 commits into from
Aug 16, 2023
Merged

Add custom repository support for selective repositories #41177

Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
83bec0a
Add support to pull command
Thevakumar-Luheerathan Jul 25, 2023
cd7cd77
Add resolution support for custom repos
Thevakumar-Luheerathan Jul 26, 2023
995bafa
Add changes for DefaultPackageResolver
Thevakumar-Luheerathan Jul 28, 2023
3739dc4
Add support to the push command
Thevakumar-Luheerathan Jul 31, 2023
8efa456
Add proxy support
Thevakumar-Luheerathan Jul 31, 2023
f167ee8
Refactor CustomPackageRepository
Thevakumar-Luheerathan Aug 2, 2023
34eb2e7
Add tests
Thevakumar-Luheerathan Aug 7, 2023
0d6028f
Merge remote-tracking branch 'upstream/master' into fix-ballerina-lan…
Thevakumar-Luheerathan Aug 7, 2023
d34db6a
Add tests for PushCommand
Thevakumar-Luheerathan Aug 7, 2023
8370a4e
Add tests for CustomPackageRepository
Thevakumar-Luheerathan Aug 8, 2023
a4f8825
Refactor Settings.toml creation
Thevakumar-Luheerathan Aug 8, 2023
5432753
Fix build failure
Thevakumar-Luheerathan Aug 9, 2023
7af69c9
Fix build failure
Thevakumar-Luheerathan Aug 10, 2023
a4514eb
Address the code review
Thevakumar-Luheerathan Aug 11, 2023
ec3a681
Refactor the MavenPackageRepository
Thevakumar-Luheerathan Aug 14, 2023
7646f99
Add help text
Thevakumar-Luheerathan Aug 14, 2023
60b1bb0
Modify settings schema
Thevakumar-Luheerathan Aug 15, 2023
64982e4
Address the suggestions
Thevakumar-Luheerathan Aug 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions cli/ballerina-cli/spotbugs-exclude.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,14 @@
-->

<FindBugsFilter>
<Match>
<Class name="io.ballerina.cli.cmd.PullCommand"/>
<Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"/>
</Match>
<Match>
<Class name="io.ballerina.cli.cmd.PullCommand"/>
<Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE"/>
</Match>
<Match>
<Class name="io.ballerina.cli.cmd.BuildCommand"/>
<Bug pattern="URF_UNREAD_FIELD"/>
Expand Down
105 changes: 97 additions & 8 deletions cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PullCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,28 +18,41 @@

package io.ballerina.cli.cmd;

import com.google.gson.Gson;
import com.google.gson.JsonObject;
import io.ballerina.cli.BLauncherCmd;
import io.ballerina.projects.ProjectException;
import io.ballerina.projects.SemanticVersion;
import io.ballerina.projects.Settings;
import io.ballerina.projects.internal.model.Proxy;
import io.ballerina.projects.internal.model.Repository;
import io.ballerina.projects.util.ProjectConstants;
import io.ballerina.projects.util.ProjectUtils;
import org.ballerinalang.central.client.CentralAPIClient;
import org.ballerinalang.central.client.CentralClientConstants;
import org.ballerinalang.central.client.exceptions.CentralClientException;
import org.ballerinalang.central.client.exceptions.PackageAlreadyExistsException;
import org.ballerinalang.maven.bala.client.MavenResolverClient;
import org.ballerinalang.maven.bala.client.MavenResolverClientException;
import org.ballerinalang.toml.exceptions.SettingsTomlException;
import org.wso2.ballerinalang.compiler.util.Names;
import org.wso2.ballerinalang.util.RepoUtils;
import picocli.CommandLine;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Comparator;
import java.util.List;

import static io.ballerina.cli.cmd.Constants.PULL_COMMAND;
import static io.ballerina.cli.launcher.LauncherUtils.createLauncherException;
import static io.ballerina.projects.util.ProjectConstants.BALA_EXTENSION;
import static io.ballerina.projects.util.ProjectConstants.PLATFORM;
import static io.ballerina.projects.util.ProjectUtils.getAccessTokenOfCLI;
import static io.ballerina.projects.util.ProjectUtils.initializeProxy;
import static io.ballerina.projects.util.ProjectUtils.validateOrgName;
Expand Down Expand Up @@ -71,6 +84,9 @@ public class PullCommand implements BLauncherCmd {
@CommandLine.Option(names = "--debug", hidden = true)
private String debugPort;

@CommandLine.Option(names = "--repository")
private String repositoryName;

public PullCommand() {
this.errStream = System.err;
this.exitWhenFinish = true;
Expand Down Expand Up @@ -163,6 +179,87 @@ public void execute() {
}
}

Settings settings;
try {
settings = RepoUtils.readSettings();
} catch (SettingsTomlException e) {
settings = Settings.from();
}

Repository targetRepository = null;
if (repositoryName != null) {
for (Repository repository : settings.getRepositories()) {
if (repository.id().equals(repositoryName)) {
Thevakumar-Luheerathan marked this conversation as resolved.
Show resolved Hide resolved
targetRepository = repository;
break;
}
}
}

if (targetRepository == null && repositoryName != null) {
String errMsg = "unsupported repository '" + repositoryName + "' found. Only " +
"repositories mentioned in the Settings.toml are supported.";
CommandUtil.printError(this.errStream, errMsg, null, false);
CommandUtil.exitError(this.exitWhenFinish);
return;
}

if (targetRepository != null) {
MavenResolverClient mavenResolverClient = new MavenResolverClient();
if (!"".equals(targetRepository.username()) && !"".equals(targetRepository.password())) {
mavenResolverClient.addRepository(targetRepository.id(), targetRepository.url(),
targetRepository.username(), targetRepository.password());
} else {
mavenResolverClient.addRepository(targetRepository.id(), targetRepository.url());
}
Proxy proxy = settings.getProxy();
mavenResolverClient.setProxy(proxy.host(), proxy.port(), proxy.username(), proxy.password());

Path mavenBalaCachePath = RepoUtils.createAndGetHomeReposPath()
.resolve(ProjectConstants.REPOSITORIES_DIR)
.resolve(targetRepository.id())
.resolve(ProjectConstants.BALA_DIR_NAME);

try {
mavenResolverClient.pullPackage(orgName, packageName, version,
String.valueOf(mavenBalaCachePath.toAbsolutePath()));
Path balaDownloadPath = mavenBalaCachePath.resolve(orgName).resolve(packageName).resolve(version)
.resolve(packageName + "-" + version + BALA_EXTENSION);
Path balaHashPath = mavenBalaCachePath.resolve(orgName).resolve(packageName).resolve(version)
.resolve(packageName + "-" + version + BALA_EXTENSION + ".sha1");
Path temporaryExtractionPath = mavenBalaCachePath.resolve(orgName).resolve(packageName)
.resolve(version).resolve(PLATFORM);
ProjectUtils.extractBala(balaDownloadPath, temporaryExtractionPath);
Path packageJsonPath = temporaryExtractionPath.resolve("package.json");
BufferedReader bufferedReader = Files.newBufferedReader(packageJsonPath, StandardCharsets.UTF_8);
Thevakumar-Luheerathan marked this conversation as resolved.
Show resolved Hide resolved
JsonObject resultObj = new Gson().fromJson(bufferedReader, JsonObject.class);
String platform = resultObj.get(PLATFORM).getAsString();
Path actualBalaPath = mavenBalaCachePath.resolve(orgName).resolve(packageName)
.resolve(version).resolve(platform);
temporaryExtractionPath.toFile().renameTo(actualBalaPath.toFile());

Files.delete(balaDownloadPath);
Files.delete(balaHashPath);
Files.delete(balaDownloadPath.getParent().resolve("_remote.repositories"));
if (Files.exists(temporaryExtractionPath)) {
Files.walk(temporaryExtractionPath)
.sorted(Comparator.reverseOrder())
.map(Path::toFile)
.forEach(File::delete);
}

} catch (MavenResolverClientException e) {
errStream.println("unexpected error occurred while pulling package:" + e.getMessage());
CommandUtil.exitError(this.exitWhenFinish);
} catch (IOException e) {
throw createLauncherException(
"unexpected error occurred while creating package repository in bala cache: " + e.getMessage());
}
PrintStream out = System.out;
out.println("Successfully pulled the package from the custom repository.");
return;
}

Path packagePathInBalaCache = ProjectUtils.createAndGetHomeReposPath()
.resolve(ProjectConstants.REPOSITORIES_DIR).resolve(ProjectConstants.CENTRAL_REPOSITORY_CACHE_NAME)
.resolve(ProjectConstants.BALA_DIR_NAME)
Expand All @@ -180,14 +277,6 @@ public void execute() {
for (int i = 0; i < SUPPORTED_PLATFORMS.length; i++) {
String supportedPlatform = SUPPORTED_PLATFORMS[i];
try {
Settings settings;
try {
settings = RepoUtils.readSettings();
// Ignore Settings.toml diagnostics in the pull command
} catch (SettingsTomlException e) {
// Ignore 'Settings.toml' parsing errors and return empty Settings object
settings = Settings.from();
}
CentralAPIClient client = new CentralAPIClient(RepoUtils.getRemoteRepoURL(),
initializeProxy(settings.getProxy()), settings.getProxy().username(),
settings.getProxy().password(), getAccessTokenOfCLI(settings));
Expand Down
106 changes: 101 additions & 5 deletions cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PushCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,17 @@
import io.ballerina.projects.Settings;
import io.ballerina.projects.bala.BalaProject;
import io.ballerina.projects.directory.BuildProject;
import io.ballerina.projects.internal.model.Proxy;
import io.ballerina.projects.internal.model.Repository;
import io.ballerina.projects.repos.TempDirCompilationCache;
import io.ballerina.projects.util.ProjectConstants;
import io.ballerina.projects.util.ProjectUtils;
import org.ballerinalang.central.client.CentralAPIClient;
import org.ballerinalang.central.client.CentralClientConstants;
import org.ballerinalang.central.client.exceptions.CentralClientException;
import org.ballerinalang.central.client.exceptions.NoPackageException;
import org.ballerinalang.maven.bala.client.MavenResolverClient;
import org.ballerinalang.maven.bala.client.MavenResolverClientException;
import org.ballerinalang.toml.exceptions.SettingsTomlException;
import org.wso2.ballerinalang.util.RepoUtils;
import picocli.CommandLine;
Expand All @@ -48,6 +52,7 @@
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
Expand All @@ -61,6 +66,7 @@
import static io.ballerina.projects.util.ProjectUtils.getAccessTokenOfCLI;
import static io.ballerina.projects.util.ProjectUtils.initializeProxy;
import static io.ballerina.runtime.api.constants.RuntimeConstants.SYSTEM_PROP_BAL_DEBUG;
import static java.nio.file.Files.createDirectories;
import static org.wso2.ballerinalang.programfile.ProgramFileConstants.SUPPORTED_PLATFORMS;

/**
Expand Down Expand Up @@ -145,19 +151,33 @@ public void execute() {
System.setProperty(CentralClientConstants.ENABLE_OUTPUT_STREAM, "true");

try {
Settings settings = RepoUtils.readSettings();

// If the repository flag is specified, validate and push to the provided repo
if (repositoryName != null) {
if (!repositoryName.equals(ProjectConstants.LOCAL_REPOSITORY_NAME)) {
boolean isCustomRepository = false;
Repository targetRepository = null;
for (Repository repository : settings.getRepositories()) {
if (repository.id().equals(repositoryName)) {
isCustomRepository = true;
targetRepository = repository;
break;
}
}

if (!repositoryName.equals(ProjectConstants.LOCAL_REPOSITORY_NAME) && !isCustomRepository) {
String errMsg = "unsupported repository '" + repositoryName + "' found. Only '"
+ ProjectConstants.LOCAL_REPOSITORY_NAME + "' repository is supported.";
+ ProjectConstants.LOCAL_REPOSITORY_NAME +
"' repository and repositories mentioned in the Settings.toml are supported.";
CommandUtil.printError(this.errStream, errMsg, null, false);
CommandUtil.exitError(this.exitWhenFinish);
return;
}

if (balaPath == null) {
if (balaPath == null && repositoryName.equals(ProjectConstants.LOCAL_REPOSITORY_NAME)) {
pushPackage(project);
} else {
return;
} else if (repositoryName.equals(ProjectConstants.LOCAL_REPOSITORY_NAME)) {
if (!balaPath.toFile().exists()) {
throw new ProjectException("path provided for the bala file does not exist: " + balaPath + ".");
}
Expand All @@ -166,9 +186,41 @@ public void execute() {
}
validatePackageMdAndBalToml(balaPath);
pushBalaToCustomRepo(balaPath);
return;
}

MavenResolverClient mvnClient = new MavenResolverClient();
if (!"".equals(targetRepository.username()) && !"".equals(targetRepository.password())) {
Thevakumar-Luheerathan marked this conversation as resolved.
Show resolved Hide resolved
mvnClient.addRepository(targetRepository.id(), targetRepository.url(), targetRepository.username(),
targetRepository.password());
} else {
mvnClient.addRepository(targetRepository.id(), targetRepository.url());
}
Proxy proxy = settings.getProxy();
mvnClient.setProxy(proxy.host(), proxy.port(), proxy.username(), proxy.password());

Path customRepoLocalCache = RepoUtils.createAndGetHomeReposPath()
.resolve(ProjectConstants.REPOSITORIES_DIR)
.resolve(targetRepository.id())
.resolve(ProjectConstants.BALA_DIR_NAME)
.resolve("temp");


if (balaPath == null && isCustomRepository) {
pushPackage(project, mvnClient, customRepoLocalCache);
} else if (isCustomRepository) {
if (!balaPath.toFile().exists()) {
throw new ProjectException("path provided for the bala file does not exist: " + balaPath + ".");
}
if (!FileUtils.getExtension(balaPath).equals("bala")) {
throw new ProjectException("file provided is not a bala file: " + balaPath + ".");
}
validatePackageMdAndBalToml(balaPath);
pushBalaToCustomRepo(balaPath, mvnClient, customRepoLocalCache);
}


} else {
Settings settings = RepoUtils.readSettings();
if (settings.diagnostics().hasErrors()) {
CommandUtil.printError(this.errStream, settings.getErrorMessage(), null, false);
CommandUtil.exitError(this.exitWhenFinish);
Expand Down Expand Up @@ -226,6 +278,11 @@ private void pushPackage(BuildProject project) {
pushBalaToCustomRepo(balaFilePath);
}

private void pushPackage(BuildProject project, MavenResolverClient client, Path customRepoPath) {
Path balaFilePath = validateBalaFile(project, this.balaPath);
pushBalaToCustomRepo(balaFilePath, client, customRepoPath);
}

private void pushPackage(BuildProject project, CentralAPIClient client)
throws CentralClientException {
Path balaFilePath = validateBala(project, client, this.balaPath);
Expand Down Expand Up @@ -402,6 +459,45 @@ private void pushBalaToRemote(Path balaPath, CentralAPIClient client) {
}
}

private void pushBalaToCustomRepo(Path balaPath, MavenResolverClient client, Path customRepoPath) {
try {
createDirectories(customRepoPath);
} catch (IOException e) {
throw new ProjectException(e.getMessage());
}
Path balaFileName = balaPath.getFileName();
if (null != balaFileName) {
ProjectEnvironmentBuilder defaultBuilder = ProjectEnvironmentBuilder.getDefaultBuilder();
defaultBuilder.addCompilationCacheFactory(TempDirCompilationCache::from);
BalaProject balaProject = BalaProject.loadProject(defaultBuilder, balaPath);

String org = balaProject.currentPackage().manifest().org().toString();
String name = balaProject.currentPackage().manifest().name().toString();
String version = balaProject.currentPackage().manifest().version().toString();

try {
client.pushPackage(balaPath, org, name, version, customRepoPath);
if (Files.exists(customRepoPath)) {
Files.walk(customRepoPath)
.sorted(Comparator.reverseOrder())
.map(Path::toFile)
.forEach(File::delete);
}
} catch (MavenResolverClientException | IOException e) {
throw new ProjectException(e.getMessage());
}

Path relativePathToBalaFile;
if (this.balaPath != null) {
relativePathToBalaFile = balaPath;
} else {
relativePathToBalaFile = userDir.relativize(balaPath);
}
outStream.println("Successfully pushed " + relativePathToBalaFile
+ " to '" + repositoryName + "' repository.");
}
}

/**
* Check if package already available in the remote.
*
Expand Down
Loading