Newer
Older
package fr.emse.gitlab.saref.jobs;
import java.io.Console;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
import org.apache.commons.text.StringSubstitutor;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.ListBranchCommand;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.transport.CredentialsProvider;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import fr.emse.gitlab.saref.Constants;
import fr.emse.gitlab.saref.SAREFPipelineException;
import fr.emse.gitlab.saref.entities.git.MasterVersion;
import fr.emse.gitlab.saref.entities.git.ReleaseVersion;
import fr.emse.gitlab.saref.entities.git.Repositories;
import fr.emse.gitlab.saref.entities.git.Repository;
import fr.emse.gitlab.saref.entities.git.Version;
private static final Logger LOG = LoggerFactory.getLogger(ReadRepositories.class);
public static final String REGEX_REPO_STRING = "^saref(-core|4[a-z]{4})$";
public static final Pattern REGEX_REPO_PATTERN = Pattern.compile(REGEX_REPO_STRING, Pattern.CASE_INSENSITIVE);
public static final String CONFIGURATION_FILE_NAME = ".saref-repositories.yml";
public static final String MAIN_BRANCH = "master";
public static final String SOURCE_DIR = "sources";
public ReadRepositories(String testSuiteName) {
super(testSuiteName);
public Repositories readRepositories(File directory) throws SAREFPipelineException {
final Repositories repositories = new Repositories();
final File sourceDirectory = new File(directory, Constants.TARGET_DIR + File.separator + SOURCE_DIR);
if (!Constants.PRODUCTION) {
String REGEX = "^(?<ext>saref-core|saref4[a-z][a-z][a-z][a-z])";
String name = directory.getName();
Pattern pattern = Pattern.compile(REGEX);
Matcher matcher = pattern.matcher(name);
if (!matcher.find()) {
logger.error(String.format(
"When not for production, the SAREF pipeline must be run inside a directory whose name begins with `saref-core`, or `saref4abcd` (where abcd can be any sequence of four letters). Got: %s",
name));
throw new SAREFPipelineException(String.format(
"When not for production, the SAREF pipeline must be run inside a directory whose name begins with `saref-core`, or `saref4abcd` (where abcd can be any sequence of four letters). Got: %s",
name));
}
String repositoryName = matcher.group("ext");
Repository repository = new Repository(directory, repositoryName);
repositories.setDefaultRepository(repository);
readRepository(repository);
}
final File confFile = new File(directory, CONFIGURATION_FILE_NAME);
YAMLRepos repos;
try {
ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
repos = mapper.readValue(confFile, YAMLRepos.class);
logger.warn(String.format("Error while reading the list of repositories \"%s\"", CONFIGURATION_FILE_NAME), ex);
logger.warn(String.format("Error while creating repository directory %s", sourceDirectory.getPath()), ex);
return repositories;
}
for (YAMLRepos.Entry<String, YAMLHost> entry : repos.entrySet()) {
String hostName = entry.getKey();
YAMLHost host = entry.getValue();
CredentialsProvider credentialsProvider = getCredentialsProvider(hostName, host.credentials);
for (String repo : host.repos) {
String name = repo.substring(repo.lastIndexOf("/") + 1);
if (!REGEX_REPO_PATTERN.matcher(name).matches()) {
logger.warn(String.format("The project name shall match the regular expression \\%s\\. got: %s",
REGEX_REPO_STRING, name));
continue;
}
File repositoryDirectory = new File(sourceDirectory, name);
Repository repository = new Repository(repositoryDirectory);
if (repositoryDirectory.isDirectory()) {
try (Git git = Git.open(repositoryDirectory)) {
LOG.info(String.format("Updating repository %s", name));
git.fetch().setCredentialsProvider(credentialsProvider).setRemoveDeletedRefs(true).call();
git.checkout().setName(MAIN_BRANCH).call();
} catch (Exception ex) {
logger.warn(String.format("Failed to pull latest changes of repository %s", name), ex);
continue;
}
} else {
String http_url_to_repo = String.format("https://%s/%s.git", hostName, repo);
try (Git git = Git.cloneRepository().setCredentialsProvider(credentialsProvider)
.setURI(http_url_to_repo).setDirectory(repositoryDirectory).call()) {
LOG.info(String.format("Cloning repository %s to %s", http_url_to_repo, name));
} catch (Exception ex) {
logger.warn(String.format("Failed to clone repository %s to %s", http_url_to_repo, name), ex);
continue;
}
readRepository(repository);
repositories.add(repository);
}
}
return repositories;
}
private CredentialsProvider getCredentialsProvider(String hostName, YAMLCredentials credentials) {
final String username;
final char[] password;
final Console console = System.console();
if (credentials != null && credentials.username != null) {
username = StringSubstitutor.replace(credentials.username, System.getenv());
} else {
if (console != null) {
username = console.readLine(String.format("Please enter your account username for %s:\n", hostName));
} else if (System.getenv("GITLAB_CI") != null) {
LOG.info("using username 'gitlab-ci-token'");
username = "gitlab-ci-token";
} else {
LOG.warn("using empty username");
username = "";
}
}
if (credentials != null && credentials.password != null) {
password = StringSubstitutor.replace(credentials.password, System.getenv()).toCharArray();
} else {
if (console != null) {
password = console
.readPassword(String.format("Please enter your account password for %s:\n", hostName));
} else if (System.getenv("GITLAB_CI") != null) {
password = System.getenv("CI_JOB_TOKEN").toCharArray();
} else {
LOG.warn("using empty password");
password = "".toCharArray();
}
}
return new UsernamePasswordCredentialsProvider(username, password);
}
private void readRepository(Repository repository) {
try (Git git = Git.open(repository.getDirectory())) {
//check the repository is clean
Status status = git.status().call();
if(!status.isClean()) {
logger.error("The git repository is not clean. All other branches will be ignored.");
throw new RuntimeException("The git repository is not clean. All other branches will be ignored.");
String currentBranch = git.getRepository().getBranch();
repository.setCurrentBranch(currentBranch);
List<Ref> remoteBranches = git.branchList().setListMode(ListBranchCommand.ListMode.REMOTE).call();
for (Ref ref : remoteBranches) {
String branch = ref.getName();
RevCommit commit = git.log().add(ref.getObjectId()).call().iterator().next();
Date issued = commit.getCommitterIdent().getWhen();
if (Constants.INCLUDE_MASTER && branch.equals(Constants.MASTER_BRANCH)) {
repository.addMasterVersion(ref, issued);
}
Matcher m = Constants.REGEX_RELEASE_BRANCH_PATTERN.matcher(branch);
if (m.find()) {
int major = Integer.parseInt(m.group("major"));
int minor = Integer.parseInt(m.group("minor"));
int patch = Integer.parseInt(m.group("patch"));
repository.addReleaseVersion(ref, issued, major, minor, patch);
}
if (Constants.INCLUDE_ALL) {
m = Constants.REGEX_OTHER_BRANCH_PATTERN.matcher(branch);
if (m.find()) {
String branchName = m.group("name");
repository.addBranchVersion(ref, issued, branchName);
}
}
}
} catch (Exception ex) {
repository.addMasterVersion(null, new Date());
// sort versions
Collections.sort(repository.getVersions(), new Comparator<Version>() {
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
@Override
public int compare(Version o1, Version o2) {
if (o1.equals(o2)) {
return 0;
}
if (o1 instanceof MasterVersion && o2 instanceof MasterVersion) {
return 0;
}
if (o1 instanceof MasterVersion) {
return 1;
}
if (o2 instanceof MasterVersion) {
return -1;
}
if (o1 instanceof ReleaseVersion && o2 instanceof ReleaseVersion) {
ReleaseVersion r1 = (ReleaseVersion) o1;
ReleaseVersion r2 = (ReleaseVersion) o2;
if (r1.getMajor() - r2.getMajor() != 0) {
return r1.getMajor() - r2.getMajor();
}
if (r1.getMinor() - r2.getMinor() != 0) {
return r1.getMinor() - r2.getMinor();
}
return r1.getPatch() - r2.getPatch();
}
if (o1 instanceof ReleaseVersion) {
return 1;
}
if (o2 instanceof ReleaseVersion) {
return -1;
}
return o1.getRepositoryName().compareTo(o2.getRepositoryName());
// compute priorVersion and nextVersion
for(int i = 0 ; i < repository.getVersions().size() ; i++) {
Version version = repository.getVersions().get(i);
if(i>1) {
Version priorVersion = repository.getVersions().get(i-1);
version.setPriorVersion(priorVersion);
}
if(i+1<repository.getVersions().size()) {
Version nextVersion = repository.getVersions().get(i+1);
version.setNextVersion(nextVersion);
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
}
}
}
private static class YAMLRepos extends HashMap<String, YAMLHost> {
private static final long serialVersionUID = 3434677850580166200L;
}
private static class YAMLHost {
private YAMLCredentials credentials;
private List<String> repos;
@JsonCreator
public YAMLHost(@JsonProperty(value = "credentials", required = false) YAMLCredentials credentials,
@JsonProperty(value = "repos", required = false) List<String> repos) {
this.credentials = credentials;
this.repos = repos;
}
}
private static class YAMLCredentials {
private String username;
private String password;
@JsonCreator
public YAMLCredentials(@JsonProperty(value = "username", required = false) String username,
@JsonProperty(value = "password", required = false) String password) {
this.username = username;
this.password = password;
}
}
}