Newer
Older
package fr.emse.gitlab.saref.jobs;
import java.io.Console;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
import org.apache.commons.text.StringSubstitutor;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.ListBranchCommand;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.transport.CredentialsProvider;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import fr.emse.gitlab.saref.Constants;
import fr.emse.gitlab.saref.SAREFPipelineException;
import fr.emse.gitlab.saref.entities.git.MasterVersion;
import fr.emse.gitlab.saref.entities.git.ReleaseVersion;
import fr.emse.gitlab.saref.entities.git.Repositories;
import fr.emse.gitlab.saref.entities.git.Repository;
import fr.emse.gitlab.saref.entities.git.Version;
private static final Logger LOG = LoggerFactory.getLogger(ReadRepositories.class);
public static final String REGEX_REPO_STRING = "^saref(-core|4[a-z]{4})$";
public static final Pattern REGEX_REPO_PATTERN = Pattern.compile(REGEX_REPO_STRING, Pattern.CASE_INSENSITIVE);
public static final String REMOTE_ORIGIN_MASTER_BRANCH = "refs/remotes/origin/master";
public static final String REGEX_REMOTE_ORIGIN_RELEASE_BRANCH = "^refs/remotes/origin/release-" + Constants.REGEX_VERSION + "$";
public static final String REGEX_REMOTE_ORIGIN_OTHER_BRANCH = "^refs/remotes/origin/(?<name>[^/]+)$";
public static final Pattern REGEX_REMOTE_ORIGIN_RELEASE_BRANCH_PATTERN = Pattern.compile(REGEX_REMOTE_ORIGIN_RELEASE_BRANCH);
public static final Pattern REGEX_REMOTE_ORIGIN_OTHER_BRANCH_PATTERN = Pattern.compile(REGEX_REMOTE_ORIGIN_OTHER_BRANCH);
public static final String REGEX_RELEASE_BRANCH = "release-" + Constants.REGEX_VERSION + "$";
public static final Pattern REGEX_RELEASE_BRANCH_PATTERN = Pattern.compile(REGEX_RELEASE_BRANCH);
public static final String CONFIGURATION_FILE_NAME = ".saref-repositories.yml";
public static final String MAIN_BRANCH = "master";
public static final String SOURCE_DIR = "sources";
public ReadRepositories(String testSuiteName) {
super(testSuiteName);
public Repositories readRepositories(File directory, boolean ignoreLocal, boolean ignoreGit, boolean includeMaster) throws SAREFPipelineException {
final Repositories repositories = new Repositories();
final File sourceDirectory = new File(directory, Constants.TARGET_DIR + File.separator + SOURCE_DIR);
if (!ignoreLocal) {
String REGEX = "(?<ext>saref-core|saref4[a-z][a-z][a-z][a-z])";
String name = directory.getName();
Pattern pattern = Pattern.compile(REGEX);
Matcher matcher = pattern.matcher(name);
if (!matcher.find()) {
logger.error(String.format(
"When not for production, the SAREF pipeline must be run inside a directory whose name contains `saref-core`, or `saref4abcd` (where abcd can be any sequence of four letters). Got: %s",
name));
throw new SAREFPipelineException(String.format(
"When not for production, the SAREF pipeline must be run inside a directory whose name contains `saref-core`, or `saref4abcd` (where abcd can be any sequence of four letters). Got: %s",
name));
}
String repositoryName = matcher.group("ext");
Repository repository = new Repository(directory, repositoryName);
repositories.setDefaultRepository(repository);
if(ignoreGit) {
repository.addBranchVersion(null, new Date(), "SNAPSHOT");
} else {
readGitRepository(repository, true, includeMaster);
}
}
final File confFile = new File(directory, CONFIGURATION_FILE_NAME);
YAMLRepos repos;
try {
ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
repos = mapper.readValue(confFile, YAMLRepos.class);
logger.warn(String.format("Error while reading the list of repositories \"%s\"", CONFIGURATION_FILE_NAME), ex);
logger.warn(String.format("Error while creating repository directory %s", sourceDirectory.getPath()), ex);
return repositories;
}
for (YAMLRepos.Entry<String, YAMLHost> entry : repos.entrySet()) {
String hostName = entry.getKey();
YAMLHost host = entry.getValue();
CredentialsProvider credentialsProvider = getCredentialsProvider(hostName, host.credentials);
for (String repo : host.repos) {
String name = repo.substring(repo.lastIndexOf("/") + 1);
if (!REGEX_REPO_PATTERN.matcher(name).matches()) {
logger.warn(String.format("The project name shall match the regular expression \\%s\\. got: %s",
REGEX_REPO_STRING, name));
continue;
}
File repositoryDirectory = new File(sourceDirectory, name);
Repository repository = new Repository(repositoryDirectory);
if (repositoryDirectory.isDirectory()) {
try (Git git = Git.open(repositoryDirectory)) {
LOG.info(String.format("Updating repository %s", name));
git.fetch().setCredentialsProvider(credentialsProvider).setRemoveDeletedRefs(true).call();
git.checkout().setName(MAIN_BRANCH).call();
} catch (Exception ex) {
logger.warn(String.format("Failed to pull latest changes of repository %s", name), ex);
continue;
}
} else {
String http_url_to_repo = String.format("https://%s/%s.git", hostName, repo);
try (Git git = Git.cloneRepository().setCredentialsProvider(credentialsProvider)
.setURI(http_url_to_repo).setDirectory(repositoryDirectory).call()) {
LOG.info(String.format("Cloning repository %s to %s", http_url_to_repo, name));
} catch (Exception ex) {
logger.warn(String.format("Failed to clone repository %s to %s", http_url_to_repo, name), ex);
continue;
}
}
}
return repositories;
}
private CredentialsProvider getCredentialsProvider(String hostName, YAMLCredentials credentials) {
final String username;
final char[] password;
final Console console = System.console();
if (credentials != null && credentials.username != null) {
username = StringSubstitutor.replace(credentials.username, System.getenv());
} else {
if (console != null) {
username = console.readLine(String.format("Please enter your account username for %s:\n", hostName));
} else if (System.getenv("GITLAB_CI") != null) {
LOG.info("using username 'gitlab-ci-token'");
username = "gitlab-ci-token";
} else {
LOG.warn("using empty username");
username = "";
}
}
if (credentials != null && credentials.password != null) {
password = StringSubstitutor.replace(credentials.password, System.getenv()).toCharArray();
} else {
if (console != null) {
password = console
.readPassword(String.format("Please enter your account password for %s:\n", hostName));
} else if (System.getenv("GITLAB_CI") != null) {
password = System.getenv("CI_JOB_TOKEN").toCharArray();
} else {
LOG.warn("using empty password");
password = "".toCharArray();
}
}
return new UsernamePasswordCredentialsProvider(username, password);
}
private void readGitRepository(Repository repository, boolean local, boolean includeMaster) {
try (Git git = Git.open(repository.getDirectory())) {
Status status = git.status().call();
if(!status.isClean()) {
logger.error("The git repository is not clean. Commit first, or use option --no-git");
throw new RuntimeException("The git repository is not clean. Commit first, or use option --no-git");
String currentBranch = git.getRepository().getBranch();
repository.setCurrentBranch(currentBranch);
List<Ref> allBranches = git.branchList().setListMode(ListBranchCommand.ListMode.ALL).call();
for (Ref ref : allBranches) {
String branch = ref.getName();
RevCommit commit = git.log().add(ref.getObjectId()).call().iterator().next();
Date issued = commit.getCommitterIdent().getWhen();
if (local) {
if(branch.contains("refs/remotes/")) {
continue;
}
if(includeMaster && branch.equals("master")) {
repository.addMasterVersion(ref, issued);
continue;
}
Matcher m = REGEX_RELEASE_BRANCH_PATTERN.matcher(branch);
if (m.find()) {
int major = Integer.parseInt(m.group("major"));
int minor = Integer.parseInt(m.group("minor"));
int patch = Integer.parseInt(m.group("patch"));
repository.addReleaseVersion(ref, issued, major, minor, patch);
continue;
}
} else {
if(!branch.contains("refs/remotes/")) {
continue;
}
if(includeMaster && branch.equals(REMOTE_ORIGIN_MASTER_BRANCH)) {
repository.addMasterVersion(ref, issued);
continue;
}
Matcher m = REGEX_REMOTE_ORIGIN_RELEASE_BRANCH_PATTERN.matcher(branch);
int major = Integer.parseInt(m.group("major"));
int minor = Integer.parseInt(m.group("minor"));
int patch = Integer.parseInt(m.group("patch"));
repository.addReleaseVersion(ref, issued, major, minor, patch);
continue;
}
}
}
} catch (Exception ex) {
repository.addMasterVersion(null, new Date());
// sort versions
Collections.sort(repository.getVersions(), new Comparator<Version>() {
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
@Override
public int compare(Version o1, Version o2) {
if (o1.equals(o2)) {
return 0;
}
if (o1 instanceof MasterVersion && o2 instanceof MasterVersion) {
return 0;
}
if (o1 instanceof MasterVersion) {
return 1;
}
if (o2 instanceof MasterVersion) {
return -1;
}
if (o1 instanceof ReleaseVersion && o2 instanceof ReleaseVersion) {
ReleaseVersion r1 = (ReleaseVersion) o1;
ReleaseVersion r2 = (ReleaseVersion) o2;
if (r1.getMajor() - r2.getMajor() != 0) {
return r1.getMajor() - r2.getMajor();
}
if (r1.getMinor() - r2.getMinor() != 0) {
return r1.getMinor() - r2.getMinor();
}
return r1.getPatch() - r2.getPatch();
}
if (o1 instanceof ReleaseVersion) {
return 1;
}
if (o2 instanceof ReleaseVersion) {
return -1;
}
return o1.getRepositoryName().compareTo(o2.getRepositoryName());
// compute priorVersion and nextVersion
for(int i = 0 ; i < repository.getVersions().size() ; i++) {
Version version = repository.getVersions().get(i);
if(i>1) {
Version priorVersion = repository.getVersions().get(i-1);
version.setPriorVersion(priorVersion);
}
if(i+1<repository.getVersions().size()) {
Version nextVersion = repository.getVersions().get(i+1);
version.setNextVersion(nextVersion);
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
}
}
}
private static class YAMLRepos extends HashMap<String, YAMLHost> {
private static final long serialVersionUID = 3434677850580166200L;
}
private static class YAMLHost {
private YAMLCredentials credentials;
private List<String> repos;
@JsonCreator
public YAMLHost(@JsonProperty(value = "credentials", required = false) YAMLCredentials credentials,
@JsonProperty(value = "repos", required = false) List<String> repos) {
this.credentials = credentials;
this.repos = repos;
}
}
private static class YAMLCredentials {
private String username;
private String password;
@JsonCreator
public YAMLCredentials(@JsonProperty(value = "username", required = false) String username,
@JsonProperty(value = "password", required = false) String password) {
this.username = username;
this.password = password;
}
}
}