package fr.emse.gitlab.saref.jobs; import java.io.Console; import java.io.File; import java.io.IOException; import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.io.FileUtils; import org.apache.commons.text.StringSubstitutor; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.ListBranchCommand; import org.eclipse.jgit.api.Status; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.transport.CredentialsProvider; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import fr.emse.gitlab.saref.Constants; import fr.emse.gitlab.saref.SAREFPipelineException; import fr.emse.gitlab.saref.entities.git.MasterVersion; import fr.emse.gitlab.saref.entities.git.ReleaseVersion; import fr.emse.gitlab.saref.entities.git.Repositories; import fr.emse.gitlab.saref.entities.git.Repository; import fr.emse.gitlab.saref.entities.git.Version; public class ReadRepositories extends JobRunner { private static final Logger LOG = LoggerFactory.getLogger(ReadRepositories.class); public static final String REGEX_REPO_STRING = "^saref(-core|4[a-z]{4})$"; public static final Pattern REGEX_REPO_PATTERN = Pattern.compile(REGEX_REPO_STRING, Pattern.CASE_INSENSITIVE); public static final String CONFIGURATION_FILE_NAME = ".saref-repositories.yml"; public static final String MAIN_BRANCH = "master"; public static final String SOURCE_DIR = "sources"; public ReadRepositories(String testSuiteName) { super(testSuiteName); } public Repositories readRepositories(File directory) throws SAREFPipelineException { final Repositories repositories = new Repositories(); final File sourceDirectory = new File(directory, Constants.TARGET_DIR + File.separator + SOURCE_DIR); if (!Constants.PRODUCTION) { String REGEX = "^(?saref-core|saref4[a-z][a-z][a-z][a-z])"; String name = directory.getName(); Pattern pattern = Pattern.compile(REGEX); Matcher matcher = pattern.matcher(name); if (!matcher.find()) { logger.error(String.format( "When not for production, the SAREF pipeline must be run inside a directory whose name begins with `saref-core`, or `saref4abcd` (where abcd can be any sequence of four letters). Got: %s", name)); throw new SAREFPipelineException(String.format( "When not for production, the SAREF pipeline must be run inside a directory whose name begins with `saref-core`, or `saref4abcd` (where abcd can be any sequence of four letters). Got: %s", name)); } String repositoryName = matcher.group("ext"); Repository repository = new Repository(directory, repositoryName); repositories.setDefaultRepository(repository); readRepository(repository); } final File confFile = new File(directory, CONFIGURATION_FILE_NAME); YAMLRepos repos; try { ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); repos = mapper.readValue(confFile, YAMLRepos.class); } catch (Exception ex) { logger.warn(String.format("Error while reading the list of repositories \"%s\"", CONFIGURATION_FILE_NAME), ex); return repositories; } try { FileUtils.forceMkdir(sourceDirectory); } catch (IOException ex) { logger.warn(String.format("Error while creating repository directory %s", sourceDirectory.getPath()), ex); return repositories; } for (YAMLRepos.Entry entry : repos.entrySet()) { String hostName = entry.getKey(); YAMLHost host = entry.getValue(); CredentialsProvider credentialsProvider = getCredentialsProvider(hostName, host.credentials); for (String repo : host.repos) { String name = repo.substring(repo.lastIndexOf("/") + 1); if (!REGEX_REPO_PATTERN.matcher(name).matches()) { logger.warn(String.format("The project name shall match the regular expression \\%s\\. got: %s", REGEX_REPO_STRING, name)); continue; } File repositoryDirectory = new File(sourceDirectory, name); Repository repository = new Repository(repositoryDirectory); if (repositoryDirectory.isDirectory()) { try (Git git = Git.open(repositoryDirectory)) { LOG.info(String.format("Updating repository %s", name)); git.fetch().setCredentialsProvider(credentialsProvider).setRemoveDeletedRefs(true).call(); git.checkout().setName(MAIN_BRANCH).call(); } catch (Exception ex) { logger.warn(String.format("Failed to pull latest changes of repository %s", name), ex); continue; } } else { String http_url_to_repo = String.format("https://%s/%s.git", hostName, repo); try (Git git = Git.cloneRepository().setCredentialsProvider(credentialsProvider) .setURI(http_url_to_repo).setDirectory(repositoryDirectory).call()) { LOG.info(String.format("Cloning repository %s to %s", http_url_to_repo, name)); } catch (Exception ex) { logger.warn(String.format("Failed to clone repository %s to %s", http_url_to_repo, name), ex); continue; } } readRepository(repository); repositories.add(repository); } } return repositories; } private CredentialsProvider getCredentialsProvider(String hostName, YAMLCredentials credentials) { final String username; final char[] password; final Console console = System.console(); if (credentials != null && credentials.username != null) { username = StringSubstitutor.replace(credentials.username, System.getenv()); } else { if (console != null) { username = console.readLine(String.format("Please enter your account username for %s:\n", hostName)); } else if (System.getenv("GITLAB_CI") != null) { LOG.info("using username 'gitlab-ci-token'"); username = "gitlab-ci-token"; } else { LOG.warn("using empty username"); username = ""; } } if (credentials != null && credentials.password != null) { password = StringSubstitutor.replace(credentials.password, System.getenv()).toCharArray(); } else { if (console != null) { password = console .readPassword(String.format("Please enter your account password for %s:\n", hostName)); } else if (System.getenv("GITLAB_CI") != null) { password = System.getenv("CI_JOB_TOKEN").toCharArray(); } else { LOG.warn("using empty password"); password = "".toCharArray(); } } return new UsernamePasswordCredentialsProvider(username, password); } private void readRepository(Repository repository) { try (Git git = Git.open(repository.getDirectory())) { //check the repository is clean Status status = git.status().call(); if(!status.isClean()) { logger.error("The git repository is not clean. All other branches will be ignored."); throw new RuntimeException("The git repository is not clean. All other branches will be ignored."); } String currentBranch = git.getRepository().getBranch(); repository.setCurrentBranch(currentBranch); List remoteBranches = git.branchList().setListMode(ListBranchCommand.ListMode.REMOTE).call(); for (Ref ref : remoteBranches) { String branch = ref.getName(); RevCommit commit = git.log().add(ref.getObjectId()).call().iterator().next(); Date issued = commit.getCommitterIdent().getWhen(); if (Constants.INCLUDE_MASTER && branch.equals(Constants.MASTER_BRANCH)) { repository.addMasterVersion(ref, issued); } Matcher m = Constants.REGEX_RELEASE_BRANCH_PATTERN.matcher(branch); if (m.find()) { int major = Integer.parseInt(m.group("major")); int minor = Integer.parseInt(m.group("minor")); int patch = Integer.parseInt(m.group("patch")); repository.addReleaseVersion(ref, issued, major, minor, patch); } if (Constants.INCLUDE_ALL) { m = Constants.REGEX_OTHER_BRANCH_PATTERN.matcher(branch); if (m.find()) { String branchName = m.group("name"); repository.addBranchVersion(ref, issued, branchName); } } } } catch (Exception ex) { repository.addMasterVersion(null, new Date()); } // sort versions Collections.sort(repository.getVersions(), new Comparator() { @Override public int compare(Version o1, Version o2) { if (o1.equals(o2)) { return 0; } if (o1 instanceof MasterVersion && o2 instanceof MasterVersion) { return 0; } if (o1 instanceof MasterVersion) { return 1; } if (o2 instanceof MasterVersion) { return -1; } if (o1 instanceof ReleaseVersion && o2 instanceof ReleaseVersion) { ReleaseVersion r1 = (ReleaseVersion) o1; ReleaseVersion r2 = (ReleaseVersion) o2; if (r1.getMajor() - r2.getMajor() != 0) { return r1.getMajor() - r2.getMajor(); } if (r1.getMinor() - r2.getMinor() != 0) { return r1.getMinor() - r2.getMinor(); } return r1.getPatch() - r2.getPatch(); } if (o1 instanceof ReleaseVersion) { return 1; } if (o2 instanceof ReleaseVersion) { return -1; } return o1.getRepositoryName().compareTo(o2.getRepositoryName()); } }); // compute priorVersion and nextVersion for(int i = 0 ; i < repository.getVersions().size() ; i++) { Version version = repository.getVersions().get(i); if(i>1) { Version priorVersion = repository.getVersions().get(i-1); version.setPriorVersion(priorVersion); } if(i+1 { private static final long serialVersionUID = 3434677850580166200L; } private static class YAMLHost { private YAMLCredentials credentials; private List repos; @JsonCreator public YAMLHost(@JsonProperty(value = "credentials", required = false) YAMLCredentials credentials, @JsonProperty(value = "repos", required = false) List repos) { this.credentials = credentials; this.repos = repos; } } private static class YAMLCredentials { private String username; private String password; @JsonCreator public YAMLCredentials(@JsonProperty(value = "username", required = false) String username, @JsonProperty(value = "password", required = false) String password) { this.username = username; this.password = password; } } }