ReadRepositories.java

package fr.emse.gitlab.saref.jobs;

import java.io.Console;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.io.FileUtils;
import org.apache.commons.text.StringSubstitutor;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.ListBranchCommand;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.transport.CredentialsProvider;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;

import fr.emse.gitlab.saref.Constants;
import fr.emse.gitlab.saref.entities.git.BranchVersion;
import fr.emse.gitlab.saref.entities.git.MasterVersion;
import fr.emse.gitlab.saref.entities.git.ReleaseVersion;
import fr.emse.gitlab.saref.entities.git.Repositories;
import fr.emse.gitlab.saref.entities.git.Repository;
import fr.emse.gitlab.saref.entities.git.Version;

public class ReadRepositories extends AbstractJobRunner<Repositories> {

	static final Logger LOG = LoggerFactory.getLogger(ReadRepositories.class);

	static final String REGEX_REPO_STRING = "^saref(-core|4[a-z]{4})$";
	static final Pattern REGEX_REPO_PATTERN = Pattern.compile(REGEX_REPO_STRING, Pattern.CASE_INSENSITIVE);

	public ReadRepositories(File dir) {
		super("Read and clone repositories in \".saref-repositories.yml\"", dir);
	}

	/*
	 * Needs to work the same way: - on the computer of a knowledge engineer - when
	 * I trigger a CI job. see
	 * https://docs.gitlab.com/ee/user/project/new_ci_build_permissions_model.html
	 * 
	 * by default, on the machine that will trigger the job, if it is gitlab, we may
	 * use username gitlab-ci-token and password $CI_JOB_TOKEN --> that is, if the
	 * environment variable GITLAB_CI is set.
	 * 
	 * 
	 * 
	 * but on the other machines, we need to provide a username and password. This
	 * is especially true if we want to use repositories from different platforms
	 * (gitlab of emse, gitlab of etsi, github, ...) Environment variables are
	 * substituted in these values.
	 * 
	 * If they are not set, then we may prompt the user. If the credentials are not
	 * set, then by default we use gitlab-ci-token and password $CI_JOB_TOKEN
	 * 
	 */
	@Override
	protected Repositories getValue() {
		final Repositories repositories = new Repositories();
		final File file = new File(directory, ".saref-repositories.yml");
		YAMLRepos repos;
		try {
			ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
			repos = mapper.readValue(file, YAMLRepos.class);
		} catch (Exception ex) {
			error("Error while reading the list of repositories \".saref-repositories.yml\"", ex);
			return repositories;
		}
		File repoDir = new File(directory, Constants.GIT_DIR);
		try {
			FileUtils.forceDelete(repoDir);
			FileUtils.forceMkdir(repoDir);
		} catch (IOException ex) {
			error(String.format("Error while creating repository directory %s", repoDir.getPath()), ex);
			return repositories;
		}
		for (YAMLRepos.Entry<String, YAMLHost> entry : repos.entrySet()) {
			String hostName = entry.getKey();
			YAMLHost host = entry.getValue();
			CredentialsProvider crendentialsProvider = getCredentialsProvider(hostName, host.credentials);
			for (String repo : host.repos) {
				String name = repo.substring(repo.lastIndexOf("/") + 1);
				if (!REGEX_REPO_PATTERN.matcher(name).matches()) {
					failure(String.format("The project name shall match the regular expression \\%s\\. got: %s",
							REGEX_REPO_STRING, name));
					continue;
				}
				File directory = new File(repoDir, name);
				String http_url_to_repo = String.format("https://%s/%s.git", hostName, repo);
				try (Git git = Git.cloneRepository().setCredentialsProvider(crendentialsProvider)
						.setURI(http_url_to_repo).setDirectory(directory).call()) {
					LOG.info(String.format("Cloned repository %s to %s", http_url_to_repo, name));
					List<Version> versions = readRepository(git, name, directory);
					Repository repository = new Repository(name, directory, http_url_to_repo, versions);
					repositories.add(repository);
				} catch (Exception ex) {
					failure(String.format("Failed to clone repository %s to %s", http_url_to_repo, name), ex);
					continue;
				}
			}
		}
		return repositories;
	}

	private CredentialsProvider getCredentialsProvider(String hostName, YAMLCredentials credentials) {
		final String username;
		final char[] password;
		final Console console = System.console();
		if (credentials != null && credentials.username != null) {
			username = StringSubstitutor.replace(credentials.username, System.getenv());
		} else {
			if (console != null) {
				username = console
						.readLine(String.format("Please enter your EOL account username for %s:\n", hostName));
			} else if (System.getenv("GITLAB_CI") != null) {
				LOG.info("using username 'gitlab-ci-token'");
				username = "gitlab-ci-token";
			} else {
				LOG.warn("using empty username");
				username = "";
			}
		}
		if (credentials != null && credentials.password != null) {
			password = StringSubstitutor.replace(credentials.password, System.getenv()).toCharArray();
		} else {
			if (console != null) {
				password = console
						.readPassword(String.format("Please enter your EOL account password for %s:\n", hostName));
			} else if (System.getenv("GITLAB_CI") != null) {
				password = System.getenv("CI_JOB_TOKEN").toCharArray();
			} else {
				LOG.warn("using empty password");
				password = "".toCharArray();
			}
		}
		return new UsernamePasswordCredentialsProvider(username, password);
	}

	private List<Version> readRepository(Git git, String name, File directory) throws Exception {
		List<Version> versions = new ArrayList<>();
		List<Ref> remoteBranches = git.branchList().setListMode(ListBranchCommand.ListMode.REMOTE).call();
		for (Ref ref : remoteBranches) {
			Version v = getVersion(git, name, ref);
			if (v != null) {
				versions.add(v);
			}
		}
		Collections.sort(versions, new Comparator<Version>() {
			@Override
			public int compare(Version o1, Version o2) {
				if (o1.equals(o2)) {
					return 0;
				}
				if (o1.getName() != o2.getName()) {
					return o1.getName().compareTo(o2.getName());
				}
				if (o1 instanceof MasterVersion && o2 instanceof MasterVersion) {
					return 0;
				}
				if (o1 instanceof MasterVersion) {
					return 1;
				}
				if (o2 instanceof MasterVersion) {
					return -1;
				}
				if (o1 instanceof ReleaseVersion && o2 instanceof ReleaseVersion) {
					ReleaseVersion r1 = (ReleaseVersion) o1;
					ReleaseVersion r2 = (ReleaseVersion) o2;
					if (r1.getMajor() - r2.getMajor() != 0) {
						return r1.getMajor() - r2.getMajor();
					}
					if (r1.getMinor() - r2.getMinor() != 0) {
						return r1.getMinor() - r2.getMinor();
					}
					return r1.getPatch() - r2.getPatch();
				}
				if (o1 instanceof ReleaseVersion) {
					return 1;
				}
				if (o2 instanceof ReleaseVersion) {
					return -1;
				}
				return 0;
			}
		});
		return versions;
	}

	private Version getVersion(Git git, String name, Ref ref) throws Exception {
		String branch = ref.getName();
		RevCommit commit = git.log().add(ref.getObjectId()).call().iterator().next();
		Date issued = commit.getCommitterIdent().getWhen();
		if (Constants.INCLUDE_MASTER && branch.equals(Constants.REGEX_MASTER_BRANCH)) {
			return new MasterVersion(name, ref, issued);
		}
		Matcher m = Constants.REGEX_RELEASE_BRANCH_PATTERN.matcher(branch);
		if (m.find()) {
			int major = Integer.parseInt(m.group("major"));
			int minor = Integer.parseInt(m.group("minor"));
			int patch = Integer.parseInt(m.group("patch"));
			return new ReleaseVersion(name, ref, issued, major, minor, patch);
		}
		if (Constants.INCLUDE_ALL) {
			m = Constants.REGEX_OTHER_BRANCH_PATTERN.matcher(branch);
			if (m.find()) {
				String branchName = m.group("name");
				return new BranchVersion(name, ref, issued, branchName);
			}
		}
		return null;
	}

	private static class YAMLRepos extends HashMap<String, YAMLHost> {
		private static final long serialVersionUID = 3434677850580166200L;
	}

	private static class YAMLHost {
		private YAMLCredentials credentials;
		private List<String> repos;

		@JsonCreator
		public YAMLHost(@JsonProperty(value = "credentials", required = false) YAMLCredentials credentials,
				@JsonProperty(value = "repos", required = false) List<String> repos) {
			this.credentials = credentials;
			this.repos = repos;
		}
	}

	private static class YAMLCredentials {
		private String username;
		private String password;

		@JsonCreator
		public YAMLCredentials(@JsonProperty(value = "username", required = false) String username,
				@JsonProperty(value = "password", required = false) String password) {
			this.username = username;
			this.password = password;
		}
	}

}