ReadRepositories.java

package fr.emse.gitlab.saref.jobs;

import java.io.Console;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.io.FileUtils;
import org.apache.commons.text.StringSubstitutor;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.ListBranchCommand;
import org.eclipse.jgit.api.Status;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.transport.CredentialsProvider;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;

import fr.emse.gitlab.saref.Constants;
import fr.emse.gitlab.saref.SAREFPipelineException;
import fr.emse.gitlab.saref.entities.git.MasterVersion;
import fr.emse.gitlab.saref.entities.git.ReleaseVersion;
import fr.emse.gitlab.saref.entities.git.Repositories;
import fr.emse.gitlab.saref.entities.git.Repository;
import fr.emse.gitlab.saref.entities.git.Version;

public class ReadRepositories extends JobRunner {

	private static final Logger LOG = LoggerFactory.getLogger(ReadRepositories.class);

	public static final String REGEX_REPO_STRING = "^saref(-core|4[a-z]{4})$";
	public static final Pattern REGEX_REPO_PATTERN = Pattern.compile(REGEX_REPO_STRING, Pattern.CASE_INSENSITIVE);

	public static final String REMOTE_ORIGIN_MASTER_BRANCH = "refs/remotes/origin/master";
	public static final String REGEX_REMOTE_ORIGIN_RELEASE_BRANCH = "^refs/remotes/origin/release-" + Constants.REGEX_VERSION + "$";
	public static final String REGEX_REMOTE_ORIGIN_OTHER_BRANCH = "^refs/remotes/origin/(?<name>[^/]+)$";
	public static final Pattern REGEX_REMOTE_ORIGIN_RELEASE_BRANCH_PATTERN = Pattern.compile(REGEX_REMOTE_ORIGIN_RELEASE_BRANCH);
	public static final Pattern REGEX_REMOTE_ORIGIN_OTHER_BRANCH_PATTERN = Pattern.compile(REGEX_REMOTE_ORIGIN_OTHER_BRANCH);

	public static final String REGEX_RELEASE_BRANCH = "release-" + Constants.REGEX_VERSION + "$";
	public static final Pattern REGEX_RELEASE_BRANCH_PATTERN = Pattern.compile(REGEX_RELEASE_BRANCH);

	public static final String CONFIGURATION_FILE_NAME = ".saref-repositories.yml";
	public static final String MAIN_BRANCH = "master";
	public static final String SOURCE_DIR = "sources";

	public ReadRepositories(String testSuiteName) {
		super(testSuiteName);
	}
	
	public Repositories readRepositories(File directory, boolean ignoreLocal, boolean ignoreGit, boolean includeMaster) throws SAREFPipelineException {
		final Repositories repositories = new Repositories();
		final File sourceDirectory = new File(directory, Constants.TARGET_DIR + File.separator + SOURCE_DIR);
		if (!ignoreLocal) {
			String REGEX = "(?<ext>saref-core|saref4[a-z][a-z][a-z][a-z])";
			String name = directory.getName();
			Pattern pattern = Pattern.compile(REGEX);
			Matcher matcher = pattern.matcher(name);
			if (!matcher.find()) {
				logger.error(String.format(
						"When not for production, the SAREF pipeline must be run inside a directory whose name contains `saref-core`, or `saref4abcd` (where abcd can be any sequence of four letters). Got: %s",
						name));
				throw new SAREFPipelineException(String.format(
						"When not for production, the SAREF pipeline must be run inside a directory whose name contains `saref-core`, or `saref4abcd` (where abcd can be any sequence of four letters). Got: %s",
						name));
			}
			String repositoryName = matcher.group("ext");
			Repository repository = new Repository(directory, repositoryName);
			repositories.setDefaultRepository(repository);
			
			if(ignoreGit) {
				repository.addBranchVersion(null, new Date(), "SNAPSHOT");
			} else {
				readGitRepository(repository, true, includeMaster);
			}
		}
		final File confFile = new File(directory, CONFIGURATION_FILE_NAME);
		if(!confFile.exists()) {
			return repositories;
		}
		YAMLRepos repos;
		try {
			ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
			repos = mapper.readValue(confFile, YAMLRepos.class);
		} catch (IOException ex) {
			logger.warn(String.format("Error while reading the list of repositories \"%s\"", CONFIGURATION_FILE_NAME), ex);
			return repositories;
		}
		try {
			FileUtils.forceMkdir(sourceDirectory);
		} catch (IOException ex) {
			logger.warn(String.format("Error while creating repository directory %s", sourceDirectory.getPath()), ex);
			return repositories;
		}
		for (YAMLRepos.Entry<String, YAMLHost> entry : repos.entrySet()) {
			String hostName = entry.getKey();
			YAMLHost host = entry.getValue();
			CredentialsProvider credentialsProvider = getCredentialsProvider(hostName, host.credentials);
			for (String repo : host.repos) {
				String name = repo.substring(repo.lastIndexOf("/") + 1);
				if (!REGEX_REPO_PATTERN.matcher(name).matches()) {
					logger.warn(String.format("The project name shall match the regular expression \\%s\\. got: %s",
							REGEX_REPO_STRING, name));
					continue;
				}
				File repositoryDirectory = new File(sourceDirectory, name);
				Repository repository = new Repository(repositoryDirectory);
				if (repositoryDirectory.isDirectory()) {
					try (Git git = Git.open(repositoryDirectory)) {
						LOG.info(String.format("Updating repository %s", name));
						git.fetch().setCredentialsProvider(credentialsProvider).setRemoveDeletedRefs(true).call();
						git.checkout().setName(MAIN_BRANCH).call();
					} catch (Exception ex) {
						logger.warn(String.format("Failed to pull latest changes of repository %s", name), ex);
						continue;
					}
				} else {
					String http_url_to_repo = String.format("https://%s/%s.git", hostName, repo);
					try (Git git = Git.cloneRepository().setCredentialsProvider(credentialsProvider)
							.setURI(http_url_to_repo).setDirectory(repositoryDirectory).call()) {
						LOG.info(String.format("Cloning repository %s to %s", http_url_to_repo, name));
					} catch (Exception ex) {
						logger.warn(String.format("Failed to clone repository %s to %s", http_url_to_repo, name), ex);
						continue;
					}
				}
				readGitRepository(repository, false, includeMaster);
				repositories.add(repository);
			}
		}
		return repositories;
	}

	private CredentialsProvider getCredentialsProvider(String hostName, YAMLCredentials credentials) {
		final String username;
		final char[] password;
		final Console console = System.console();
		if (credentials != null && credentials.username != null) {
			username = StringSubstitutor.replace(credentials.username, System.getenv());
		} else {
			if (console != null) {
				username = console.readLine(String.format("Please enter your account username for %s:\n", hostName));
			} else if (System.getenv("GITLAB_CI") != null) {
				LOG.info("using username 'gitlab-ci-token'");
				username = "gitlab-ci-token";
			} else {
				LOG.warn("using empty username");
				username = "";
			}
		}
		if (credentials != null && credentials.password != null) {
			password = StringSubstitutor.replace(credentials.password, System.getenv()).toCharArray();
		} else {
			if (console != null) {
				password = console
						.readPassword(String.format("Please enter your account password for %s:\n", hostName));
			} else if (System.getenv("GITLAB_CI") != null) {
				password = System.getenv("CI_JOB_TOKEN").toCharArray();
			} else {
				LOG.warn("using empty password");
				password = "".toCharArray();
			}
		}
		return new UsernamePasswordCredentialsProvider(username, password);
	}

	private void readGitRepository(Repository repository, boolean local, boolean includeMaster) {
		try (Git git = Git.open(repository.getDirectory())) {
			Status status = git.status().call();
			if(!status.isClean()) {
				logger.error("The git repository is not clean. Commit first, or use option --no-git");
				throw new RuntimeException("The git repository is not clean. Commit first, or use option --no-git");
			}
			String currentBranch = git.getRepository().getBranch();
			repository.setCurrentBranch(currentBranch);
			List<Ref> allBranches = git.branchList().setListMode(ListBranchCommand.ListMode.ALL).call();
			for (Ref ref : allBranches) {
				String branch = ref.getName();
				RevCommit commit = git.log().add(ref.getObjectId()).call().iterator().next();
				Date issued = commit.getCommitterIdent().getWhen();
				if (local) {
					if(branch.contains("refs/remotes/")) {
						continue;
					}
					if(includeMaster && branch.equals("master")) {
						repository.addMasterVersion(ref, issued);
						continue;
					}
					Matcher m = REGEX_RELEASE_BRANCH_PATTERN.matcher(branch);
					if (m.find()) {
						int major = Integer.parseInt(m.group("major"));
						int minor = Integer.parseInt(m.group("minor"));
						int patch = Integer.parseInt(m.group("patch"));
						repository.addReleaseVersion(ref, issued, major, minor, patch);
						continue;
					}
				} else {
					if(!branch.contains("refs/remotes/")) {
						continue;
					}
					if(includeMaster && branch.equals(REMOTE_ORIGIN_MASTER_BRANCH)) {
						repository.addMasterVersion(ref, issued);
						continue;
					}
					Matcher m = REGEX_REMOTE_ORIGIN_RELEASE_BRANCH_PATTERN.matcher(branch);
					if (m.find()) {
						int major = Integer.parseInt(m.group("major"));
						int minor = Integer.parseInt(m.group("minor"));
						int patch = Integer.parseInt(m.group("patch"));
						repository.addReleaseVersion(ref, issued, major, minor, patch);
						continue;
					}
				}
			}
		} catch (Exception ex) {
			repository.addMasterVersion(null, new Date());
		}
		// sort versions
		Collections.sort(repository.getVersions(), new Comparator<Version>() {
			@Override
			public int compare(Version o1, Version o2) {
				if (o1.equals(o2)) {
					return 0;
				}
				if (o1 instanceof MasterVersion && o2 instanceof MasterVersion) {
					return 0;
				}
				if (o1 instanceof MasterVersion) {
					return 1;
				}
				if (o2 instanceof MasterVersion) {
					return -1;
				}
				if (o1 instanceof ReleaseVersion && o2 instanceof ReleaseVersion) {
					ReleaseVersion r1 = (ReleaseVersion) o1;
					ReleaseVersion r2 = (ReleaseVersion) o2;
					if (r1.getMajor() - r2.getMajor() != 0) {
						return r1.getMajor() - r2.getMajor();
					}
					if (r1.getMinor() - r2.getMinor() != 0) {
						return r1.getMinor() - r2.getMinor();
					}
					return r1.getPatch() - r2.getPatch();
				}
				if (o1 instanceof ReleaseVersion) {
					return 1;
				}
				if (o2 instanceof ReleaseVersion) {
					return -1;
				}
				return o1.getRepositoryName().compareTo(o2.getRepositoryName());
			}
		});
		// compute priorVersion and nextVersion
		for(int i = 0 ; i < repository.getVersions().size() ; i++) {
			Version version = repository.getVersions().get(i);
			if(i>1) {
				Version priorVersion = repository.getVersions().get(i-1);
				version.setPriorVersion(priorVersion);
			}
			if(i+1<repository.getVersions().size()) {
				Version nextVersion = repository.getVersions().get(i+1);
				version.setNextVersion(nextVersion);
			}
		}
	}

	private static class YAMLRepos extends HashMap<String, YAMLHost> {
		private static final long serialVersionUID = 3434677850580166200L;
	}

	private static class YAMLHost {
		private YAMLCredentials credentials;
		private List<String> repos;

		@JsonCreator
		public YAMLHost(@JsonProperty(value = "credentials", required = false) YAMLCredentials credentials,
				@JsonProperty(value = "repos", required = false) List<String> repos) {
			this.credentials = credentials;
			this.repos = repos;
		}
	}

	private static class YAMLCredentials {
		private String username;
		private String password;

		@JsonCreator
		public YAMLCredentials(@JsonProperty(value = "username", required = false) String username,
				@JsonProperty(value = "password", required = false) String password) {
			this.username = username;
			this.password = password;
		}
	}

}