changed flags

src/main/java/fr/emse/gitlab/saref/CMDConfigurations.java

@@ -18,25 +18,39 @@ public class CMDConfigurations {
 	public static final String ARG_HELP = "h";
 	public static final String ARG_HELP_LONG = "help";
 	public static final String ARG_HELP_MAN = "Show help";
+	
 	public static final String ARG_INIT = "i";
 	public static final String ARG_INIT_LONG = "init";
 	public static final String ARG_INIT_MAN = "Initialize a new SAREF extension development in the current directory";
+
 	public static final String ARG_DIRECTORY = "d";
-	public static final String ARG_DIRECTORY_DEFAULT = "";
+	public static final String ARG_DIRECTORY_DEFAULT = "";	
 	public static final String ARG_DIRECTORY_LONG = "dir";
 	public static final String ARG_DIRECTORY_MAN = "Location of the SAREF extension directory (default is .)";
-	public static final String ARG_INCLUDE_ALL = "a";
-	public static final String ARG_INCLUDE_ALL_LONG = "all";
-	public static final String ARG_INCLUDE_ALL_MAN = "Include all branches";
+
+	public static final String ARG_REMOTE_ONLY = "r";
+	public static final String ARG_REMOTE_ONLY_LONG = "remote-only";
+	public static final String ARG_REMOTE_ONLY_MAN = "Do not check the directory itself. Only consider the repositories listed in the `.saref-repositories.yml` document. Used to generate the website for several extensions.";
+
+	public static final String ARG_IGNORE_GIT = "g";
+	public static final String ARG_IGNORE_GIT_LONG = "no-git";
+	public static final String ARG_IGNORE_GIT_MAN = "Only check the current state of the repository";
+
+	public static final String ARG_IGNORE_EXAMPLES = "e";
+	public static final String ARG_IGNORE_EXAMPLES_LONG = "ignore-examples";
+	public static final String ARG_IGNORE_EXAMPLES_MAN = "Only check the SAREF extension ontology. Ignore the examples.";
+	
+	public static final String ARG_NO_SITE = "s";
+	public static final String ARG_NO_SITE_LONG = "no-site";
+	public static final String ARG_NO_SITE_MAN = "Do not generate the static website";
+	
 	public static final String ARG_INCLUDE_MASTER = "m";
 	public static final String ARG_INCLUDE_MASTER_LONG = "master";
-	public static final String ARG_INCLUDE_MASTER_MAN = "Include the HEAD of the master branches";
-	public static final String ARG_PRODUCTION = "p";
-	public static final String ARG_PRODUCTION_LONG = "production";
-	public static final String ARG_PRODUCTION_MAN = "Production mode: the directory does not contain a SAREF extension, but only a file `.saref-repositories.yml`";
-	public static final String ARG_DEBUG_TEMPLATE = "debug";
-	public static final String ARG_DEBUG_TEMPLATE_LONG = "debug-template";
-	public static final String ARG_DEBUG_TEMPLATE_MAN = "Use the debug-template mode of SPARQL-Generate when generating the documentation";
+	public static final String ARG_INCLUDE_MASTER_MAN = "Check the master branches of the remote repositories listed in the `.saref-repositories.yml` document";
+
+	public static final String ARG_VERBOSE = "v";
+	public static final String ARG_VERBOSE_LONG = "verbose";
+	public static final String ARG_VERBOSE_MAN = "Use verbose mode. (For example, use SPARQL-Generate in --debug-template mode when generating the documentation)";
 	
 	public static CommandLine parseArguments(String[] args) throws ParseException {
 		DefaultParser commandLineParser = new DefaultParser();
@@ -48,10 +62,12 @@ public class CMDConfigurations {
 		return new Options().addOption(ARG_HELP, ARG_HELP_LONG, false, ARG_HELP_MAN)
 //				.addOption(ARG_INIT, ARG_INIT_LONG, true, ARG_INIT_MAN)
 				.addOption(ARG_DIRECTORY, ARG_DIRECTORY_LONG, true, ARG_DIRECTORY_MAN)
+				.addOption(ARG_REMOTE_ONLY, ARG_REMOTE_ONLY_LONG, false, ARG_REMOTE_ONLY_MAN)
+				.addOption(ARG_IGNORE_EXAMPLES, ARG_IGNORE_EXAMPLES_LONG, false, ARG_IGNORE_EXAMPLES_MAN)
+				.addOption(ARG_IGNORE_GIT, ARG_IGNORE_GIT_LONG, false, ARG_IGNORE_GIT_MAN)
+				.addOption(ARG_NO_SITE, ARG_NO_SITE_LONG, false, ARG_NO_SITE_MAN)
 				.addOption(ARG_INCLUDE_MASTER, ARG_INCLUDE_MASTER_LONG, false, ARG_INCLUDE_MASTER_MAN)
-				.addOption(ARG_INCLUDE_ALL, ARG_INCLUDE_ALL_LONG, false, ARG_INCLUDE_ALL_MAN)
-				.addOption(ARG_PRODUCTION, ARG_PRODUCTION_LONG, false, ARG_PRODUCTION_MAN)
-				.addOption(ARG_DEBUG_TEMPLATE, ARG_DEBUG_TEMPLATE_LONG, false, ARG_DEBUG_TEMPLATE_MAN);
+				.addOption(ARG_VERBOSE, ARG_VERBOSE_LONG, false, ARG_VERBOSE_MAN);
 	}
 
 	public static void displayHelp() {

src/main/java/fr/emse/gitlab/saref/Constants.java

@@ -7,27 +7,13 @@ import java.util.regex.Pattern;
 
 public class Constants {
 	public final static String BASE = "https://saref.etsi.org/";
-	public final static String BASE_DOC = BASE + "documentation/";
 
 	public final static String LOGGER_BASE = "fr.emse.gitlab.saref.logger";
 	
 	public static final String REGEX_EXT = "(?<ext>core|saref4[a-z]{4})";
 	public static final String REGEX_VERSION = "v(?<major>[1-9][0-9]*)\\.(?<minor>[0-9]+)\\.(?<patch>[0-9]+)";
-	
-	public static final String MASTER_BRANCH = "refs/remotes/origin/master";
-	public static final String REGEX_RELEASE_BRANCH = "^refs/remotes/origin/release-" + REGEX_VERSION + "$";
-	public static final String REGEX_OTHER_BRANCH = "^refs/remotes/origin/(?<name>[^/]+)$";
-	public static final Pattern REGEX_RELEASE_BRANCH_PATTERN = Pattern.compile(REGEX_RELEASE_BRANCH);
-	public static final Pattern REGEX_OTHER_BRANCH_PATTERN = Pattern.compile(REGEX_OTHER_BRANCH);
 
-	public static final String REGEX_TERM_URI = "^" + BASE + REGEX_EXT + "/(?<localName>[^/]+)$";
-	public static final Pattern REGEX_TERM_PATTERN = Pattern.compile(REGEX_TERM_URI);
-	
-	public static final String REGEX_ONTO_URI = "^" + BASE + REGEX_EXT + "/" + REGEX_VERSION + "/$";
-	public static final Pattern REGEX_ONTO_PATTERN = Pattern.compile(REGEX_ONTO_URI);
-	
 	public static final String REGEX_ONTO_SERIES_URI = "^" + BASE + REGEX_EXT + "/$";
-	public static final Pattern REGEX_ONTO_SERIES_PATTERN = Pattern.compile(REGEX_ONTO_SERIES_URI);
 	
 	// output 
 	
@@ -37,19 +23,6 @@ public class Constants {
 	public static final String SITE_DIR = TARGET_DIR + File.separator + "site";
 	public static final String STATIC_TARGET_DIR = SITE_DIR + File.separator + "static";
 
-	public static final String SAREF_ACCESS_TOKEN = "SAREF_ACCESS_TOKEN";
-	public static final String SAREF_ACCESS_USERNAME = "SAREF_ACCESS_USERNAME";
-	
-	public static final String propertyFile = ".saref-repositories";
-
 	public static final String CONFIG = BASE + "config";
 	
-	public static final PathMatcher ttlMatcher = FileSystems.getDefault().getPathMatcher("glob:**/*.ttl");
-
-
-	public static boolean INCLUDE_MASTER = false;
-	public static boolean INCLUDE_ALL = false;
-	public static boolean PRODUCTION = false;
-	public static boolean DEBUG_TEMPLATE = false;
-
 }

src/main/java/fr/emse/gitlab/saref/Main.java

 package fr.emse.gitlab.saref;
 
-import static fr.emse.gitlab.saref.CMDConfigurations.ARG_DEBUG_TEMPLATE;
-import static fr.emse.gitlab.saref.CMDConfigurations.ARG_DIRECTORY;
-import static fr.emse.gitlab.saref.CMDConfigurations.ARG_DIRECTORY_DEFAULT;
-import static fr.emse.gitlab.saref.CMDConfigurations.ARG_HELP;
-import static fr.emse.gitlab.saref.CMDConfigurations.ARG_INCLUDE_ALL;
-import static fr.emse.gitlab.saref.CMDConfigurations.ARG_INCLUDE_MASTER;
-import static fr.emse.gitlab.saref.CMDConfigurations.ARG_PRODUCTION;
+import static fr.emse.gitlab.saref.CMDConfigurations.*;
 
 import java.awt.Desktop;
 import java.io.File;
@@ -21,6 +15,7 @@ import java.nio.file.FileSystem;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.PathMatcher;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -28,12 +23,10 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.function.Consumer;
 
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.JAXBException;
 import javax.xml.bind.Marshaller;
-import javax.xml.bind.Unmarshaller;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.ParseException;
@@ -60,7 +53,6 @@ import org.eclipse.jgit.api.errors.GitAPIException;
 import org.eclipse.jgit.api.errors.InvalidRefNameException;
 import org.eclipse.jgit.api.errors.RefAlreadyExistsException;
 import org.eclipse.jgit.api.errors.RefNotFoundException;
-import org.eclipse.jgit.revwalk.RevCommit;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -81,16 +73,18 @@ import fr.emse.gitlab.saref.jobs.CheckConfig;
 import fr.emse.gitlab.saref.jobs.CheckOWLProfile;
 import fr.emse.gitlab.saref.jobs.CheckRepositoryStructure;
 import fr.emse.gitlab.saref.jobs.CheckoutJob;
+import fr.emse.gitlab.saref.jobs.GeneratePortal;
 import fr.emse.gitlab.saref.jobs.ReadExamples;
 import fr.emse.gitlab.saref.jobs.ReadOntology;
 import fr.emse.gitlab.saref.jobs.ReadRepositories;
-import fr.emse.gitlab.saref.jobs.GeneratePortal;
 import fr.emse.gitlab.saref.utils.TestSuitesAppender;
 import fr.emse.gitlab.saref.vocabs.EX;
 
 public class Main {
 
 	static final Logger LOG = LoggerFactory.getLogger(Constants.LOGGER_BASE + ".Main");
+	public final static String BASE_DOC = Constants.BASE + "documentation/";
+	public static final PathMatcher ttlMatcher = FileSystems.getDefault().getPathMatcher("glob:**/*.ttl");
 
 	private static TestSuites testSuites = new TestSuites("SAREF pipeline");
 	private static final Map<String, String> testSuiteNames = new HashMap<>();
@@ -103,50 +97,79 @@ public class Main {
 	private static Dataset dataset;
 	private static Var VAR_TEST_SUITES = VarUtils.allocVar("testsuites");
 
+
+	static boolean remoteOnly;
+	static boolean ignoreGit;
+	static boolean includeMaster;
+	static boolean generateSite;
+	static boolean ignoreExamples;
+	static boolean verbose;
+		
+	
+	
 	public static void main(String[] args)
 			throws IOException, InterruptedException, URISyntaxException, JAXBException, ParseException, RefAlreadyExistsException, RefNotFoundException, InvalidRefNameException, CheckoutConflictException, GitAPIException {
-		parseCommandArguments(args);
+		
+		CommandLine cl = CMDConfigurations.parseArguments(args);
+		if (cl.getOptions().length == 0 || cl.hasOption(ARG_HELP)) {
+			CMDConfigurations.displayHelp();
+			return;
+		}
+
+		String dirName = cl.getOptionValue(ARG_DIRECTORY, ARG_DIRECTORY_DEFAULT);
+		directory = new File(dirName).getCanonicalFile();
+
+		remoteOnly = cl.hasOption(ARG_REMOTE_ONLY);
+		ignoreGit = cl.hasOption(ARG_IGNORE_GIT);
+		includeMaster = cl.hasOption(ARG_INCLUDE_MASTER);
+		generateSite = !cl.hasOption(ARG_NO_SITE);
+		ignoreExamples = cl.hasOption(ARG_IGNORE_EXAMPLES);
+		verbose = cl.hasOption(ARG_VERBOSE);
+					
 		setLogAppenders();
 		prepareDirectory();
 		streamManager = initializeStreamManager();
 		dataset = createFreshDataset();
 		LOG.info("Starting pipeline in " + directory);
-		Repositories repositories = readRepositories();
-		accept(repositories, Main::testRepository);		
-		checkOWLProfile();
-		new CheckConfig().doJob(dataset);
-		new GeneratePortal("Generate static files for the portal", streamManager).doJob(dataset, directory);
-		accept(repositories, Main::resetCheckout);
-		testSuites.prune();
-		reportAndExit(0);
-	}
 
-	private static void parseCommandArguments(String[] args) throws ParseException, IOException {
-		CommandLine cl = CMDConfigurations.parseArguments(args);
-		if (cl.getOptions().length == 0 || cl.hasOption(ARG_HELP)) {
-			CMDConfigurations.displayHelp();
+		Repositories repositories;
+		try {
+			repositories = new ReadRepositories("Fetching required repositories").readRepositories(directory, remoteOnly, ignoreGit, includeMaster);
+		} catch (Exception ex) {
+			reportAndExit(-1);
 			return;
 		}
-		if (cl.hasOption(ARG_INCLUDE_MASTER)) {
-			Constants.INCLUDE_MASTER = true;
+
+		if (repositories.getDefaultRepository() != null) {
+			Repository repository = repositories.getDefaultRepository();
+			boolean checkExamples = !ignoreExamples;
+			testRepository(repository, checkExamples);
 		}
-		if (cl.hasOption(ARG_INCLUDE_ALL)) {
-			Constants.INCLUDE_MASTER = true;
-			Constants.INCLUDE_ALL = true;
+		for(Repository repository : repositories.getNamedRepositories()) {
+			boolean checkExamples = !ignoreExamples && remoteOnly;
+			testRepository(repository, checkExamples);
 		}
-		if (cl.hasOption(ARG_PRODUCTION)) {
-			Constants.PRODUCTION = true;
+
+		checkOWLProfile();
+		new CheckConfig().doJob(dataset, ignoreExamples);
+
+		if (generateSite) {
+			new GeneratePortal("Generate static files for the portal", streamManager).doJob(dataset, directory, verbose);
 		}
-		if (cl.hasOption(ARG_DEBUG_TEMPLATE)) {
-			Constants.DEBUG_TEMPLATE = true;
+
+
+		if (repositories.getDefaultRepository() != null && !ignoreGit) {
+			Repository repository = repositories.getDefaultRepository();
+			resetCheckout(repository);
 		}
-		String dirName = cl.getOptionValue(ARG_DIRECTORY, ARG_DIRECTORY_DEFAULT);
-		if (dirName.equals("")) {
-			dirName = ARG_DIRECTORY_DEFAULT;
+		for(Repository repository : repositories.getNamedRepositories()) {
+			resetCheckout(repository);
 		}
-		directory = new File(dirName).getCanonicalFile();
-	}
 
+		testSuites.clean();
+		
+		reportAndExit((int) -Math.signum(testSuites.getErrors()));
+	}
 
 	private static void setLogAppenders() throws IOException {
 		File logFile = new File(directory, Constants.LOG_FILE_NAME);
@@ -156,18 +179,20 @@ public class Main {
 
 		org.apache.log4j.Logger loggerBase = org.apache.log4j.Logger.getLogger(Constants.LOGGER_BASE);
 		TestSuitesAppender appender = new TestSuitesAppender(testSuites);
-		loggerBase.addAppender(appender);		
+		loggerBase.addAppender(appender);
 	}	
 
 	private static void prepareDirectory() throws IOException {
 		target = new File(directory, Constants.TARGET_DIR);
 		FileUtils.forceMkdir(target);
-		File staticTargetDir = new File(directory, Constants.STATIC_TARGET_DIR);
-		File staticTargetZip = new File(directory, Constants.STATIC_TARGET_DIR + ".zip");
-		URL url = Main.class.getClassLoader().getResource("static.zip");
-		FileUtils.copyURLToFile(url, staticTargetZip);
-    	UnzipFile.unzip(staticTargetZip, staticTargetDir);
-    	FileUtils.deleteQuietly(staticTargetZip);
+		if(generateSite) {
+			File staticTargetDir = new File(directory, Constants.STATIC_TARGET_DIR);
+			File staticTargetZip = new File(directory, Constants.STATIC_TARGET_DIR + ".zip");
+			URL url = Main.class.getClassLoader().getResource("static.zip");
+			FileUtils.copyURLToFile(url, staticTargetZip);
+	    	UnzipFile.unzip(staticTargetZip, staticTargetDir);
+	    	FileUtils.deleteQuietly(staticTargetZip);
+		}
 	}
 
 
@@ -188,26 +213,7 @@ public class Main {
 		return dataset;
 	}
 
-	private static void accept(Repositories repositories, Consumer<Repository> consumer) {
-		if (repositories.getDefaultRepository() != null) {
-			Repository repository = repositories.getDefaultRepository();
-			consumer.accept(repository);
-		}
-		for(Repository repository : repositories.getNamedRepositories()) {
-			consumer.accept(repository);
-		}
-	}
-
-	private static Repositories readRepositories() {
-		try {
-			return new ReadRepositories("Fetching required repositories").readRepositories(directory);
-		} catch (Exception ex) {
-			reportAndExit(-1);
-			return null;
-		}
-	}
-
-	private static void testRepository(Repository repository) {
+	private static void testRepository(Repository repository, boolean checkExamples) {
 		for (Version version : repository.getVersions()) {
 			try {
 				String versionTestSuiteName = version.toString() + " - testing repository structure";
@@ -228,10 +234,13 @@ public class Main {
 				continue;
 			}
 
+			if(!checkExamples) {
+				return;
+			}
 			File examplesDir = new File(version.getRepository().getDirectory(), "examples");
 			try {
 				for (Iterator<Path> it = Files.walk(examplesDir.toPath()).filter(p -> {
-					return Constants.ttlMatcher.matches(p);
+					return ttlMatcher.matches(p);
 				}).iterator();it.hasNext();) {
 					Path p = it.next();
 					Path rel = examplesDir.toPath().relativize(p);
@@ -293,7 +302,7 @@ public class Main {
 		sm.setLocationMapper(mapper);
 		Files.walk(dirPath).forEach((p) -> {
 			String relativePath = dirPath.relativize(p).toString().replace("\\", "/");
-			String fileurl = Constants.BASE_DOC + relativePath;
+			String fileurl = BASE_DOC + relativePath;
 			if(uri.getScheme().equals("jar")) {
 				mapper.addAltEntry(fileurl, p.toString().substring(1));
 			} else {
@@ -312,7 +321,7 @@ public class Main {
 			}
 		} catch (IOException | GitAPIException ex) {
 			LOG.warn("Error while reseting repository " + repository, ex);
-		}		
+		}
 	}
 	
 	private static void reportAndExit(int code) {
@@ -326,27 +335,29 @@ public class Main {
 			final StringWriter sw = new StringWriter();
 			jaxbMarshaller.marshal(testSuites, sw);
 
-			// generate the report.html
-			File reportHTML = new File(directory, Constants.SITE_DIR + File.separator + "report.html");
-			try (IndentedWriter writer = new IndentedWriter(new FileOutputStream(reportHTML));) {
-				Context context = ContextUtils.build(writer).setBase(Constants.BASE)
-						.setDebugTemplate(!Constants.PRODUCTION).setStreamManager(streamManager).build();
-				BindingHashMap binding = new BindingHashMap();
-				binding.add(VAR_TEST_SUITES, NodeFactory.createLiteral(sw.toString()));
-				List<Binding> bindings = new ArrayList<>();
-				bindings.add(binding);
-				String query = IOUtils.toString(
-						streamManager.open(new LookUpRequest("documentation/report/main.rqg", SPARQLExt.MEDIA_TYPE)),
-						StandardCharsets.UTF_8);
-				RootPlan reportPlan = PlanFactory.create(query, Constants.BASE_DOC);
-				reportPlan.execTemplateStream(bindings, context);
-			}
-
-			if (openBrowser) {
-				if (Desktop.isDesktopSupported() && Desktop.getDesktop().isSupported(Desktop.Action.BROWSE)) {
-					Desktop.getDesktop().browse(reportHTML.toURI());
-				} else {
-					System.out.println("\n\n\nURL to the SAREF pipeline report:\n" + reportHTML.toURI());
+			if(generateSite) {
+				// generate the report.html
+				File reportHTML = new File(directory, Constants.SITE_DIR + File.separator + "report.html");
+				try (IndentedWriter writer = new IndentedWriter(new FileOutputStream(reportHTML));) {
+					Context context = ContextUtils.build(writer).setBase(Constants.BASE)
+							.setDebugTemplate(verbose).setStreamManager(streamManager).build();
+					BindingHashMap binding = new BindingHashMap();
+					binding.add(VAR_TEST_SUITES, NodeFactory.createLiteral(sw.toString()));
+					List<Binding> bindings = new ArrayList<>();
+					bindings.add(binding);
+					String query = IOUtils.toString(
+							streamManager.open(new LookUpRequest("documentation/report/main.rqg", SPARQLExt.MEDIA_TYPE)),
+							StandardCharsets.UTF_8);
+					RootPlan reportPlan = PlanFactory.create(query, BASE_DOC);
+					reportPlan.execTemplateStream(bindings, context);
+				}
+	
+				if (openBrowser) {
+					if (Desktop.isDesktopSupported() && Desktop.getDesktop().isSupported(Desktop.Action.BROWSE)) {
+						Desktop.getDesktop().browse(reportHTML.toURI());
+					} else {
+						System.out.println("\n\n\nURL to the SAREF pipeline report:\n" + reportHTML.toURI());
+					}
 				}
 			}
 		} catch (JAXBException | IOException ex) {

src/main/java/fr/emse/gitlab/saref/entities/tests/TestSuites.java

@@ -5,6 +5,8 @@ package fr.emse.gitlab.saref.entities.tests;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -14,6 +16,10 @@ import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
 import javax.xml.bind.annotation.XmlType;
 
+import fr.emse.gitlab.saref.entities.git.MasterVersion;
+import fr.emse.gitlab.saref.entities.git.ReleaseVersion;
+import fr.emse.gitlab.saref.entities.git.Version;
+
 /**
  * @author Omar Qawasmeh
  *
@@ -74,13 +80,33 @@ public class TestSuites implements Serializable {
 		return testSuite;
 	}
 
-	public void prune() {
+	public void clean() {
 		testSuitesList.removeIf((ts) -> {
 			return ts.getErrors() + ts.getFailures() == 0;
 		});		
 		testSuitesMap.values().removeIf((ts) -> {
 			return ts.getErrors() + ts.getFailures() == 0;
-		});		
+		});