Commit ce30ef66 authored by Maxime Lefrançois's avatar Maxime Lefrançois
Browse files

changed flags

parent 71cb160d

src/main/java/fr/emse/gitlab/saref/CMDConfigurations.java

+30 −14
Original line number Diff line number Diff line
@@ -18,25 +18,39 @@ public class CMDConfigurations { 
	public static final String ARG_HELP = "h";

	public static final String ARG_HELP_LONG = "help";

	public static final String ARG_HELP_MAN = "Show help";

	

	public static final String ARG_INIT = "i";

	public static final String ARG_INIT_LONG = "init";

	public static final String ARG_INIT_MAN = "Initialize a new SAREF extension development in the current directory";



	public static final String ARG_DIRECTORY = "d";

	public static final String ARG_DIRECTORY_DEFAULT = "";	

	public static final String ARG_DIRECTORY_LONG = "dir";

	public static final String ARG_DIRECTORY_MAN = "Location of the SAREF extension directory (default is .)";

	public static final String ARG_INCLUDE_ALL = "a";

	public static final String ARG_INCLUDE_ALL_LONG = "all";

	public static final String ARG_INCLUDE_ALL_MAN = "Include all branches";



	public static final String ARG_REMOTE_ONLY = "r";

	public static final String ARG_REMOTE_ONLY_LONG = "remote-only";

	public static final String ARG_REMOTE_ONLY_MAN = "Do not check the directory itself. Only consider the repositories listed in the `.saref-repositories.yml` document. Used to generate the website for several extensions.";



	public static final String ARG_IGNORE_GIT = "g";

	public static final String ARG_IGNORE_GIT_LONG = "no-git";

	public static final String ARG_IGNORE_GIT_MAN = "Only check the current state of the repository";



	public static final String ARG_IGNORE_EXAMPLES = "e";

	public static final String ARG_IGNORE_EXAMPLES_LONG = "ignore-examples";

	public static final String ARG_IGNORE_EXAMPLES_MAN = "Only check the SAREF extension ontology. Ignore the examples.";

	

	public static final String ARG_NO_SITE = "s";

	public static final String ARG_NO_SITE_LONG = "no-site";

	public static final String ARG_NO_SITE_MAN = "Do not generate the static website";

	

	public static final String ARG_INCLUDE_MASTER = "m";

	public static final String ARG_INCLUDE_MASTER_LONG = "master";

	public static final String ARG_INCLUDE_MASTER_MAN = "Include the HEAD of the master branches";

	public static final String ARG_PRODUCTION = "p";

	public static final String ARG_PRODUCTION_LONG = "production";

	public static final String ARG_PRODUCTION_MAN = "Production mode: the directory does not contain a SAREF extension, but only a file `.saref-repositories.yml`";

	public static final String ARG_DEBUG_TEMPLATE = "debug";

	public static final String ARG_DEBUG_TEMPLATE_LONG = "debug-template";

	public static final String ARG_DEBUG_TEMPLATE_MAN = "Use the debug-template mode of SPARQL-Generate when generating the documentation";

	public static final String ARG_INCLUDE_MASTER_MAN = "Check the master branches of the remote repositories listed in the `.saref-repositories.yml` document";



	public static final String ARG_VERBOSE = "v";

	public static final String ARG_VERBOSE_LONG = "verbose";

	public static final String ARG_VERBOSE_MAN = "Use verbose mode. (For example, use SPARQL-Generate in --debug-template mode when generating the documentation)";

	

	public static CommandLine parseArguments(String[] args) throws ParseException {

		DefaultParser commandLineParser = new DefaultParser();
@@ -48,10 +62,12 @@ public class CMDConfigurations { 
		return new Options().addOption(ARG_HELP, ARG_HELP_LONG, false, ARG_HELP_MAN)

//				.addOption(ARG_INIT, ARG_INIT_LONG, true, ARG_INIT_MAN)

				.addOption(ARG_DIRECTORY, ARG_DIRECTORY_LONG, true, ARG_DIRECTORY_MAN)

				.addOption(ARG_REMOTE_ONLY, ARG_REMOTE_ONLY_LONG, false, ARG_REMOTE_ONLY_MAN)

				.addOption(ARG_IGNORE_EXAMPLES, ARG_IGNORE_EXAMPLES_LONG, false, ARG_IGNORE_EXAMPLES_MAN)

				.addOption(ARG_IGNORE_GIT, ARG_IGNORE_GIT_LONG, false, ARG_IGNORE_GIT_MAN)

				.addOption(ARG_NO_SITE, ARG_NO_SITE_LONG, false, ARG_NO_SITE_MAN)

				.addOption(ARG_INCLUDE_MASTER, ARG_INCLUDE_MASTER_LONG, false, ARG_INCLUDE_MASTER_MAN)

				.addOption(ARG_INCLUDE_ALL, ARG_INCLUDE_ALL_LONG, false, ARG_INCLUDE_ALL_MAN)

				.addOption(ARG_PRODUCTION, ARG_PRODUCTION_LONG, false, ARG_PRODUCTION_MAN)

				.addOption(ARG_DEBUG_TEMPLATE, ARG_DEBUG_TEMPLATE_LONG, false, ARG_DEBUG_TEMPLATE_MAN);

				.addOption(ARG_VERBOSE, ARG_VERBOSE_LONG, false, ARG_VERBOSE_MAN);

	}



	public static void displayHelp() {

src/main/java/fr/emse/gitlab/saref/Constants.java

+0 −27
Original line number Diff line number Diff line
@@ -7,27 +7,13 @@ import java.util.regex.Pattern; 


public class Constants {

	public final static String BASE = "https://saref.etsi.org/";

	public final static String BASE_DOC = BASE + "documentation/";



	public final static String LOGGER_BASE = "fr.emse.gitlab.saref.logger";

	

	public static final String REGEX_EXT = "(?<ext>core|saref4[a-z]{4})";

	public static final String REGEX_VERSION = "v(?<major>[1-9][0-9]*)\\.(?<minor>[0-9]+)\\.(?<patch>[0-9]+)";



	public static final String MASTER_BRANCH = "refs/remotes/origin/master";

	public static final String REGEX_RELEASE_BRANCH = "^refs/remotes/origin/release-" + REGEX_VERSION + "$";

	public static final String REGEX_OTHER_BRANCH = "^refs/remotes/origin/(?<name>[^/]+)$";

	public static final Pattern REGEX_RELEASE_BRANCH_PATTERN = Pattern.compile(REGEX_RELEASE_BRANCH);

	public static final Pattern REGEX_OTHER_BRANCH_PATTERN = Pattern.compile(REGEX_OTHER_BRANCH);



	public static final String REGEX_TERM_URI = "^" + BASE + REGEX_EXT + "/(?<localName>[^/]+)$";

	public static final Pattern REGEX_TERM_PATTERN = Pattern.compile(REGEX_TERM_URI);

	

	public static final String REGEX_ONTO_URI = "^" + BASE + REGEX_EXT + "/" + REGEX_VERSION + "/$";

	public static final Pattern REGEX_ONTO_PATTERN = Pattern.compile(REGEX_ONTO_URI);

	

	public static final String REGEX_ONTO_SERIES_URI = "^" + BASE + REGEX_EXT + "/$";

	public static final Pattern REGEX_ONTO_SERIES_PATTERN = Pattern.compile(REGEX_ONTO_SERIES_URI);

	

	// output
@@ -37,19 +23,6 @@ public class Constants { 
	public static final String SITE_DIR = TARGET_DIR + File.separator + "site";

	public static final String STATIC_TARGET_DIR = SITE_DIR + File.separator + "static";



	public static final String SAREF_ACCESS_TOKEN = "SAREF_ACCESS_TOKEN";

	public static final String SAREF_ACCESS_USERNAME = "SAREF_ACCESS_USERNAME";

	

	public static final String propertyFile = ".saref-repositories";



	public static final String CONFIG = BASE + "config";

	

	public static final PathMatcher ttlMatcher = FileSystems.getDefault().getPathMatcher("glob:**/*.ttl");





	public static boolean INCLUDE_MASTER = false;

	public static boolean INCLUDE_ALL = false;

	public static boolean PRODUCTION = false;

	public static boolean DEBUG_TEMPLATE = false;



}

src/main/java/fr/emse/gitlab/saref/Main.java

+101 −90
Original line number Diff line number Diff line 
package fr.emse.gitlab.saref;



import static fr.emse.gitlab.saref.CMDConfigurations.ARG_DEBUG_TEMPLATE;

import static fr.emse.gitlab.saref.CMDConfigurations.ARG_DIRECTORY;

import static fr.emse.gitlab.saref.CMDConfigurations.ARG_DIRECTORY_DEFAULT;

import static fr.emse.gitlab.saref.CMDConfigurations.ARG_HELP;

import static fr.emse.gitlab.saref.CMDConfigurations.ARG_INCLUDE_ALL;

import static fr.emse.gitlab.saref.CMDConfigurations.ARG_INCLUDE_MASTER;

import static fr.emse.gitlab.saref.CMDConfigurations.ARG_PRODUCTION;

import static fr.emse.gitlab.saref.CMDConfigurations.*;



import java.awt.Desktop;

import java.io.File;
@@ -21,6 +15,7 @@ import java.nio.file.FileSystem; 
import java.nio.file.FileSystems;

import java.nio.file.Files;

import java.nio.file.Path;

import java.nio.file.PathMatcher;

import java.nio.file.Paths;

import java.util.ArrayList;

import java.util.Collections;
@@ -28,12 +23,10 @@ import java.util.HashMap; 
import java.util.Iterator;

import java.util.List;

import java.util.Map;

import java.util.function.Consumer;



import javax.xml.bind.JAXBContext;

import javax.xml.bind.JAXBException;

import javax.xml.bind.Marshaller;

import javax.xml.bind.Unmarshaller;



import org.apache.commons.cli.CommandLine;

import org.apache.commons.cli.ParseException;
@@ -60,7 +53,6 @@ import org.eclipse.jgit.api.errors.GitAPIException; 
import org.eclipse.jgit.api.errors.InvalidRefNameException;

import org.eclipse.jgit.api.errors.RefAlreadyExistsException;

import org.eclipse.jgit.api.errors.RefNotFoundException;

import org.eclipse.jgit.revwalk.RevCommit;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;
@@ -81,16 +73,18 @@ import fr.emse.gitlab.saref.jobs.CheckConfig; 
import fr.emse.gitlab.saref.jobs.CheckOWLProfile;

import fr.emse.gitlab.saref.jobs.CheckRepositoryStructure;

import fr.emse.gitlab.saref.jobs.CheckoutJob;

import fr.emse.gitlab.saref.jobs.GeneratePortal;

import fr.emse.gitlab.saref.jobs.ReadExamples;

import fr.emse.gitlab.saref.jobs.ReadOntology;

import fr.emse.gitlab.saref.jobs.ReadRepositories;

import fr.emse.gitlab.saref.jobs.GeneratePortal;

import fr.emse.gitlab.saref.utils.TestSuitesAppender;

import fr.emse.gitlab.saref.vocabs.EX;



public class Main {



	static final Logger LOG = LoggerFactory.getLogger(Constants.LOGGER_BASE + ".Main");

	public final static String BASE_DOC = Constants.BASE + "documentation/";

	public static final PathMatcher ttlMatcher = FileSystems.getDefault().getPathMatcher("glob:**/*.ttl");



	private static TestSuites testSuites = new TestSuites("SAREF pipeline");

	private static final Map<String, String> testSuiteNames = new HashMap<>();
@@ -103,50 +97,79 @@ public class Main { 
	private static Dataset dataset;

	private static Var VAR_TEST_SUITES = VarUtils.allocVar("testsuites");





	static boolean remoteOnly;

	static boolean ignoreGit;

	static boolean includeMaster;

	static boolean generateSite;

	static boolean ignoreExamples;

	static boolean verbose;

		

	

	

	public static void main(String[] args)

			throws IOException, InterruptedException, URISyntaxException, JAXBException, ParseException, RefAlreadyExistsException, RefNotFoundException, InvalidRefNameException, CheckoutConflictException, GitAPIException {

		parseCommandArguments(args);

		

		CommandLine cl = CMDConfigurations.parseArguments(args);

		if (cl.getOptions().length == 0 || cl.hasOption(ARG_HELP)) {

			CMDConfigurations.displayHelp();

			return;

		}



		String dirName = cl.getOptionValue(ARG_DIRECTORY, ARG_DIRECTORY_DEFAULT);

		directory = new File(dirName).getCanonicalFile();



		remoteOnly = cl.hasOption(ARG_REMOTE_ONLY);

		ignoreGit = cl.hasOption(ARG_IGNORE_GIT);

		includeMaster = cl.hasOption(ARG_INCLUDE_MASTER);

		generateSite = !cl.hasOption(ARG_NO_SITE);

		ignoreExamples = cl.hasOption(ARG_IGNORE_EXAMPLES);

		verbose = cl.hasOption(ARG_VERBOSE);

					

		setLogAppenders();

		prepareDirectory();

		streamManager = initializeStreamManager();

		dataset = createFreshDataset();

		LOG.info("Starting pipeline in " + directory);

		Repositories repositories = readRepositories();

		accept(repositories, Main::testRepository);		

		checkOWLProfile();

		new CheckConfig().doJob(dataset);

		new GeneratePortal("Generate static files for the portal", streamManager).doJob(dataset, directory);

		accept(repositories, Main::resetCheckout);

		testSuites.prune();

		reportAndExit(0);

	}



	private static void parseCommandArguments(String[] args) throws ParseException, IOException {

		CommandLine cl = CMDConfigurations.parseArguments(args);

		if (cl.getOptions().length == 0 || cl.hasOption(ARG_HELP)) {

			CMDConfigurations.displayHelp();

		Repositories repositories;

		try {

			repositories = new ReadRepositories("Fetching required repositories").readRepositories(directory, remoteOnly, ignoreGit, includeMaster);

		} catch (Exception ex) {

			reportAndExit(-1);

			return;

		}

		if (cl.hasOption(ARG_INCLUDE_MASTER)) {

			Constants.INCLUDE_MASTER = true;

		}

		if (cl.hasOption(ARG_INCLUDE_ALL)) {

			Constants.INCLUDE_MASTER = true;

			Constants.INCLUDE_ALL = true;



		if (repositories.getDefaultRepository() != null) {

			Repository repository = repositories.getDefaultRepository();

			boolean checkExamples = !ignoreExamples;

			testRepository(repository, checkExamples);

		}

		if (cl.hasOption(ARG_PRODUCTION)) {

			Constants.PRODUCTION = true;

		for(Repository repository : repositories.getNamedRepositories()) {

			boolean checkExamples = !ignoreExamples && remoteOnly;

			testRepository(repository, checkExamples);

		}

		if (cl.hasOption(ARG_DEBUG_TEMPLATE)) {

			Constants.DEBUG_TEMPLATE = true;



		checkOWLProfile();

		new CheckConfig().doJob(dataset, ignoreExamples);



		if (generateSite) {

			new GeneratePortal("Generate static files for the portal", streamManager).doJob(dataset, directory, verbose);

		}

		String dirName = cl.getOptionValue(ARG_DIRECTORY, ARG_DIRECTORY_DEFAULT);

		if (dirName.equals("")) {

			dirName = ARG_DIRECTORY_DEFAULT;





		if (repositories.getDefaultRepository() != null && !ignoreGit) {

			Repository repository = repositories.getDefaultRepository();

			resetCheckout(repository);

		}

		directory = new File(dirName).getCanonicalFile();

		for(Repository repository : repositories.getNamedRepositories()) {

			resetCheckout(repository);

		}



		testSuites.clean();

		

		reportAndExit((int) -Math.signum(testSuites.getErrors()));

	}



	private static void setLogAppenders() throws IOException {

		File logFile = new File(directory, Constants.LOG_FILE_NAME);
@@ -162,6 +185,7 @@ public class Main { 
	private static void prepareDirectory() throws IOException {

		target = new File(directory, Constants.TARGET_DIR);

		FileUtils.forceMkdir(target);

		if(generateSite) {

			File staticTargetDir = new File(directory, Constants.STATIC_TARGET_DIR);

			File staticTargetZip = new File(directory, Constants.STATIC_TARGET_DIR + ".zip");

			URL url = Main.class.getClassLoader().getResource("static.zip");
@@ -169,6 +193,7 @@ public class Main { 
	    	UnzipFile.unzip(staticTargetZip, staticTargetDir);

	    	FileUtils.deleteQuietly(staticTargetZip);

		}

	}





	private static Dataset createFreshDataset() throws IOException {
@@ -188,26 +213,7 @@ public class Main { 
		return dataset;

	}



	private static void accept(Repositories repositories, Consumer<Repository> consumer) {

		if (repositories.getDefaultRepository() != null) {

			Repository repository = repositories.getDefaultRepository();

			consumer.accept(repository);

		}

		for(Repository repository : repositories.getNamedRepositories()) {

			consumer.accept(repository);

		}

	}



	private static Repositories readRepositories() {

		try {

			return new ReadRepositories("Fetching required repositories").readRepositories(directory);

		} catch (Exception ex) {

			reportAndExit(-1);

			return null;

		}

	}



	private static void testRepository(Repository repository) {

	private static void testRepository(Repository repository, boolean checkExamples) {

		for (Version version : repository.getVersions()) {

			try {

				String versionTestSuiteName = version.toString() + " - testing repository structure";
@@ -228,10 +234,13 @@ public class Main { 
				continue;

			}



			if(!checkExamples) {

				return;

			}

			File examplesDir = new File(version.getRepository().getDirectory(), "examples");

			try {

				for (Iterator<Path> it = Files.walk(examplesDir.toPath()).filter(p -> {

					return Constants.ttlMatcher.matches(p);

					return ttlMatcher.matches(p);

				}).iterator();it.hasNext();) {

					Path p = it.next();

					Path rel = examplesDir.toPath().relativize(p);
@@ -293,7 +302,7 @@ public class Main { 
		sm.setLocationMapper(mapper);

		Files.walk(dirPath).forEach((p) -> {

			String relativePath = dirPath.relativize(p).toString().replace("\\", "/");

			String fileurl = Constants.BASE_DOC + relativePath;

			String fileurl = BASE_DOC + relativePath;

			if(uri.getScheme().equals("jar")) {

				mapper.addAltEntry(fileurl, p.toString().substring(1));

			} else {
@@ -326,11 +335,12 @@ public class Main { 
			final StringWriter sw = new StringWriter();

			jaxbMarshaller.marshal(testSuites, sw);



			if(generateSite) {

				// generate the report.html

				File reportHTML = new File(directory, Constants.SITE_DIR + File.separator + "report.html");

				try (IndentedWriter writer = new IndentedWriter(new FileOutputStream(reportHTML));) {

					Context context = ContextUtils.build(writer).setBase(Constants.BASE)

						.setDebugTemplate(!Constants.PRODUCTION).setStreamManager(streamManager).build();

							.setDebugTemplate(verbose).setStreamManager(streamManager).build();

					BindingHashMap binding = new BindingHashMap();

					binding.add(VAR_TEST_SUITES, NodeFactory.createLiteral(sw.toString()));

					List<Binding> bindings = new ArrayList<>();
@@ -338,7 +348,7 @@ public class Main { 
					String query = IOUtils.toString(

							streamManager.open(new LookUpRequest("documentation/report/main.rqg", SPARQLExt.MEDIA_TYPE)),

							StandardCharsets.UTF_8);

				RootPlan reportPlan = PlanFactory.create(query, Constants.BASE_DOC);

					RootPlan reportPlan = PlanFactory.create(query, BASE_DOC);

					reportPlan.execTemplateStream(bindings, context);

				}
@@ -349,6 +359,7 @@ public class Main { 
						System.out.println("\n\n\nURL to the SAREF pipeline report:\n" + reportHTML.toURI());

					}

				}

			}

		} catch (JAXBException | IOException ex) {

			LOG.error("Exception:", ex);

			ex.printStackTrace();

src/main/java/fr/emse/gitlab/saref/entities/tests/TestSuites.java

+28 −2
Original line number Diff line number Diff line
@@ -5,6 +5,8 @@ package fr.emse.gitlab.saref.entities.tests; 


import java.io.Serializable;

import java.util.ArrayList;

import java.util.Collections;

import java.util.Comparator;

import java.util.HashMap;

import java.util.List;

import java.util.Map;
@@ -14,6 +16,10 @@ import javax.xml.bind.annotation.XmlElement; 
import javax.xml.bind.annotation.XmlRootElement;

import javax.xml.bind.annotation.XmlType;



import fr.emse.gitlab.saref.entities.git.MasterVersion;

import fr.emse.gitlab.saref.entities.git.ReleaseVersion;

import fr.emse.gitlab.saref.entities.git.Version;



/**

 * @author Omar Qawasmeh

 *
@@ -74,13 +80,33 @@ public class TestSuites implements Serializable { 
		return testSuite;

	}



	public void prune() {

	public void clean() {

		testSuitesList.removeIf((ts) -> {

			return ts.getErrors() + ts.getFailures() == 0;

		});		

		testSuitesMap.values().removeIf((ts) -> {

			return ts.getErrors() + ts.getFailures() == 0;

		});

		

		testSuitesList.forEach((ts)->{

			// put errors first

			Collections.sort(ts.getTestCases(), new Comparator<TestCase>() {

				@Override

				public int compare(TestCase o1, TestCase o2) {

					if (o1.getStatus() == o2.getStatus()) {

						return 0;

					}

					if (o1.getStatus() == TestCase.Status.ERROR.getName()) {

						return 1;

					}

					if (o2.getStatus() == TestCase.Status.ERROR.getName()) {

						return 1;

					}

					return 0;

				}

			});

			

		});

	}



}
 
 No newline at end of file

src/main/java/fr/emse/gitlab/saref/jobs/CheckConfig.java

+25 −20
Original line number Diff line number Diff line
@@ -54,7 +54,7 @@ public class CheckConfig { 
			"} GROUP BY ?term \n"

			+ "ORDER BY ?term";



	public void doJob(Dataset dataset) {

	public void doJob(Dataset dataset, boolean ignoreExamples) {

		dataset.begin(ReadWrite.READ);

		Model config = dataset.getNamedModel(Constants.CONFIG);
@@ -69,6 +69,8 @@ public class CheckConfig { 
			}

		}

		

		if(!ignoreExamples) {

			

			// check which terms are exemplified but not defined

			logger = LoggerFactory.getLogger(Constants.LOGGER_BASE + "." + "Terms that have examples, but never defined");

			try (QueryExecution exec = QueryExecutionFactory.create(FIND_EXAMPLIFIED_UNDEFINED, config);) {
@@ -90,6 +92,9 @@ public class CheckConfig { 
					logger.warn(String.format("Term %s is defined in %s, but there is no example that use it.", term, ontology));

				}

			}

		}





		dataset.end();

	}

}