checked carefully all clauses (5f5e1572) · Commits · SAREF / saref-pipeline

src/main/java/fr/emse/gitlab/saref/SAREF.java

+1 −1

Original line number	Diff line number	Diff line
		@@ -36,7 +36,7 @@ public class SAREF {

		private final static ResourceBundle BUNDLE = ResourceBundle.getBundle("messages/SAREF");
		public final static String BASE = "https://saref.etsi.org/";
		public final static String FORGE = "https://forge.etsi.org/rep/saref/";
		public final static String FORGE = "https://forge.etsi.org/rep/SAREF/";
		public final static String LICENSE = "https://forge.etsi.org/etsi-software-license";
		public final static String PUBLISHER = "https://www.etsi.org/";

src/main/java/fr/emse/gitlab/saref/SAREFErrorLogger.java

+16 −0

Original line number	Diff line number	Diff line
		@@ -75,6 +75,22 @@ public abstract class SAREFErrorLogger {
		log(errorLogger, msg, t, errorModes);
		}

		protected void logWarning(String msg) {
		errorLogger.warn(msg);
		}

		protected void logWarning(String msg, Throwable t) {
		errorLogger.warn(msg, t);
		}

		protected void logError(String msg) {
		errorLogger.error(msg);
		}

		protected void logError(String msg, Throwable t) {
		errorLogger.error(msg, t);
		}

		protected void log(Logger logger, String msg, SAREFPipeline.Mode... errorModes) {
		if (Arrays.asList(errorModes).contains(pipeline.mode)) {
		logger.error(msg);

src/main/java/fr/emse/gitlab/saref/SAREFPipeline.java

+8 −2

Original line number	Diff line number	Diff line
		@@ -36,6 +36,7 @@ import javax.xml.bind.JAXBException;
		import javax.xml.bind.Marshaller;

		import org.apache.commons.io.FileUtils;
		import org.apache.jena.atlas.logging.Log;
		import org.apache.log4j.Layout;
		import org.apache.log4j.PatternLayout;
		import org.slf4j.Logger;
		@@ -126,6 +127,7 @@ public class SAREFPipeline {

		sourcesManager = new SourcesManager(this, logger);
		sourcesManager.fetchRepositories();

		sourcesManager.check();

		siteManager.generateSite();
		@@ -135,15 +137,19 @@ public class SAREFPipeline {
		testSuites.clean();
		if (testSuites.getErrors() > 0) {
		throw new SAREFPipelineException();
		} else {
		logger.info("The SAREF Pipeline completed successfully");
		}

		return writeReport();
		} catch (IOException ex) {
		LOG.warn("IOException:", ex);
		writeReport();
		throw new SAREFPipelineException(ex);
		return true;
		} catch (SAREFPipelineException ex) {
		LOG.warn("SAREFPipelineException:", ex);
		writeReport();
		throw ex;
		return true;
		}
		}

src/main/java/fr/emse/gitlab/saref/checkers/AbstractShaclChecker.java

+50 −61

Original line number	Diff line number	Diff line
		@@ -27,10 +27,10 @@ package fr.emse.gitlab.saref.checkers;

		import java.io.InputStream;
		import java.io.StringWriter;
		import java.util.ArrayList;
		import java.util.HashMap;
		import java.util.List;
		import java.util.HashSet;
		import java.util.Map;
		import java.util.Set;
		import java.util.regex.Matcher;
		import java.util.regex.Pattern;

		@@ -46,6 +46,7 @@ import org.apache.jena.rdf.model.Model;
		import org.apache.jena.rdf.model.RDFNode;
		import org.apache.jena.rdf.model.Resource;
		import org.apache.jena.rdf.model.ResourceFactory;
		import org.apache.jena.sparql.util.FmtUtils;
		import org.slf4j.Logger;
		import org.slf4j.LoggerFactory;
		import org.topbraid.jenax.util.JenaUtil;
		@@ -54,7 +55,6 @@ import org.topbraid.shacl.vocabulary.SH;

		import fr.emse.gitlab.saref.SAREF;
		import fr.emse.gitlab.saref.SAREFPipeline;
		import fr.emse.gitlab.saref.SAREFPipeline.Mode;
		import fr.emse.gitlab.saref.SAREFPipelineException;
		import fr.emse.gitlab.saref.managers.RepositoryManager;
		import fr.emse.gitlab.saref.vocabs.SHACL;
		@@ -67,20 +67,6 @@ public abstract class AbstractShaclChecker extends AbstractClauseChecker {
		public static final Pattern SHAPE_PATTERN = Pattern.compile("^(?<shape>Clause((_[0-9]+)+))_Checker$");
		private static final String NS = "https://saref.etsi.org/shape#";

		protected enum Severity {
		WARNING(SHACL.Warning), VIOLATION(SHACL.Violation);

		private final Resource resource;

		private Severity(Resource resource) {
		this.resource = resource;
		}

		public Resource getResource() {
		return resource;
		}
		}

		protected static interface MessageResource {
		default Resource asResource() {
		return ResourceFactory.createResource(NS + toString());
		@@ -88,13 +74,13 @@ public abstract class AbstractShaclChecker extends AbstractClauseChecker {
		}

		protected static final String SELECT_VIOLATION = "PREFIX sh: <http://www.w3.org/ns/shacl#>\n"
		+ "SELECT ?severity ?resultMessage ?focusNode ?value \n" + "WHERE { \n"
		+ "SELECT ?sourceShape ?severity ?resultMessage ?focusNode ?value \n" + "WHERE { \n"
		+ " ?violation sh:resultSeverity ?severity ; sh:resultMessage ?resultMessage ; sh:focusNode ?focusNode .\n"
		+ " OPTIONAL { ?violation sh:value ?value . } \n" + "}"
		+ " OPTIONAL { ?violation sh:value ?value . } \n"
		+ " OPTIONAL { ?violation sh:sourceShape ?sourceShape . } \n" + "}"
		+ "ORDER BY ?severity ?resultMessage ?focusNode ";

		protected final Model shapeModel;
		protected final Severity defaultSeverity;

		public AbstractShaclChecker(RepositoryManager repositoryManager, Class<?> clazz, String name)
		throws SAREFPipelineException {
		@@ -111,11 +97,6 @@ public abstract class AbstractShaclChecker extends AbstractClauseChecker {
		} catch (Exception ex) {
		throw new SAREFPipelineException("Exception while reading the shape file", ex);
		}
		if (pipeline.mode == Mode.DEVELOP) {
		defaultSeverity = Severity.WARNING;
		} else {
		defaultSeverity = Severity.VIOLATION;
		}
		}

		public AbstractShaclChecker(RepositoryManager repositoryManager, Class<?> clazz) throws SAREFPipelineException {
		@@ -124,23 +105,35 @@ public abstract class AbstractShaclChecker extends AbstractClauseChecker {

		protected abstract void updateShapeModel();

		protected Model getModel() {
		return datasetManager.getModel(version);
		}

		@Override
		public final void checkClause() throws SAREFPipelineException {
		Dataset dataset = datasetManager.getDataset();
		dataset.begin(ReadWrite.READ);
		Model model = datasetManager.getModel(version);

		updateShapeModel();
		Model model = getModel();

		Resource reportResource = ValidationUtil.validateModel(model, shapeModel, true);
		boolean conforms = reportResource.getProperty(SH.conforms).getBoolean();
		if (!conforms) {

		Model reportModel = reportResource.getModel();

		StringWriter sw = new StringWriter();
		if (LOG.isTraceEnabled()) {
		StringWriter sw;
		sw = new StringWriter();
		shapeModel.write(sw, "TTL");
		LOG.trace("SHACL is " + sw.toString());

		sw = new StringWriter();
		reportModel.write(sw, "TTL");
		LOG.trace("Report model is " + sw.toString());
		}

		if (!conforms) {
		if (LOG.isTraceEnabled()) {
		try (QueryExecution exec = QueryExecutionFactory.create(SELECT_VIOLATION, reportModel);) {
		LOG.trace(ResultSetFormatter.asText(exec.execSelect()));
		@@ -149,7 +142,8 @@ public abstract class AbstractShaclChecker extends AbstractClauseChecker {
		try (QueryExecution exec = QueryExecutionFactory.create(SELECT_VIOLATION, reportModel);) {

		Literal previousResultMessage = null;
		Map<Resource, List<RDFNode>> valuesMap = new HashMap<>();
		Resource previousResultSeverity = null;
		Map<Resource, Set<RDFNode>> valuesMap = new HashMap<>();

		for (ResultSet resultSet = exec.execSelect(); resultSet.hasNext();) {
		QuerySolution sol = resultSet.next();
		@@ -159,12 +153,12 @@ public abstract class AbstractShaclChecker extends AbstractClauseChecker {
		Resource focusNode = sol.get("focusNode").asResource();
		RDFNode value = sol.get("value");
		if (previousResultMessage != null && !resultMessage.equals(previousResultMessage)) {
		report(severity, previousResultMessage, valuesMap);
		report(previousResultSeverity, previousResultMessage, valuesMap);
		valuesMap = new HashMap<>();
		}
		List<RDFNode> values = valuesMap.get(focusNode);
		Set<RDFNode> values = valuesMap.get(focusNode);
		if (values == null) {
		values = new ArrayList<RDFNode>();
		values = new HashSet<>();
		valuesMap.put(focusNode, values);
		}
		if (value != null && !value.equals(focusNode)) {
		@@ -174,6 +168,7 @@ public abstract class AbstractShaclChecker extends AbstractClauseChecker {
		report(severity, resultMessage, valuesMap);
		}
		previousResultMessage = resultMessage;
		previousResultSeverity = severity;
		}
		}
		}
		@@ -181,24 +176,26 @@ public abstract class AbstractShaclChecker extends AbstractClauseChecker {
		dataset.end();
		}

		private void report(Resource severity, Literal resultMessage, Map<Resource, List<RDFNode>> valuesMap) {
		private void report(Resource severity, Literal resultMessage, Map<Resource, Set<RDFNode>> valuesMap) {
		StringWriter sw = new StringWriter();
		for (Resource resource : valuesMap.keySet()) {
		sw.append("- ").append("`").append(resource.toString()).append("`");
		List<RDFNode> values = valuesMap.get(resource);
		sw.append("- ").append("`").append(FmtUtils.stringForRDFNode(resource)).append("`");
		Set<RDFNode> values = valuesMap.get(resource);
		if (!values.isEmpty()) {
		sw.append(" got: ");
		for (RDFNode value : values) {
		sw.append("`").append(value.toString()).append("`").append(" ; ");
		sw.append("`").append(FmtUtils.stringForRDFNode(value)).append("`").append(" ; ");
		}
		}
		sw.append("\n");
		}
		String message = String.format("%s\n\n%s", resultMessage.getString(), sw.toString());
		String message = String.format("%s\n\n%s\n\n", resultMessage.getString(), sw.toString());
		if (severity.equals(SHACL.Violation)) {
		errorLogger.error(message);
		} else {
		} else if (severity.equals(SHACL.Warning)) {
		errorLogger.warn(message);
		} else {
		errorLogger.info(message);
		}
		}

		@@ -208,26 +205,18 @@ public abstract class AbstractShaclChecker extends AbstractClauseChecker {
		}

		protected void add(MessageResource message, Object... args) {
		add(null, null, message, args);
		}

		protected void add(Severity severity, MessageResource message, Object... args) {
		add(severity, null, message, args);
		add(null, message, args);
		}

		protected void add(String pattern, MessageResource message, Object... args) {
		add(null, pattern, message, args);
		}

		protected void add(Severity severity, String pattern, MessageResource message, Object... args) {
		Resource r = message.asResource();
		if (severity != null) {
		shapeModel.add(r, SHACL.severity, severity.getResource());
		}
		if (pattern != null) {
		shapeModel.add(r, SHACL.pattern, pattern);
		}
		shapeModel.add(r, SHACL.message, getMessage(message, args));
		}

		protected String exactly(String string) {
		return String.format("^%s$", string);
		}
		}
		No newline at end of file

src/main/java/fr/emse/gitlab/saref/checkers/Clause_9_2_Checker.java

+13 −18

Original line number	Diff line number	Diff line
		@@ -39,7 +39,6 @@ import java.util.Set;
		import java.util.regex.Matcher;
		import java.util.regex.Pattern;

		import fr.emse.gitlab.saref.SAREFPipeline.Mode;
		import fr.emse.gitlab.saref.SAREFPipelineException;
		import fr.emse.gitlab.saref.managers.RepositoryManager;

		@@ -79,13 +78,13 @@ public class Clause_9_2_Checker extends AbstractClauseChecker {
		if (file.isFile()) {
		return;
		}
		log(getMessage(MESSAGE.readme_missing), Mode.RELEASE, Mode.PORTAL);
		logError(getMessage(MESSAGE.readme_missing));
		}

		private void checkGitignore() throws SAREFPipelineException {
		final File file = new File(repository.getDirectory(), ".gitignore");
		if (!file.isFile()) {
		log(getMessage(MESSAGE.gitignore_missing), Mode.RELEASE, Mode.PORTAL);
		logError(getMessage(MESSAGE.gitignore_missing));
		return;
		}
		try (BufferedReader br = Files.newBufferedReader(file.toPath())) {
		@@ -93,10 +92,10 @@ public class Clause_9_2_Checker extends AbstractClauseChecker {
		gitignoreLines.remove(line);
		}
		if (!gitignoreLines.isEmpty()) {
		log(getMessage(MESSAGE.gitignore_incomplete, gitignoreLines.toString()), Mode.RELEASE, Mode.PORTAL);
		logError(getMessage(MESSAGE.gitignore_incomplete, gitignoreLines.toString()));
		}
		} catch (IOException ex) {
		log(getMessage(MESSAGE.gitignore_ioexception, gitignoreLines.toString()), Mode.RELEASE, Mode.PORTAL);
		logError(getMessage(MESSAGE.gitignore_ioexception, ex));
		}
		}

		@@ -104,7 +103,7 @@ public class Clause_9_2_Checker extends AbstractClauseChecker {
		private void checkLICENSE() throws SAREFPipelineException {
		final File licenseFile = new File(repository.getDirectory(), "LICENSE");
		if (!licenseFile.exists()) {
		log(getMessage(MESSAGE.license_missing), Mode.RELEASE, Mode.PORTAL);
		logError(getMessage(MESSAGE.license_missing));
		return;
		}
		InputStream licenseModelFile = Clause_9_2_Checker.class.getClassLoader()
		@@ -119,13 +118,13 @@ public class Clause_9_2_Checker extends AbstractClauseChecker {
		Pattern pattern = Pattern.compile(String.format("^%s$", regex));
		Matcher regexMatcher = pattern.matcher(licenseLine);
		if (!regexMatcher.find()) {
		log(getMessage(MESSAGE.license_firstline, licenseLine, modelLine), Mode.RELEASE, Mode.PORTAL);
		logError(getMessage(MESSAGE.license_firstline, licenseLine, modelLine));
		return;
		}
		int year = Integer.parseInt(regexMatcher.group("year"));
		int currentYear = Calendar.getInstance().get(Calendar.YEAR);
		if (year < 2015 \|\| year > currentYear) {
		log(getMessage(MESSAGE.license_year, 2015, currentYear), Mode.RELEASE, Mode.PORTAL);
		logWarning(getMessage(MESSAGE.license_year, 2015, currentYear));
		return;
		}

		@@ -135,24 +134,22 @@ public class Clause_9_2_Checker extends AbstractClauseChecker {
		while (modelLine != null && licenseLine != null) {
		i++;
		if (!modelLine.equals(licenseLine)) {
		log(getMessage(MESSAGE.license_line, i, modelLine), Mode.RELEASE, Mode.PORTAL);
		logError(getMessage(MESSAGE.license_line, i, modelLine));
		return;
		}
		modelLine = licenseModelFileReader.readLine();
		licenseLine = licenseFileReader.readLine();
		}
		if (modelLine == null && licenseLine != null) {
		log(getMessage(MESSAGE.license_overcomplete, i, modelLine), Mode.RELEASE, Mode.PORTAL);
		logError(getMessage(MESSAGE.license_overcomplete, i, modelLine));
		return;
		}
		if (modelLine != null && licenseLine == null) {
		log(getMessage(MESSAGE.license_incomplete, i), Mode.RELEASE, Mode.PORTAL);
		logError(getMessage(MESSAGE.license_incomplete, i));
		return;
		}
		} catch (Exception ex) {
		String msg = getMessage(MESSAGE.license_ioexception);
		log(msg, ex, Mode.RELEASE, Mode.PORTAL);
		throw new SAREFPipelineException(msg, ex);
		logError(getMessage(MESSAGE.license_ioexception, ex));
		}
		}

		@@ -161,12 +158,10 @@ public class Clause_9_2_Checker extends AbstractClauseChecker {
		try {
		if (!dir.isDirectory()) {
		String msg = getMessage(message, dirName);
		log(msg, Mode.RELEASE, Mode.PORTAL);
		logError(msg);
		}
		} catch (SecurityException ex) {
		String msg = getMessage(MESSAGE.securityexception, dirName);
		log(msg, ex, Mode.RELEASE, Mode.PORTAL);
		throw new SAREFPipelineException(msg, ex);
		logError(getMessage(MESSAGE.securityexception, ex));
		}
		}