Commit 91403ce3 authored by Maxime Lefrançois's avatar Maxime Lefrançois
Browse files

removed dependency any23

parent b2e5c160

pom.xml

+2 −2
Original line number Diff line number Diff line
@@ -176,7 +176,7 @@ 
			<version>11</version>

		</dependency>



		<dependency>

		<!-- <dependency>

			<groupId>org.apache.any23</groupId>

			<artifactId>apache-any23-core</artifactId>

			<version>2.0</version>
@@ -186,7 +186,7 @@ 
			<groupId>org.openrdf.sesame</groupId>

			<artifactId>sesame-rio-api</artifactId>

			<version>4.0.2</version>

		</dependency>

		</dependency> -->



		<dependency>

			<groupId>com.squareup.okhttp3</groupId>

src/main/java/fr/emse/gitlab/saref/checkers/Clause_9_3_Checker.java

+60 −57
Original line number Diff line number Diff line
@@ -26,18 +26,28 @@ 
package fr.emse.gitlab.saref.checkers;



import java.io.File;

import java.io.FileOutputStream;

import java.io.FileReader;

import java.io.IOException;

import java.nio.file.Files;

import java.nio.file.Path;

import java.util.Optional;

import java.util.stream.Collectors;

import java.nio.charset.StandardCharsets;

import java.util.List;



import org.apache.commons.io.FileUtils;

import org.apache.jena.rdf.model.Model;

import org.apache.jena.rdf.model.Resource;

import org.apache.jena.vocabulary.DCTerms;

import org.apache.jena.vocabulary.RDF;



import com.opencsv.CSVParser;

import com.opencsv.CSVParserBuilder;

import com.opencsv.CSVReader;

import com.opencsv.CSVReaderBuilder;



import fr.emse.gitlab.saref.SAREF;

import fr.emse.gitlab.saref.SAREFPipelineException;

import fr.emse.gitlab.saref.managers.GenerateRDFaManager;

import fr.emse.gitlab.saref.managers.RepositoryManager;

import fr.emse.gitlab.saref.utils.Languages;

import fr.emse.gitlab.saref.vocabs.VTC;



/**

 * Checks TS 103 673 Clause 9.3: Ontology requirements
@@ -48,7 +58,7 @@ public class Clause_9_3_Checker extends AbstractClauseChecker { 
	private static final String FIRST_LINE = "Id;Category;Requirement";



	private static enum MESSAGE {

		directories, csv, missing, ioexception, line;

		missing, ioexception, line;

	}



	public Clause_9_3_Checker(RepositoryManager repositoryManager) {
@@ -62,66 +72,48 @@ public class Clause_9_3_Checker extends AbstractClauseChecker { 
			return;

		}

		try {

			checkExists(dir.toPath());

			checkFirstLine(dir.toPath());

			requirementsRDFaGenerator();

		} catch (IOException ex) {

			logError(getMessage(MESSAGE.ioexception), ex);

			File file = new File(dir, "requirements.csv");

			if (!file.exists()) {

				logError(getMessage(MESSAGE.missing));

				return;

			}

			List<String> lines = FileUtils.readLines(file, StandardCharsets.UTF_8);

			if (lines.size() < 1 || !lines.get(0).equals(FIRST_LINE)) {

				logError(getMessage(MESSAGE.line));

				return;

			}



	private void checkExists(Path path) throws IOException {

		String directories = Files.walk(path).filter(p -> {

			try {

				return p.toFile().isDirectory() && !Files.isSameFile(path, p);

			readRequirements();

			requirementsRDFaGenerator();

		} catch (IOException ex) {

				return false;

			}

		}).map(p -> p.toString()).collect(Collectors.joining(", "));

		if(directories.length()>0) {

			logWarning(getMessage(MESSAGE.directories, directories));

		}



		String nonCsv = Files.walk(path, 1).filter(p -> {

			try {

				return p.toFile().isFile() && !SAREF.CSV_MATCHER.matches(p) && !p.toFile().getName().startsWith(".");

			} catch (Exception ex) {

				return false;

			logError(getMessage(MESSAGE.ioexception), ex);

		}

		}).map(p -> p.getFileName().toString()).collect(Collectors.joining(", "));

		if(nonCsv.length()>0) {

			logError(getMessage(MESSAGE.csv, nonCsv));

	}



		boolean containsFile = Files.walk(path, 1).anyMatch(p -> {

			return SAREF.CSV_MATCHER.matches(p);

	public void readRequirements() throws SAREFPipelineException {

		Model requirements = version.getRequirements();

		File file = new File(repository.getDirectory(), "requirements/requirements.csv");

		CSVParser parser = new CSVParserBuilder().withSeparator(';').withQuoteChar('"').build();

		try (FileReader filereader = new FileReader(file);

				CSVReader csvReader = new CSVReaderBuilder(filereader).withCSVParser(parser).build()) {

			csvReader.forEach(row -> {

				String id = row[0]; // WATR-1

				Resource resource = requirements.getResource(String.format("%srequirements#%s", version.getIRI(), id));

				String category = row[1]; // Water infrastructure

				String requirement = row[2]; // Which assets compose a water distribution infrastructure? ...

				requirements.add(resource, RDF.type, VTC.Requirement);

				requirements.add(resource, VTC.requirementId, id);

				requirements.add(resource, VTC.category, category);

				requirements.add(resource, DCTerms.description, requirement);

			});

		if (!containsFile) {

			logWarning(getMessage(MESSAGE.missing));

		} catch (Exception e) {

			logWarning(getMessage(MESSAGE.ioexception), e);

		}

	}



	private void checkFirstLine(Path path) throws IOException {

		Files.walk(path).filter(p -> {

			return SAREF.CSV_MATCHER.matches(p);

		}).forEach(p -> {

			Optional<String> firstLine;

			try {

				firstLine = Files.lines(p).findFirst();

				if (!firstLine.isPresent()) {

					logError(getMessage(MESSAGE.line, p.getFileName()));

				} else {

					if (!firstLine.get().equals(FIRST_LINE)) {

						logError(getMessage(MESSAGE.line, p.getFileName()));

					}

				}

			} catch (IOException e) {

				logError(getMessage(MESSAGE.ioexception), e);

			}

		});

	}



	private void requirementsRDFaGenerator() throws SAREFPipelineException, IOException {

		if (pipeline.ignoreSite) {

			return;

		}

		String categoryChanger = "";

		String repoName = project.getName();

		String href = project.getNamespace();
@@ -130,6 +122,17 @@ public class Clause_9_3_Checker extends AbstractClauseChecker { 
		FileUtils.forceMkdir(versionSite);

		File reqHTML = new File(versionSite, "requirements.html");

		GenerateRDFaManager.GenerateRDFa(categoryChanger, repoName, href, reqCSV, reqHTML, "requirements");

		Model model = version.getRequirements();

		for (Languages l : Languages.values()) {

			String fileName = String.format("requirements.%s", l.getExt());

			File file = new File(versionSite, fileName);

			try (FileOutputStream fos = new FileOutputStream(file)) {

				model.write(fos, l.getLang());

			} catch (IOException ex) {

				String msg = getMessage(MESSAGE.ioexception, ex);

				errorLogger.warn(msg, ex);

			}

		}

	}



}

src/main/java/fr/emse/gitlab/saref/checkers/Clause_9_4_5_Checker.java

+19 −3
Original line number Diff line number Diff line
@@ -39,6 +39,10 @@ import org.semanticweb.owlapi.model.OWLAxiom; 
import org.semanticweb.owlapi.model.OWLOntology;

import org.semanticweb.owlapi.profiles.OWL2DLProfile;

import org.semanticweb.owlapi.profiles.OWLProfileReport;

import org.semanticweb.owlapi.profiles.violations.UseOfDefinedDatatypeInDatatypeRestriction;

import org.semanticweb.owlapi.profiles.violations.UseOfDefinedDatatypeInLiteral;

import org.semanticweb.owlapi.profiles.violations.UseOfUndeclaredDatatype;

import org.semanticweb.owlapi.profiles.violations.UseOfUnknownDatatype;



import fr.emse.gitlab.saref.SAREFPipelineException;

import fr.emse.gitlab.saref.managers.OntologyManager;
@@ -75,6 +79,15 @@ public class Clause_9_4_5_Checker extends AbstractClauseChecker { 
		}



		final OWLProfileReport report = new OWL2DLProfile().checkOntology(ontology);

		report.getViolations().removeIf(v->{

			if(v instanceof UseOfDefinedDatatypeInDatatypeRestriction

					|| v instanceof UseOfDefinedDatatypeInLiteral

					|| v instanceof UseOfUndeclaredDatatype

					|| v instanceof UseOfUnknownDatatype) {

				return true;

			}

			return false;

		});

		if (!report.getViolations().isEmpty()) {

			String violations = report.getViolations().stream().map(Object::toString)

					.collect(Collectors.joining("\n- ", "\n\n- ", "\n\n"));
@@ -133,6 +146,9 @@ public class Clause_9_4_5_Checker extends AbstractClauseChecker { 
					.addHeader("Content-Type", "application/xml").build();



			Response response = httpClient.newCall(request).execute();

			if (response.code() != 200) {

				throw new SAREFPipelineException("response", "Unexpected response code " + response.code());

			}

			

			String result = response.body().string();
@@ -148,8 +164,8 @@ public class Clause_9_4_5_Checker extends AbstractClauseChecker { 


			arrayData = pureData.toArray(); // response into array



		} catch (IOException e) {

			logWarning(getMessage(MESSAGE.oopsError));

		} catch (IOException | SAREFPipelineException e) {

			logWarning(getMessage(MESSAGE.oopsError), e);

		}



		int count = OopsManager.counter(arrayData);

src/main/java/fr/emse/gitlab/saref/checkers/Clause_9_5_Checker.java

+98 −134
Original line number Diff line number Diff line
@@ -25,37 +25,37 @@ 
 */

package fr.emse.gitlab.saref.checkers;



import java.io.ByteArrayOutputStream;

import java.io.File;

import java.io.FileOutputStream;

import java.io.FileReader;

import java.io.IOException;

import java.io.StringWriter;

import java.nio.charset.StandardCharsets;

import java.nio.file.FileSystems;

import java.nio.file.Files;

import java.nio.file.Path;

import java.nio.file.PathMatcher;

import java.util.ArrayList;

import java.util.Optional;

import java.util.List;

import java.util.stream.Collectors;



import org.apache.any23.Any23;

import org.apache.any23.extractor.ExtractionException;

import org.apache.any23.source.DocumentSource;

import org.apache.any23.source.FileDocumentSource;

import org.apache.any23.writer.RDFXMLWriter;

import org.apache.any23.writer.TripleHandler;

import org.apache.any23.writer.TripleHandlerException;

import org.apache.commons.io.FileUtils;

import org.apache.jena.rdf.model.Model;

import org.apache.jena.rdf.model.Resource;

import org.apache.jena.vocabulary.RDF;

import org.w3c.dom.Document;

import org.w3c.dom.Element;

import org.w3c.dom.NodeList;



import com.opencsv.CSVParser;

import com.opencsv.CSVParserBuilder;

import com.opencsv.CSVReader;

import com.opencsv.CSVReaderBuilder;



import fr.emse.gitlab.saref.SAREFPipelineException;

import fr.emse.gitlab.saref.entities.SAREFRepository;

import fr.emse.gitlab.saref.entities.SAREFVersion;

import fr.emse.gitlab.saref.managers.DatasetManager;

import fr.emse.gitlab.saref.managers.GenerateRDFaManager;

import fr.emse.gitlab.saref.managers.RepositoryManager;

import fr.emse.gitlab.saref.managers.ThemisManager;

import fr.emse.gitlab.saref.utils.Languages;

import fr.emse.gitlab.saref.vocabs.VTC;

import okhttp3.MediaType;

import okhttp3.OkHttpClient;

import okhttp3.Request;
@@ -72,7 +72,7 @@ public class Clause_9_5_Checker extends AbstractClauseChecker { 
	private static final String FIRST_LINE = "Id;Requirement;Category;Test";



	private static enum MESSAGE {

		directories, csv, missing, ioexception, line, themis, themisError;

		missing, ioexception, line, themis, themisError;

	}



	/**
@@ -90,145 +90,97 @@ public class Clause_9_5_Checker extends AbstractClauseChecker { 
			return;

		}

		try {

			checkExists(dir.toPath());

			checkFirstLine(dir.toPath());

			testsRDFaGenerator();

			checkThemis();

		} catch (IOException ex) {

			logError(getMessage(MESSAGE.ioexception));

			File file = new File(dir, "tests.csv");

			if (!file.exists()) {

				logError(getMessage(MESSAGE.missing));

				return;

			}

			List<String> lines = FileUtils.readLines(file, StandardCharsets.UTF_8);

			if (lines.size() < 1 || !lines.get(0).equals(FIRST_LINE)) {

				logError(getMessage(MESSAGE.line));

				return;

			}



	private void checkExists(Path path) throws IOException {

		String directories = Files.walk(path).filter(p -> {

			try {

				return p.toFile().isDirectory() && !Files.isSameFile(path, p);

			readTests();

			callThemis();

			testsRDFaGenerator();

		} catch (IOException ex) {

				return false;

			}

		}).map(p -> p.toString()).collect(Collectors.joining(", "));

		if (directories.length() > 0) {

			logWarning(getMessage(MESSAGE.directories, directories));

		}



		String nonCsv = Files.walk(path, 1).filter(p -> {

			try {

				return p.toFile().isFile() && !csvMatcher.matches(p) && !p.toFile().getName().startsWith(".");

			} catch (Exception ex) {

				return false;

			}

		}).map(p -> p.getFileName().toString()).collect(Collectors.joining(", "));

		if (nonCsv.length() > 0) {

			logError(getMessage(MESSAGE.csv, nonCsv));

		}



		boolean containsFile = Files.walk(path, 1).anyMatch(p -> {

			return csvMatcher.matches(p);

		});

		if (!containsFile) {

			logWarning(getMessage(MESSAGE.missing));

		}

	}



	private void checkFirstLine(Path path) throws IOException {

		Files.walk(path).filter(p -> {

			return csvMatcher.matches(p);

		}).forEach(p -> {

			Optional<String> firstLine;

			try {

				firstLine = Files.lines(p).findFirst();

				if (!firstLine.isPresent()) {

					logError(getMessage(MESSAGE.line, p.getFileName()));

				} else {

					if (!firstLine.get().equals(FIRST_LINE)) {

						logError(getMessage(MESSAGE.line, p.getFileName()));

					}

				}

			} catch (IOException e) {

			logError(getMessage(MESSAGE.ioexception));

		}

		});

	}



	public void checkThemis() throws SAREFPipelineException {

		File versionSite = new File(siteManager.siteDir, version.getVersionPath());

		File tHTML = new File(versionSite, "tests.html");

		DocumentSource source = new FileDocumentSource(tHTML);

		Any23 runner = new Any23();

		try(ByteArrayOutputStream out = new ByteArrayOutputStream(); TripleHandler handler = new RDFXMLWriter(out);) {

			runner.extract(source, handler);



			String xmlData = out.toString();



			ArrayList<String[]> res = makeCall(xmlData, repository, datasetManager, version);



			ArrayList<String> result = new ArrayList<String>();



			for (int i = 0; i < res.size(); i++) {

				String response = res.get(i)[0] + ", " + res.get(i)[1] + ", " + res.get(i)[2];

				result.add(response);

			}



			if (!res.isEmpty()) {

				String data = result.stream().map(e -> e.toString()).collect(Collectors.joining("\n- ", "\n\n- ", "\n\n"));

				logWarning(getMessage(MESSAGE.themis, data));

	}



		} catch (ExtractionException | IOException | TripleHandlerException e) {

			logWarning(getMessage(MESSAGE.themisError));

	public void readTests() throws SAREFPipelineException {

		Model requirements = version.getRequirements();

		File file = new File(repository.getDirectory(), "tests/tests.csv");

		CSVParser parser = new CSVParserBuilder().withSeparator(';').withQuoteChar('"').build();

		try (FileReader filereader = new FileReader(file);

				CSVReader csvReader = new CSVReaderBuilder(filereader).withCSVParser(parser).build()) {

			csvReader.forEach(row -> {

				String id = row[0]; // WATR-TEST-17

				Resource testResource = requirements.getResource(String.format("%stests#%s", version.getIRI(), id));

				String requirementId = row[1]; // WATR-9

				Resource requirementResource = requirements

						.getResource(String.format("%srequirements#", version.getIRI(), requirementId));

				String category = row[2]; // Water meter

				String test = row[3]; // WaterMeter hasManufacturer Literal

				requirements.add(testResource, RDF.type, VTC.TestCaseDesign);

				requirements.add(testResource, VTC.testId, id);

				requirements.add(testResource, VTC.comesFromRequirement, requirementResource);

				requirements.add(testResource, VTC.desiredBehaviour, test);

			});

		} catch (Exception e) {

			logWarning(getMessage(MESSAGE.ioexception), e);

		}

	}



	public ArrayList<String[]> makeCall(String xmlData, SAREFRepository repository, DatasetManager datasetManager,

			SAREFVersion version) throws SAREFPipelineException {

	public void callThemis() throws SAREFPipelineException {



		ArrayList<String[]> res = new ArrayList<String[]>();

		try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {

			String onto = "";

		StringWriter versionRDFXML = new StringWriter();

		version.getModel().write(versionRDFXML);



			Model model = version.getModel();

			model.write(out, "RDF/XML");

			onto = out.toString();

		StringWriter requirementsRDFXML = new StringWriter();

		version.getRequirements().write(requirementsRDFXML);



		OkHttpClient httpClient = new OkHttpClient().newBuilder().build();



		MediaType mediaType = MediaType.parse("application/json");



		String jsonRequest = "{\"ontologiesCode\":[\""

					+ onto.replaceAll("\\\"", "\\\\\"").replaceAll("\"", "\\\"").replaceAll("\t", " ") + "\"],"

					+ "\"testfile\":[\"" + xmlData.replaceAll("\\\"", "\\\\\"").replaceAll("\"", "\\\"").replaceAll("\t", " ")

				+ versionRDFXML.toString().replaceAll("\\\"", "\\\\\"").replaceAll("\"", "\\\"").replaceAll("\t", " ")

				+ "\"]," + "\"testfile\":[\"" + requirementsRDFXML.toString().replaceAll("\\\"", "\\\\\"")

						.replaceAll("\"", "\\\"").replaceAll("\t", " ")

				+ "\"]," + "\"format\":\"junit\"}";



		RequestBody body = RequestBody.create(jsonRequest, mediaType);



			Request request = new Request.Builder().url("http://themis.linkeddata.es/rest/api/results")

					.method("POST", body).addHeader("accept", "application/json")

					.addHeader("Content-Type", "application/json").build();



		Request request = new Request.Builder().url("http://themis.linkeddata.es/rest/api/results").method("POST", body)

				.addHeader("accept", "application/json").addHeader("Content-Type", "application/json").build();

		try {

			Response response = httpClient.newCall(request).execute();



			String result = response.body().string();



			Document doc = ThemisManager.convertStringToXMLDocument(result);



			if (response.code() != 200) {

				throw new SAREFPipelineException("response", "Unexpected response code " + response.code());

			}

			String responseBody = response.body().string();

			Document doc = ThemisManager.convertStringToXMLDocument(responseBody);

			NodeList nodeList = doc.getElementsByTagName("testcase");



			ArrayList<String> themisErrors = new ArrayList<String>();

			for (int temp = 0; temp < nodeList.getLength(); temp++) {

				org.w3c.dom.Node node = nodeList.item(temp);

				String[] part = new String[3];

				if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {

					Element element = (Element) node;

					part[0] = "" + element.getAttributeNode("id");

					part[1] = "" + element.getAttributeNode("name");

					part[2] = "" + ((Element) node).getElementsByTagName("error").item(0).getAttributes()

							.getNamedItem("message");



					res.add(part);

					String id = element.getAttributeNode("id").toString();

					String name = element.getAttributeNode("name").toString();

					String message = ((Element) node).getElementsByTagName("error").item(0).getAttributes()

							.getNamedItem("message").toString();

					String themisError = String.format("%s, %s, %s", id, name, message);

					themisErrors.add(themisError);

				}

			}

		} catch (IOException e) {

			logWarning(getMessage(MESSAGE.themisError));

			if (!themisErrors.isEmpty()) {

				String data = themisErrors.stream().map(e -> e.toString())

						.collect(Collectors.joining("\n- ", "\n\n- ", "\n\n"));

				logWarning(getMessage(MESSAGE.themis, data));

			}

		return res;

		} catch (IOException | SAREFPipelineException e) {

			logWarning(getMessage(MESSAGE.themisError), e);

		}



	}



	private void testsRDFaGenerator() throws SAREFPipelineException, IOException {
@@ -243,6 +195,18 @@ public class Clause_9_5_Checker extends AbstractClauseChecker { 
		FileUtils.forceMkdir(versionSite);

		File testHTML = new File(versionSite, "tests.html");

		GenerateRDFaManager.GenerateRDFa(categoryChanger, repoName, href, testCSV, testHTML, "tests");

		Model model = version.getRequirements();

		for (Languages l : Languages.values()) {

			String fileName = String.format("tests.%s", l.getExt());

			File file = new File(versionSite, fileName);

			try (FileOutputStream fos = new FileOutputStream(file)) {

				model.write(fos, l.getLang());

			} catch (IOException ex) {

				String msg = getMessage(MESSAGE.ioexception, ex);

				errorLogger.warn(msg, ex);

			}

		}



	}



}

src/main/java/fr/emse/gitlab/saref/checkers/Clause_9_6_4_Checker.java

+14 −0
Original line number Diff line number Diff line
@@ -35,6 +35,10 @@ import org.semanticweb.owlapi.model.OWLOntology; 
import org.semanticweb.owlapi.model.parameters.Imports;

import org.semanticweb.owlapi.profiles.OWL2DLProfile;

import org.semanticweb.owlapi.profiles.OWLProfileReport;

import org.semanticweb.owlapi.profiles.violations.UseOfDefinedDatatypeInDatatypeRestriction;

import org.semanticweb.owlapi.profiles.violations.UseOfDefinedDatatypeInLiteral;

import org.semanticweb.owlapi.profiles.violations.UseOfUndeclaredDatatype;

import org.semanticweb.owlapi.profiles.violations.UseOfUnknownDatatype;



import fr.emse.gitlab.saref.SAREFPipelineException;

import fr.emse.gitlab.saref.entities.SAREFExample;
@@ -67,6 +71,16 @@ public class Clause_9_6_4_Checker extends AbstractClauseChecker { 
		}



		final OWLProfileReport report = new OWL2DLProfile().checkOntology(ontology);

		report.getViolations().removeIf(v->{

			if(v instanceof UseOfDefinedDatatypeInDatatypeRestriction

					|| v instanceof UseOfDefinedDatatypeInLiteral

					|| v instanceof UseOfUndeclaredDatatype

					|| v instanceof UseOfUnknownDatatype) {

				return true;

			}

			return false;

		});

		

		if(!report.getViolations().isEmpty()) {

			String violations = report.getViolations().stream().map(Object::toString)

					.collect(Collectors.joining("\n- ", "\n\n- ", "\n\n"));