Added Themis, OOPS and RDFaGenerator Services to clauses 9.3, 9.4.5, 9.5 to the new code (012f43d1) · Commits · SAREF / saref-pipeline

pom.xml

+24 −0

Original line number	Original line	Diff line number	Diff line
	@@ -176,6 +176,30 @@
	<version>11</version>		<version>11</version>
	</dependency>		</dependency>

			<dependency>
			<groupId>org.apache.any23</groupId>
			<artifactId>apache-any23-core</artifactId>
			<version>2.0</version>
			</dependency>

			<dependency>
			<groupId>org.openrdf.sesame</groupId>
			<artifactId>sesame-rio-api</artifactId>
			<version>4.0.2</version>
			</dependency>

			<dependency>
			<groupId>com.squareup.okhttp3</groupId>
			<artifactId>okhttp</artifactId>
			<version>4.2.2</version>
			</dependency>

			<dependency>
			<groupId>com.opencsv</groupId>
			<artifactId>opencsv</artifactId>
			<version>4.1</version>
			</dependency>

	</dependencies>		</dependencies>

	<build>		<build>

src/main/java/fr/emse/gitlab/saref/checkers/Clause_9_3_Checker.java

+18 −0

Original line number	Original line	Diff line number	Diff line
	@@ -34,6 +34,7 @@ import java.util.stream.Collectors;

	import fr.emse.gitlab.saref.SAREF;		import fr.emse.gitlab.saref.SAREF;
	import fr.emse.gitlab.saref.SAREFPipelineException;		import fr.emse.gitlab.saref.SAREFPipelineException;
			import fr.emse.gitlab.saref.managers.GenerateRDFaManager;
	import fr.emse.gitlab.saref.managers.RepositoryManager;		import fr.emse.gitlab.saref.managers.RepositoryManager;

	/**		/**
	@@ -61,6 +62,7 @@ public class Clause_9_3_Checker extends AbstractClauseChecker {
	try {		try {
	checkExists(dir.toPath());		checkExists(dir.toPath());
	checkFirstLine(dir.toPath());		checkFirstLine(dir.toPath());
			requirementsRDFaGenerator();
	} catch (IOException ex) {		} catch (IOException ex) {
	logError(getMessage(MESSAGE.ioexception), ex);		logError(getMessage(MESSAGE.ioexception), ex);
	}		}
	@@ -117,4 +119,20 @@ public class Clause_9_3_Checker extends AbstractClauseChecker {
	});		});
	}		}

			private void requirementsRDFaGenerator() throws SAREFPipelineException{
			String categoryChanger = "";
			String repoName = project.getName();
			String href = project.getNamespace();


			File reqCSV = new File(repository.getDirectory(),"/requirements/requirements.csv");

			File reqHTML = new File(repository.getDirectory(), "/target/site/requirements.html");

			GenerateRDFaManager manager = new GenerateRDFaManager();

			manager.GenerateRDFaManager(categoryChanger, repoName, href, reqCSV, reqHTML, "requirements");

			}

	}		}

src/main/java/fr/emse/gitlab/saref/checkers/Clause_9_4_5_Checker.java

+95 −1

Original line number	Original line	Diff line number	Diff line
	@@ -25,9 +25,16 @@
	*/		*/
	package fr.emse.gitlab.saref.checkers;		package fr.emse.gitlab.saref.checkers;

			import java.io.ByteArrayOutputStream;
			import java.io.IOException;
			import java.util.ArrayList;
			import java.util.List;
	import java.util.Set;		import java.util.Set;
	import java.util.stream.Collectors;		import java.util.stream.Collectors;

			import fr.emse.gitlab.saref.SAREFPipeline;
			import fr.emse.gitlab.saref.managers.OopsManager;
			import org.apache.jena.rdf.model.Model;
	import org.semanticweb.owl.explanation.api.Explanation;		import org.semanticweb.owl.explanation.api.Explanation;
	import org.semanticweb.owl.explanation.api.ExplanationGenerator;		import org.semanticweb.owl.explanation.api.ExplanationGenerator;
	import org.semanticweb.owlapi.model.OWLAxiom;		import org.semanticweb.owlapi.model.OWLAxiom;
	@@ -39,6 +46,8 @@ import fr.emse.gitlab.saref.SAREFPipelineException;
	import fr.emse.gitlab.saref.managers.OntologyManager;		import fr.emse.gitlab.saref.managers.OntologyManager;
	import fr.emse.gitlab.saref.managers.RepositoryManager;		import fr.emse.gitlab.saref.managers.RepositoryManager;

			import okhttp3.*;

	/**		/**
	* Checks TS 103 673 Clause 9.4.5: OWL Profile, Consistency, and Satisfiability		* Checks TS 103 673 Clause 9.4.5: OWL Profile, Consistency, and Satisfiability
	* of Classes		* of Classes
	@@ -47,7 +56,7 @@ import fr.emse.gitlab.saref.managers.RepositoryManager;
	public class Clause_9_4_5_Checker extends AbstractClauseChecker {		public class Clause_9_4_5_Checker extends AbstractClauseChecker {

	private static enum MESSAGE {		private static enum MESSAGE {
	profile, consistent, satisfiable;		profile, consistent, satisfiable, oops, oopsError;
	}		}

	/**		/**
	@@ -92,6 +101,91 @@ public class Clause_9_4_5_Checker extends AbstractClauseChecker {
	}		}
	});		});
	}		}
			checkOops();
			}

			public void checkOops() throws SAREFPipelineException{

			ByteArrayOutputStream out = new ByteArrayOutputStream();
			String onto = "";

			Model model = version.getModel();
			model.write(out,"RDF/XML"); // write in string and then replace
			onto = out.toString().replace("]]","] ]");

			String xmlRequest = "<?xml version='1.0' encoding='UTF-8'?>" +
			"<OOPSRequest>" +
			"<OntologyURI></OntologyURI>" +
			"<OntologyContent><![CDATA[" + onto + "]]></OntologyContent>" +
			"<Pitfalls>10</Pitfalls>" +
			"<OutputFormat>RDF/XML</OutputFormat>" +
			"</OOPSRequest>";


			OopsManager manager = new OopsManager();

			ArrayList<ArrayList<String>> res = new ArrayList<ArrayList<String>>();

			Object[] arrayData = new Object[0];

			try{

			OkHttpClient httpClient = new OkHttpClient().newBuilder().build();

			MediaType mediaType = MediaType.parse("application/xml");

			RequestBody body = RequestBody.create(xmlRequest, mediaType);

			Request request = new Request.Builder()
			.url("http://oops.linkeddata.es/rest")
			.method("POST", body)
			.addHeader("Content-Type", "application/xml")
			.build();

			Response response = httpClient.newCall(request).execute();


			String result = response.body().string();


			String[] results = result.split("\\n");
			List<String> pureData = new ArrayList<String>();
			for(int i=0; i<results.length-1;i++){
			if(results[i].startsWith("<")) {
			for(int j=i; j<results.length;j++){
			pureData.add(results[j]);
			}
			}
			}


			arrayData = pureData.toArray(); // response into array

			}
			catch (IOException e) {
			log(getMessage(MESSAGE.oopsError), SAREFPipeline.Mode.RELEASE, SAREFPipeline.Mode.PORTAL);
	}		}


			int count = manager.counter(arrayData);

			ArrayList<ArrayList<String>> result = manager.getData(arrayData);

			int s = 0;

			ArrayList<String> results = new ArrayList<String>();

			while(s < count) {
			String data = result.get(0).get(s)+result.get(1).get(s)+result.get(2).get(s);
			results.add(data);
			s++;
			}

			String printData = results.stream().map(e -> e.toString()).collect(Collectors.joining("\n- ", "\n\n- ", "\n\n"));
			if(count!=0) {
			log(getMessage(MESSAGE.oops, printData), SAREFPipeline.Mode.RELEASE, SAREFPipeline.Mode.PORTAL);
			}
			}


	}		}

src/main/java/fr/emse/gitlab/saref/checkers/Clause_9_5_Checker.java

+148 −7

Original line number	Original line	Diff line number	Diff line
	@@ -25,17 +25,32 @@
	*/		*/
	package fr.emse.gitlab.saref.checkers;		package fr.emse.gitlab.saref.checkers;

	import java.io.File;		import java.io.*;
	import java.io.IOException;		import java.nio.file.*;
	import java.nio.file.FileSystems;		import java.util.ArrayList;
	import java.nio.file.Files;
	import java.nio.file.Path;
	import java.nio.file.PathMatcher;
	import java.util.Optional;		import java.util.Optional;
	import java.util.stream.Collectors;		import java.util.stream.Collectors;

			import fr.emse.gitlab.saref.SAREFPipeline;
	import fr.emse.gitlab.saref.SAREFPipelineException;		import fr.emse.gitlab.saref.SAREFPipelineException;
			import fr.emse.gitlab.saref.entities.SAREFRepository;
			import fr.emse.gitlab.saref.entities.SAREFVersion;
			import fr.emse.gitlab.saref.managers.DatasetManager;
			import fr.emse.gitlab.saref.managers.GenerateRDFaManager;
	import fr.emse.gitlab.saref.managers.RepositoryManager;		import fr.emse.gitlab.saref.managers.RepositoryManager;
			import fr.emse.gitlab.saref.managers.ThemisManager;
			import okhttp3.*;
			import org.apache.any23.Any23;
			import org.apache.any23.extractor.ExtractionException;
			import org.apache.any23.source.DocumentSource;
			import org.apache.any23.source.FileDocumentSource;
			import org.apache.any23.writer.RDFXMLWriter;
			import org.apache.any23.writer.TripleHandler;
			import org.apache.any23.writer.TripleHandlerException;
			import org.apache.jena.rdf.model.Model;
			import org.w3c.dom.Document;
			import org.w3c.dom.Element;
			import org.w3c.dom.NodeList;

	/**		/**
	* Checks TS 103 673 Clause 9.5: Ontology tests		* Checks TS 103 673 Clause 9.5: Ontology tests
	@@ -47,7 +62,7 @@ public class Clause_9_5_Checker extends AbstractClauseChecker {
	private static final String FIRST_LINE = "Id;Requirement;Category;Test";		private static final String FIRST_LINE = "Id;Requirement;Category;Test";

	private static enum MESSAGE {		private static enum MESSAGE {
	directories, csv, missing, ioexception, line;		directories, csv, missing, ioexception, line, themis, themisError;
	}		}

	/**		/**
	@@ -67,6 +82,8 @@ public class Clause_9_5_Checker extends AbstractClauseChecker {
	try {		try {
	checkExists(dir.toPath());		checkExists(dir.toPath());
	checkFirstLine(dir.toPath());		checkFirstLine(dir.toPath());
			testsRDFaGenerator();
			checkThemis();
	} catch (IOException ex) {		} catch (IOException ex) {
	logError(getMessage(MESSAGE.ioexception));		logError(getMessage(MESSAGE.ioexception));
	}		}
	@@ -123,4 +140,128 @@ public class Clause_9_5_Checker extends AbstractClauseChecker {
	});		});
	}		}


			public void checkThemis() throws SAREFPipelineException {
			try {

			File tHTML = new File(repository.getDirectory(),"target/site/tests.html");

			Any23 runner = new Any23();

			ByteArrayOutputStream out = new ByteArrayOutputStream();

			DocumentSource source = new FileDocumentSource(tHTML);
			TripleHandler handler = new RDFXMLWriter(out);
			runner.extract(source, handler);

			handler.close();

			String xmlData = out.toString();


			ArrayList<String []> res = makeCall(xmlData,repository,datasetManager,version);

			ArrayList<String> result = new ArrayList<String>();

			for(int i = 0; i < res.size(); i++){
			String response = res.get(i)[0]+", "+res.get(i)[1]+", "+res.get(i)[2];
			result.add(response);
			}

			String data = result.stream().map(e -> e.toString()).collect(Collectors.joining("\n- ", "\n\n- ", "\n\n"));

			if(!res.isEmpty()) {
			log(getMessage(MESSAGE.themis, data), SAREFPipeline.Mode.RELEASE, SAREFPipeline.Mode.RELEASE, SAREFPipeline.Mode.PORTAL);
			}

			} catch (FileNotFoundException e) {
			e.printStackTrace();
			} catch (ExtractionException \| UnsupportedEncodingException e) {
			e.printStackTrace();
			} catch (IOException e) {
			log(getMessage(MESSAGE.themisError), SAREFPipeline.Mode.RELEASE, SAREFPipeline.Mode.RELEASE, SAREFPipeline.Mode.PORTAL);
			} catch (TripleHandlerException e) {
			e.printStackTrace();
	}		}
			}


			public ArrayList<String []> makeCall(String xmlData, SAREFRepository repository, DatasetManager datasetManager, SAREFVersion version) throws SAREFPipelineException {

			ArrayList<String []> res = new ArrayList<String []>();
			try{
			ByteArrayOutputStream out = new ByteArrayOutputStream();
			String onto = "";

			Model model = version.getModel();
			model.write(out,"RDF/XML");
			onto = out.toString();

			OkHttpClient httpClient = new OkHttpClient().newBuilder().build();

			MediaType mediaType = MediaType.parse("application/json");

			String jsonRequest = "{\"ontologiesCode\":[\"" +
			onto.replace("\\\"", "\\\\\"").replace("\"", "\\\"").replace("\t"," ") +
			"\"]," + "\"testfile\":[\"" +
			xmlData.replace("\\\"", "\\\\\"").replace("\"", "\\\"").replace("\t"," ") +
			"\"]," + "\"format\":\"junit\"}";


			RequestBody body = RequestBody.create(jsonRequest, mediaType);

			Request request = new Request.Builder()
			.url("http://themis.linkeddata.es/rest/api/results")
			.method("POST", body)
			.addHeader("accept", "application/json")
			.addHeader("Content-Type", "application/json")
			.build();

			Response response = httpClient.newCall(request).execute();


			String result = response.body().string();

			ThemisManager manager = new ThemisManager();

			Document doc = manager.convertStringToXMLDocument(result);

			NodeList nodeList = doc.getElementsByTagName("testcase");
			for (int temp = 0; temp < nodeList.getLength(); temp++) {
			org.w3c.dom.Node node = nodeList.item(temp);
			String [] part = new String[3];
			if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
			Element element = (Element) node;
			part[0] = ""+element.getAttributeNode("id");
			part[1] = ""+element.getAttributeNode("name");
			part[2] = ""+((Element) node).getElementsByTagName("error").item(0).getAttributes().getNamedItem("message");

			res.add(part);

			}
			}

			} catch (IOException e) {
			log(getMessage(MESSAGE.themisError), SAREFPipeline.Mode.RELEASE, SAREFPipeline.Mode.RELEASE, SAREFPipeline.Mode.PORTAL);
			}
			return res;
			}

			private void testsRDFaGenerator() throws SAREFPipelineException{
			String categoryChanger = "";
			String repoName = project.getName();
			String href = project.getNamespace();

			File testCSV = new File(repository.getDirectory(),"/tests/tests.csv");

			File testHTML = new File(repository.getDirectory(), "/target/site/tests.html");

			GenerateRDFaManager manager = new GenerateRDFaManager();

			manager.GenerateRDFaManager(categoryChanger, repoName, href, testCSV, testHTML, "tests");

			}


			}

src/main/java/fr/emse/gitlab/saref/managers/GenerateRDFaManager.java

0 → 100644

+220 −0

Original line number	Original line	Diff line number	Diff line
			package fr.emse.gitlab.saref.managers;

			import com.opencsv.CSVParser;
			import com.opencsv.CSVParserBuilder;
			import com.opencsv.CSVReader;
			import com.opencsv.CSVReaderBuilder;
			import fr.emse.gitlab.saref.Main;
			import org.apache.commons.io.IOUtils;
			import org.eclipse.jgit.util.StringUtils;

			import java.io.*;
			import java.nio.charset.StandardCharsets;
			import java.util.List;

			public class GenerateRDFaManager {

			public static void GenerateRDFaManager(String categoryChanger, String repoName, String href, File CSV, File HTML, String identifier) {
			String row = "";

			FileWriter htmlWriter = null;

			try{

			htmlWriter = new FileWriter(HTML);

			String header = IOUtils.toString(Main.class.getClassLoader().getResourceAsStream("themis/header.html"), StandardCharsets.UTF_8);
			String body = IOUtils.toString(Main.class.getClassLoader().getResourceAsStream("themis/body.html"), StandardCharsets.UTF_8);
			String footer = IOUtils.toString(Main.class.getClassLoader().getResourceAsStream("themis/footer.html"), StandardCharsets.UTF_8);

			header = header.replace("@ontologyName", repoName).replace("@href", href + "/" + identifier + "/");

			body = body.replace("@docType", StringUtils.capitalize(identifier)).replace("@ontologyName", repoName);

			htmlWriter.write(header);
			htmlWriter.write(body);


			BufferedReader csvReader = new BufferedReader(new FileReader(CSV));

			List<String[]> allData = readDataFromCustomSeperator(CSV);


			if (identifier.equals("requirements")) {

			for (String[] data : allData) {
			if (data[1].equals("Category")) {
			continue;
			} else {
			if (categoryChanger.equals("") \|\| categoryChanger.equals(data[1]) \|\| !data[1].equals("Category")) {

			if (!categoryChanger.equals(data[1]) && !categoryChanger.equals("Category")) {
			if (categoryChanger.equals("")) {
			categoryChanger = data[1];
			} else {
			categoryChanger = data[1];
			String endTableData = "</table> \n" +
			"\n" +
			"<p>\n" +
			"</p>\n" +
			"\n";
			htmlWriter.write(endTableData);
			}


			categoryChanger = data[1];
			String entrytoTable2 = "<table align=\"center\" border=\"1\" prefix=\"vtc: https://w3id.org/def/vtc#\">\n";
			htmlWriter.write(entrytoTable2);
			String category = "<caption>Requirements for the \"" + categoryChanger + "\" category</caption>\n";
			htmlWriter.write(category);

			String entry = "<tr>\n" +
			" <th>Id</th>\n" +
			" <th>Requirement</th>\n" +
			" </tr>\n" +
			"\n" +
			"<tr id=\"" + data[0] + "\" resource=\"requirement/" + data[0] + "\" typeof=\"vtc:Requirement\" property=\"vtc:category\" content=\"" + data[1] + "\">\n" +
			" <td property=\"vtc:requirementId\">" + data[0] + "</td>\n" +
			" <td property=\"dc:description\">" + data[2] + "</td>\n" +
			" </tr>\n" +
			"\n";
			htmlWriter.write(entry);
			} else {
			categoryChanger = data[1];

			String entry = "<tr id=\"" + data[0] + "\" resource=\"requirement/" + data[0] + "\" typeof=\"vtc:Requirement\" property=\"vtc:category\" content=\"" + data[1] + "\">\n" +
			" <td property=\"vtc:requirementId\">" + data[0] + "</td>\n" +
			" <td property=\"dc:description\">" + data[2] + "</td>\n" +
			" </tr>\n" +
			"\n";
			htmlWriter.write(entry);
			}
			}

			}
			}

			} else {
			for (String[] data : allData) {
			if (data[2].equals("Category")) {
			continue;
			} else {
			if (categoryChanger.equals("") \|\| categoryChanger.equals(data[2]) \|\| !data[2].equals("Category")) {

			if (!categoryChanger.equals(data[2]) && !categoryChanger.equals("Category")) {
			if (categoryChanger.equals("")) {
			categoryChanger = data[2];
			} else {
			categoryChanger = data[2];
			String endTableData2 = "</table> \n" +
			"\n" +
			"<p>\n" +
			"</p>\n" +
			"\n";
			htmlWriter.write(endTableData2);
			}


			categoryChanger = data[2];
			String entrytoTable2 = "<table align=\"center\" border=\"1\" prefix=\"vtc: https://w3id.org/def/vtc#\">\n";
			htmlWriter.write(entrytoTable2);
			String category = "<caption>Tests for the \"" + categoryChanger + "\" category</caption>\n";
			htmlWriter.write(category);
			if (!data[1].isEmpty()) {
			String entry = "<tr>\n" +
			" <th>Id</th>\n" +
			" <th>Requirement</th>\n" +
			" <th>Test</th>\n" +
			" </tr>\n" +
			"\n" +
			"<tr id=\"" + data[1] + "\" resource=\"testCaseDesign/" + data[0] + "\" typeof=\"vtc:TestCaseDesign\">\n" +
			" <td property=\"vtc:testId\">" + data[0] + "</td>\n" +
			" <td><span property=\"vtc:comesFromRequirement\" href=\"requirement/" + data[1] + "\"><a href=\"../site/requirements.html#" + data[1] + "\">" + data[1] + "</a></span></td>\n" +
			" <td property=\"vtc:desiredBehaviour\">" + data[3] + "</td>\n" +
			" </tr>\n" +
			"\n";
			htmlWriter.write(entry);
			} else {
			String entry = "<tr>\n" +
			" <th>Id</th>\n" +
			" <th>Test</th>\n" +
			" </tr>\n" +
			"\n" +
			"<tr resource=\"testCaseDesign/" + data[0] + "\" typeof=\"vtc:TestCaseDesign\">\n" +
			" <td property=\"vtc:testId\">" + data[0] + "</td>\n" +
			" <td property=\"vtc:desiredBehaviour\">" + data[3] + "</td>\n" +
			" </tr>\n" +
			"\n";
			htmlWriter.write(entry);
			}
			} else {
			categoryChanger = data[2];

			String entry = "<tr resource=\"testCaseDesign/" + data[0] + "\" typeof=\"vtc:TestCaseDesign\">\n" +
			" <td property=\"vtc:testId\">" + data[0] + "</td>\n" +
			" <td><span property=\"vtc:comesFromRequirement\" href=\"requirement/" + data[1] + "\"><a href=\"../site/requirements.html#" + data[1] + "\">" + data[1] + "</a></span></td>\n" +
			" <td property=\"vtc:desiredBehaviour\">" + data[3] + "</td>\n" +
			" </tr>\n" +
			"\n";
			htmlWriter.write(entry);
			}
			}

			}
			}
			}

			String endTableData = "</table> \n" +
			"\n" +
			"<p>\n" +
			"</p>\n" +
			"\n";
			htmlWriter.write(endTableData);

			htmlWriter.write(footer);


			} catch (IOException e) {
			e.printStackTrace();
			} finally{
			try {
			if (htmlWriter != null) {
			htmlWriter.close();
			}
			} catch (IOException e) {
			e.printStackTrace();
			}
			}

			}



			public static List<String[]> readDataFromCustomSeperator(File file) {
			try {
			// Create an object of file reader class with CSV file as a parameter.
			FileReader filereader = new FileReader(file);

			// create csvParser object with
			// custom seperator semi-colon
			CSVParser parser = new CSVParserBuilder().withSeparator(';').build();

			// create csvReader object with parameter
			// filereader and parser
			CSVReader csvReader = new CSVReaderBuilder(filereader)
			.withCSVParser(parser)
			.build();

			// Read all data at once
			List<String[]> allData = csvReader.readAll();
			return allData;

			}
			catch (Exception e) {
			e.printStackTrace();
			}
			return null;
			}


			}