Worked on ShaclValidationManager. (1a81703f) · Commits · SAREF / saref-pipeline

pom.xml

+6 −0

Original line number	Diff line number	Diff line
		@@ -126,6 +126,12 @@
		<version>2.10.1</version>
		</dependency>

		<dependency>
		<groupId>com.fasterxml.jackson.core</groupId>
		<artifactId>jackson-databind</artifactId>
		<version>2.10.1</version>
		</dependency>

		<dependency>
		<!-- see Porcelain https://git-scm.com/book/uz/v2/Appendix-B%3A-Embedding-Git-in-your-Applications-JGit -->
		<groupId>org.eclipse.jgit</groupId>

src/main/java/fr/mines_stetienne/ci/saref/checkers/Clause_9_4_6_Checker.java

+54 −80

Original line number	Diff line number	Diff line
		@@ -25,95 +25,33 @@
		*/
		package fr.mines_stetienne.ci.saref.checkers;

		import java.io.File;
		import java.io.IOException;
		import java.util.HashMap;
		import java.util.Map;
		import java.net.http.HttpResponse;
		import java.nio.file.Files;
		import java.util.stream.Collectors;

		import org.apache.jena.rdf.model.Model;
		import org.apache.jena.rdf.model.ResIterator;
		import org.apache.jena.rdf.model.Resource;
		import org.apache.jena.vocabulary.DCTypes;
		import org.apache.jena.vocabulary.RDF;

		import fr.mines_stetienne.ci.saref.SAREF;
		import fr.mines_stetienne.ci.saref.SAREFPipelineException;
		import fr.mines_stetienne.ci.saref.entities.SAREFExample;
		//import fr.mines_stetienne.ci.saref.entities.SAREFVersion;
		import fr.mines_stetienne.ci.saref.entities.SAREFVersionName;
		import fr.mines_stetienne.ci.saref.managers.RepositoryManager;
		import fr.mines_stetienne.ci.saref.vocabs.SHACL;
		import fr.mines_stetienne.ci.saref.managers.ShaclValidationManager;
		import fr.mines_stetienne.ci.saref.utils.Languages;
		import org.semanticweb.owlapi.model.OWLOntology;

		/**
		* Checks TS 103 673 Clause 9.4.6: Conformance to reference ontology patterns.
		* The ontology in the ontology document of a SAREF project version shall conform to the SHACL specification of each
		* reference ontology pattern defined in this SAREF project version.
		*/
		public class Clause_9_4_6_Checker extends AbstractShaclChecker {

		private enum MESSAGE implements MessageResource {
		dataset, conformsTo1, conformsToNot, conformsToNot2, conformsTo2, conformsTo3,
		title1, title2, title3, abstract1, description1, description2,
		license, ioexception
		}

		private final SAREFExample example = null;

		public Clause_9_4_6_Checker(RepositoryManager repositoryManager) //, SAREFExample shacl
		throws SAREFPipelineException {
		super(repositoryManager, Clause_9_4_6_Checker.class); // , "example " + example.getName());
		//this.example = shacl;
		}

		@Override
		protected Model getModel() {
		Model model = example.getModel();
		final String defaultVersion = "v0.0.1";
		final String regex = "^" + example.getIRI().replace(defaultVersion, SAREF.REGEX_VERSION_NUMBER) + "$";
		int onto = 0;
		boolean found = false;
		for (ResIterator it = model.listSubjectsWithProperty(RDF.type, DCTypes.Dataset); it.hasNext();) {
		Resource r = it.next();
		if(versionName.equals(SAREFVersionName.DEFAULT)) {
		if(r.getURI().matches(regex)) {
		found = true;
		}
		} else {
		if(r.getURI().equals( example.getIRI() )) {
		found = true;
		}
		}
		onto++;
		}
		if (onto != 1 \|\| !found) {
		final String msg;
		if(versionName.equals(SAREFVersionName.DEFAULT)) {
		msg = getMessage(MESSAGE.dataset, example.getIRI()).replace("v0.0.1", "<<some SAREF version name>>");
		} else {
		msg = getMessage(MESSAGE.dataset, example.getIRI());
		}
		logError(msg);
		}
		return model;
		}
		public class Clause_9_4_6_Checker extends AbstractClauseChecker {

		protected final void updateShapeModel() {
		if(!versionName.equals(SAREFVersionName.DEFAULT)) {
		shapeModel.add(MESSAGE.conformsToNot.asResource(), SHACL.pattern, exactly(version.getIRI()));
		add(MESSAGE.conformsTo1, version.getIRI());
		private enum MESSAGE {
		one, missing, ioexception, server_unavailable
		}
		add(MESSAGE.conformsTo2);
		add(MESSAGE.conformsTo3);
		add(MESSAGE.title1);
		add(MESSAGE.title2);
		add(MESSAGE.title3);
		add(MESSAGE.abstract1);
		add(MESSAGE.description1);
		add(MESSAGE.description2);
		add(exactly(SAREF.LICENSE), MESSAGE.license);

		if(versionName.equals(SAREFVersionName.DEFAULT)) {
		remove(MESSAGE.conformsTo1);
		}
		public Clause_9_4_6_Checker(RepositoryManager repositoryManager) throws SAREFPipelineException {
		super(repositoryManager, Clause_9_4_6_Checker.class);
		}

		/**
		@@ -124,13 +62,49 @@ public class Clause_9_4_6_Checker extends AbstractShaclChecker {
		reference ontology pattern defined in this other SAREF project version.
		This method calls the generic docker image that runs a specific shacl files.
		* */
		protected final void validateOntologyPerShacl(String ontologyToValidate, String shaclFile) {
		//<<< process every file
		@Override
		public void checkClause() throws SAREFPipelineException {
		OWLOntology ontology = pipeline.getOntologyManager().loadOntology(version, errorLogger);
		if (ontology == null) {
		return;
		}
		File dir = new File(repository.getDirectory(), "ontology");
		File file = new File(dir, repository.getProject().getOntologyFileName(Languages.TEXT_TURTLE));
		String ontologyToValidate = file.getAbsolutePath();

		// First get the available directories. If none, not a problem.
		File dir2 = new File(repository.getDirectory(), "patterns");
		String[] directoryList = new String[]{};
		try {
		HttpResponse<String> response = new ShaclValidationManager(ontologyToValidate).validateOntologyWithShacl(shaclFile);
		} catch (IOException \| InterruptedException e) {
		logError(getMessage(Clause_9_4_6_Checker.MESSAGE.ioexception));
		String directories = Files.walk(dir2.toPath()).filter(p -> {
		try {
		return p.toFile().isDirectory() && !Files.isSameFile(dir2.toPath(), p);
		} catch(IOException ex) {
		return false;
		}
		}).map(p -> p.toString()).collect(Collectors.joining(", "));
		if (directories.isEmpty()) {
		return; // no SHACL directories found.
		}
		directoryList = directories.split(",");
		} catch (IOException e) {
		logError(getMessage(Clause_9_4_6_Checker.MESSAGE.ioexception), e);
		}

		ShaclValidationManager.PingValidationServer();

		Map<String, HttpResponse<String>> responseMap = new HashMap<String, HttpResponse<String>>();
		// process every shapes.ttl file in the sub-directories of the patterns directory.
		for(int i = 0; i < directoryList.length; i++) {
		String shaclFileName = directoryList[i] + "/shapes.ttl"; // "file://" +
		try {
		HttpResponse<String> response = new ShaclValidationManager(ontologyToValidate).validateOntologyWithShacl(shaclFileName);
		System.out.println(response); //<<<
		responseMap.put(shaclFileName, response); //<<< parse map for errors
		} catch (IOException \| InterruptedException e) {
		logError(getMessage(Clause_9_4_6_Checker.MESSAGE.ioexception, shaclFileName));
		}
		}
		//System.out.println(responseMap); //<<<
		}
		}
		No newline at end of file

src/main/java/fr/mines_stetienne/ci/saref/checkers/Clause_9_4_Checker.java

+1 −0

Original line number	Diff line number	Diff line
		@@ -29,6 +29,7 @@ import java.io.File;

		import fr.mines_stetienne.ci.saref.SAREFPipelineException;
		import fr.mines_stetienne.ci.saref.managers.RepositoryManager;
		import fr.mines_stetienne.ci.saref.utils.Languages;

		/**
		* Checks TS 103 673 Clause 9.4: The Ontology specification

src/main/java/fr/mines_stetienne/ci/saref/checkers/Clause_9_8_1_Checker.java

+2 −2

Original line number	Diff line number	Diff line
		@@ -66,7 +66,7 @@ public class Clause_9_8_1_Checker extends AbstractClauseChecker {
		}

		// Step 4. Fetch the list of turtle files in the vocabularies directory.
		File[] fileList = null;
		File[] fileList = new File[]{};
		try {
		if (Files.walk(dir.toPath(), 1).filter(p -> !p.toFile().isFile() && !p.toFile().getName().startsWith("."))
		.count() != 1) {
		@@ -89,7 +89,7 @@ public class Clause_9_8_1_Checker extends AbstractClauseChecker {
		return;
		}

		// <<< Step 6. Match each imports statements to the presence of each file using the last tag (e.g. "properties" +".ttl")
		// TODO: Step 6. Match each imports statements to the presence of each file using the last tag (e.g. "properties" +".ttl")
		// Ignore turtle files which do not have a corresponding imports statements.
		/*Iterator<OWLImportsDeclaration> itr = this.importDeclarations.iterator();
		while(itr.hasNext()) { // <https://saref.etsi.org/saref4auto/v2.1.1/properties/>

src/main/java/fr/mines_stetienne/ci/saref/managers/ShaclValidationManager.java

+85 −87

Original line number	Diff line number	Diff line
		@@ -26,86 +26,101 @@
		package fr.mines_stetienne.ci.saref.managers;

		import java.io.IOException;
		import java.net.HttpURLConnection;
		import java.net.URI;
		import java.net.URL;
		import java.net.URLEncoder;
		import java.net.http.*;
		import java.nio.file.Files;
		import java.nio.file.Path;
		import java.nio.charset.StandardCharsets;
		import java.util.HashMap;
		import java.util.Map;
		import java.util.Base64;

		import com.google.gson.JsonObject;
		import com.google.gson.JsonParser;
		import com.fasterxml.jackson.databind.ObjectMapper;
		//import fr.mines_stetienne.ci.saref.SAREFErrorLogger;
		import fr.mines_stetienne.ci.saref.SAREFPipelineException;

		/*
		/* docker run -d --name SAREFvalidator -p 8080:8080 isaitb/shacl-validator # runs the generic validator.
		Docs for RDF validator: https://www.itb.ec.europa.eu/docs/guides/latest/validatingRDF/index.html
		Base docker image: https://hub.docker.com/r/isaitb/shacl-validator
		Base docker image: https://hub.docker.com/r/isaitb/shacl-validator (generic validator)
		Use Swagger UI for testing: http://localhost:8080/shacl/swagger-ui/index.html#/{domain}/api/validate
		BASE64 Encoder/Decoder: https://www.base64encode.org/
		docker run -d --name SAREFvalidator -p 8080:8080 local/ontologyvalidator # this runs the general validator.
		* */
		public class ShaclValidationManager {

		private final static String BASE_URL = "http://localhost:8080/shacl/"; // <<<
		private static String VALIDATOR_URL = "http://localhost:8080/shacl/";
		private String ontologyToValidate; // file:///home/davidgnabasik/dev/real_saref4auto/ontology/saref4auto.ttl or URL
		private final HttpClient client;
		private final ObjectMapper objectMapper;
		private HttpClient client;
		private ObjectMapper objectMapper;

		//private String shaclRequestBody =
		// "{ \"contentToValidate\": \"ONTOLOGY_AS_BASE64_STRING\", \"embeddingMethod\": \"BASE64\", \"contentSyntax\": \"text/turtle\", \"reportSyntax\": \"application/json\", \"locale\": \"en\", \"rdfReportSyntax\": \"application/json\", \"externalRules\": [ { \"ruleSet\": \"SHACL_AS_BASE64_STRING\", \"embeddingMethod\": \"BASE64\", \"ruleSyntax\": \"text/turtle\" } ] }";

		private static String encode(Object obj) {
		private static String encodeBody(Object obj) {
		return URLEncoder.encode(obj.toString(), StandardCharsets.UTF_8);
		}

		private enum MESSAGE {
		private enum MESSAGE { // each of these accepts a single string parameter.
		http_exception, ioexception
		}

		// The full list of available tags: https://www.itb.ec.europa.eu/docs/guides/latest/validatingRDF/
		// Replace default values for {contentToValidate, validationType}
		private String shaclRequestBody = "{ "+ // generic validator option
		"'contentToValidate': 'RDF_CONTENT_AS_BASE64_ENCODED_STRING', "+ // entire content of ontology file (or URI)
		"'embeddingMethod': 'BASE64', "+ // {STRING,URL,BASE64}
		"'contentSyntax': 'text/turtle', "+
		"'validationType': 'string', "+
		"'reportSyntax': 'application/json', "+
		"'locale': 'en', "+
		"'rdfReportSyntax': 'application/json' "+
		"}";
		/* "contentToValidate": "",
		"contentSyntax": "text/turtle",
		"embeddingMethod": "BASE64",
		"validationType": "Clause_9_4_3_1",
		"reportSyntax": "application/json",
		"locale": "en" */

		JsonObject jsonObject = new JsonParser().parse(shaclRequestBody).getAsJsonObject();

		/**
		* Open a REST interface to the SHACL validator.
		*/
		public ShaclValidationManager(String ontologyToValidate) throws IOException {
		client = HttpClient.newHttpClient();
		objectMapper = new ObjectMapper();
		this.ontologyToValidate = ontologyToValidate; //<<< test for ontology file presence.
		this.ontologyToValidate = ontologyToValidate;
		String url = System.getenv("VALIDATOR_URL");
		if (url != null) VALIDATOR_URL = url;
		}

		/**
		* Is the validation server accessible?
		* @throws SAREFPipelineException
		*/
		public static void PingValidationServer() throws SAREFPipelineException {
		try {
		URL url = new URL(VALIDATOR_URL);
		HttpURLConnection connection = (HttpURLConnection) url.openConnection();
		connection.setRequestMethod("GET");
		connection.connect();
		connection.getResponseCode();
		} catch (IOException e) {
		throw new SAREFPipelineException("response", "Unknown host: " + VALIDATOR_URL);
		}
		//logError(getMessage(ShaclValidationManager.MESSAGE.http_exception, VALIDATOR_URL));
		}

		/**
		* Preferred validation format is BASE64. Read file; return single BASE64-encoded string.
		* @throws SAREFPipelineException
		*/
		public static String FormatAsBase64(String ontologyFile) throws SAREFPipelineException {
		try {
		String value = Files.readString(Path.of(ontologyFile)); // 2Gb file size limit.
		return Base64.getUrlEncoder().encodeToString(value.getBytes());
		} catch (IOException ex) {
		throw new SAREFPipelineException("FormatAsBase64", "Unable to format file as base64: " + ontologyFile);
		}
		}

		public static HttpRequest.BodyPublisher ofForm(Map<String, String> data) {
		public static HttpRequest.BodyPublisher ofForm(Map<String, Object> data) {
		StringBuilder body = new StringBuilder();
		for (String dataKey : data.keySet()) {
		for (Object dataKey : data.keySet()) {
		if (body.length() > 0) {
		body.append("&");
		}
		body.append(encode(dataKey))
		body.append(encodeBody(dataKey))
		.append("=")
		.append(encode(data.get(dataKey)));
		.append(encodeBody(data.get(dataKey)));
		}
		return HttpRequest.BodyPublishers.ofString(body.toString());
		}

		/**
		* POST request to Validate a single RDF instance: {domain}/api/validate/{requestBody}
		<<< This method uses the validator configured to read the Clause_9_4_3_1_3.ttl, Clause_9_4_4_2.ttl, Clause_9_6_3.ttl shacl files.
		* Response: HTTP code 200 for successful validation. POST JSON return report format:
		* POST JSON response format:
		{
		"date": "2024-04-05T10:18:24.197+0000",
		"result": "SUCCESS",
		@@ -121,57 +136,40 @@ public class ShaclValidationManager {
		"reports": {},
		"name": "SHACL Validation"
		}
		*/
		public HttpResponse<String> validateOntologyWithShacl(String shaclFile)
		throws IOException, InterruptedException {
		Map<String, String> data = new HashMap<>();
		data = objectMapper.readValue(shaclRequestBody, HashMap.class);
		data.put("contentToValidate", ontologyToValidate);
		//data.put("validationType", validationType);
		System.out.println("Map is: "+data);//<<<
		/* "contentToValidate": "RDF_CONTENT_AS_BASE64_ENCODED_STRING",
		"contentSyntax": "text/turtle",
		"embeddingMethod": "BASE64",
		"validationType": "Clause_9_4_3_1",
		"reportSyntax": "application/json",
		"locale": "en" */

		HttpRequest request = HttpRequest.newBuilder()
		.header("Content-Type", "text/turtle") // application/x-www-form-urlencoded
		.uri(URI.create(BASE_URL + "/api/validate" ))
		.POST(ofForm(data))
		.build();

		HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
		System.out.println("Status code: " + response.statusCode());
		System.out.println("\n Body: " + response.body());

		return response;
		}

		/**
		* If shaclDomain is empty, return info for all defined domains. http://localhost:8080/shacl/api/info
		* Not needed since a configured validator is not used -- the Clause_9_*.ttl files are augmented (parameterized) so they are only partially useful.
		* Generic validator option: 1. Replace {contentToValidate} value with the ontology to validate encoded in BASE64. (This can aos be done with URLs.)
		* 2. Configure the user-provided SHACL shape as a RuleSet element of the externalRules array. The content of each RuleSet is as follows:
		* [ "ruleSet": "BASE64-encoded_SHACL_shape", "embeddingMethod":"BASE64", "ruleSyntax": "text/turtle" ] We run one SHACL shape at a time.
		* POST request to Validate a single RDF instance.
		* @throws IOException
		* @throws InterruptedException
		*/
		public HttpResponse<String> getShaclDomainInfo() throws IOException, InterruptedException {
		Map<String, String> data = new HashMap<>();
		data = objectMapper.readValue(shaclRequestBody, HashMap.class);

		String url = BASE_URL + "shacl/api/info";

		public HttpResponse<String> validateOntologyWithShacl(String shaclFile) throws SAREFPipelineException, InterruptedException {
		try {
		String ontologyBase64String = FormatAsBase64(this.ontologyToValidate);
		String shaclBase64String = FormatAsBase64(shaclFile);
		// populate shaclRequestBody json struct to map (no TypeReference)
		String shaclRequestBody = "{ \"contentToValidate\": \"" + ontologyBase64String +
		"\", \"embeddingMethod\": \"BASE64\", \"contentSyntax\": \"text/turtle\", \"reportSyntax\": \"application/json\", \"locale\": \"en\", \"rdfReportSyntax\": \"application/json\", \"externalRules\": [ { \"ruleSet\": \"" +
		shaclBase64String + "\", \"embeddingMethod\": \"BASE64\", \"ruleSyntax\": \"text/turtle\" } ] }";
		byte[] mapData = shaclRequestBody.getBytes();
		Map<String,Object> myMap = objectMapper.readValue(mapData, HashMap.class);
		// now update JSON data
		//myMap.put("contentToValidate", ontologyBase64String);
		//myMap.put("ruleSet", shaclBase64String);
		HttpRequest request = HttpRequest.newBuilder()
		.header("Accept", "text/turtle")
		.header("Content-Type", "text/turtle")
		.uri(URI.create(url))
		.GET()
		.header("Content-Type", "text/turtle") // application/x-www-form-urlencoded
		.uri(URI.create(VALIDATOR_URL + "any/upload"))
		.POST(ofForm(myMap))
		.build();

		HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
		System.out.println("Status code: " + response.statusCode());
		System.out.println("Status code: " + response.statusCode()); //<<<
		System.out.println("\n Body: " + response.body());

		return response;
		} catch (IOException ex) {
		throw new SAREFPipelineException("validateOntologyWithShacl", "Unable to process SHACL file: " + shaclFile);
		}
		}

		}