Worked on ShaclValidationManager. (ddb520ba) · Commits · SAREF / saref-pipeline

src/main/java/fr/mines_stetienne/ci/saref/checkers/Clause_9_4_6_Checker.java

+15 −15

Original line number	Diff line number	Diff line
		@@ -27,8 +27,8 @@ package fr.mines_stetienne.ci.saref.checkers;

		import java.io.File;
		import java.io.IOException;
		import java.util.HashMap;
		import java.util.Map;
		import java.util.ArrayList;
		import java.util.List;
		import java.net.http.HttpResponse;
		import java.nio.file.Files;
		import java.util.stream.Collectors;
		@@ -74,8 +74,10 @@ public class Clause_9_4_6_Checker extends AbstractClauseChecker {

		// First get the available directories. If none, not a problem.
		File dir2 = new File(repository.getDirectory(), "patterns");
		String[] directoryList = new String[]{};
		List<String> shaclFileList = new ArrayList();
		// process every shapes.ttl file in the sub-directories of the patterns directory.
		try {
		String[] directoryList = new String[]{};
		String directories = Files.walk(dir2.toPath()).filter(p -> {
		try {
		return p.toFile().isDirectory() && !Files.isSameFile(dir2.toPath(), p);
		@@ -87,24 +89,22 @@ public class Clause_9_4_6_Checker extends AbstractClauseChecker {
		return; // no SHACL directories found.
		}
		directoryList = directories.split(",");
		for(int i = 0; i < directoryList.length; i++) {
		shaclFileList.add(directoryList[i] + "/shapes.ttl");
		}
		} catch (IOException e) {
		logError(getMessage(Clause_9_4_6_Checker.MESSAGE.ioexception), e);
		}

		ShaclValidationManager.PingValidationServer();

		Map<String, HttpResponse<String>> responseMap = new HashMap<String, HttpResponse<String>>();
		// process every shapes.ttl file in the sub-directories of the patterns directory.
		for(int i = 0; i < directoryList.length; i++) {
		String shaclFileName = directoryList[i] + "/shapes.ttl"; // "file://" +
		//Map<String, HttpResponse<String>> responseMap = new HashMap<String, HttpResponse<String>>();
		try {
		HttpResponse<String> response = new ShaclValidationManager(ontologyToValidate).validateOntologyWithShacl(shaclFileName);
		System.out.println(response); //<<<
		responseMap.put(shaclFileName, response); //<<< parse map for errors
		ShaclValidationManager shaclValidationManager = new ShaclValidationManager(ontologyToValidate);
		HttpResponse<String> response = shaclValidationManager.ValidateOntologyWithShacl(shaclFileList);
		//responseMap.put(shaclFileName, response); // parse map for errors
		} catch (IOException \| InterruptedException e) {
		logError(getMessage(Clause_9_4_6_Checker.MESSAGE.ioexception, shaclFileName));
		}
		logError(getMessage(Clause_9_4_6_Checker.MESSAGE.ioexception, ontologyToValidate));
		}
		//System.out.println(responseMap); //<<<
		}
		}
		No newline at end of file

src/main/java/fr/mines_stetienne/ci/saref/managers/ShaclValidationManager.java

+140 −52

Original line number	Diff line number	Diff line
		@@ -32,30 +32,35 @@ import java.net.URL;
		import java.net.URLEncoder;
		import java.net.http.*;
		import java.nio.file.Files;
		import java.nio.file.Path;
		import java.nio.charset.StandardCharsets;
		import java.util.HashMap;
		import java.nio.file.Paths;
		import java.util.Map;
		import java.util.Base64;
		import java.util.List;
		import java.util.ArrayList;
		import java.util.HashMap;

		import com.fasterxml.jackson.core.type.TypeReference;
		import com.fasterxml.jackson.databind.ObjectMapper;
		import fr.mines_stetienne.ci.saref.SAREFPipelineException;

		/* docker run -d --name SAREFvalidator -p 8080:8080 isaitb/shacl-validator # runs the generic validator.
		/* docker run -d --name SAREFvalidator -p 8080:8080 isaitb/shacl-validator # runs the generic validator with Dockerfile
		Docs for RDF validator: https://www.itb.ec.europa.eu/docs/guides/latest/validatingRDF/index.html
		Base docker image: https://hub.docker.com/r/isaitb/shacl-validator (generic validator)
		Use Swagger UI for testing: http://localhost:8080/shacl/swagger-ui/index.html#/{domain}/api/validate
		BASE64 Encoder/Decoder: https://www.base64encode.org/
		* */
		public class ShaclValidationManager {

		private static String VALIDATOR_URL = "http://localhost:8080/shacl/";
		private final String TEXT_TURTLE = "text/turtle";
		private final String JSON_FORMAT = "application/json";
		private final String STRING_FORMAT = "STRING";

		private String ontologyToValidate; // file:///home/davidgnabasik/dev/real_saref4auto/ontology/saref4auto.ttl or URL
		private List<String> shaclFileList;
		private List<RuleSet> ruleSetList;
		private HttpClient client;
		private ObjectMapper objectMapper;

		//private String shaclRequestBody =
		// "{ \"contentToValidate\": \"ONTOLOGY_AS_BASE64_STRING\", \"embeddingMethod\": \"BASE64\", \"contentSyntax\": \"text/turtle\", \"reportSyntax\": \"application/json\", \"locale\": \"en\", \"rdfReportSyntax\": \"application/json\", \"externalRules\": [ { \"ruleSet\": \"SHACL_AS_BASE64_STRING\", \"embeddingMethod\": \"BASE64\", \"ruleSyntax\": \"text/turtle\" } ] }";
		private ValidationRequest validationRequest;

		private static String encodeBody(Object obj) {
		return URLEncoder.encode(obj.toString(), StandardCharsets.UTF_8);
		@@ -65,12 +70,78 @@ public class ShaclValidationManager {
		http_exception, ioexception
		}

		class RuleSet {
		private String ruleSet; // JSON-encoded_SHACL_shape

		public String getRuleSet() {
		return ruleSet;
		}
		public void setRuleSet(String ruleSet) {
		this.ruleSet = ruleSet;
		}
		// constants for now...
		public String getEmbeddingMethod() { return STRING_FORMAT; }
		public String getRuleSyntax() { return TEXT_TURTLE; }
		}

		class ValidationRequest {
		private String contentToValidate; // JSON-encoded ontology
		private List<RuleSet> externalRules;
		private Map<String, Object> properties; // needed?

		public String getContentToValidate() {
		return contentToValidate;
		}

		public void setContentToValidate(String contentToValidate) {
		this.contentToValidate = contentToValidate;
		}

		public Map<String, Object> getProperties() {
		return properties;
		}

		public void setProperties(Map<String, Object> properties) {
		this.properties = properties;
		}

		// constants for now...
		public String getEmbeddingMethod() {
		return STRING_FORMAT;
		}

		public String getContentSyntax() {
		return TEXT_TURTLE;
		}

		public String getReportSyntax() {
		return JSON_FORMAT;
		}

		public String getRdfReportSyntax() {
		return JSON_FORMAT;
		}

		public String getLocale() {
		return "en";
		}

		public List<RuleSet> getExternalRules() {
		return externalRules;
		}

		public void setExternalRules(List<RuleSet> externalRules) {
		this.externalRules = externalRules;
		}

		}

		/**
		* Open a REST interface to the SHACL validator.
		*/
		public ShaclValidationManager(String ontologyToValidate) throws IOException {
		client = HttpClient.newHttpClient();
		objectMapper = new ObjectMapper();
		this.client = HttpClient.newHttpClient();
		this.objectMapper = new ObjectMapper();
		this.ontologyToValidate = ontologyToValidate;
		String url = System.getenv("VALIDATOR_URL");
		if (url != null) VALIDATOR_URL = url;
		@@ -94,19 +165,49 @@ public class ShaclValidationManager {
		}

		/**
		* Preferred validation format is BASE64. Read file; return single BASE64-encoded string.
		* Read shacl files. Assign this.shaclFileList, this.ruleSetList.
		* Preferred validation format is JSON-escaped text. Requires \r\n as line breaks!!!
		*/
		private void createRuleSetList(List<String> shaclFileList) {
		this.shaclFileList = new ArrayList();
		this.ruleSetList = new ArrayList();

		for (int i = 0; i < shaclFileList.size(); i++) {
		String readingFile = shaclFileList.get(i);
		try {
		String fileStr = new String(Files.readAllBytes(Paths.get(readingFile)));
		RuleSet ruleSet = new RuleSet();
		ruleSet.setRuleSet(fileStr.replaceAll("[\n]","\r\n")); // still need to json-escape.
		this.shaclFileList.add(readingFile);
		this.ruleSetList.add(ruleSet);
		} catch (IOException ex) {
		System.out.println("ShaclValidationManager unable to read SHACL file: " + readingFile);
		}
		}
		}

		/**
		* Assign ValidationRequest object.
		* Preferred validation format is JSON-escaped text. Requires \r\n as line breaks!!!
		* @throws SAREFPipelineException
		*/
		public static String FormatAsBase64(String ontologyFile) throws SAREFPipelineException {
		private void createValidationRequest(List<String> shaclFileList) throws SAREFPipelineException {
		createRuleSetList(shaclFileList);
		if (this.ruleSetList.size() < 1) {
		throw new SAREFPipelineException("ShaclValidationManager", "No SHACL files to process!");
		}
		// process ontology file
		try {
		String value = Files.readString(Path.of(ontologyFile)); // 2Gb file size limit.
		return Base64.getUrlEncoder().encodeToString(value.getBytes());
		String fileStr = new String(Files.readAllBytes(Paths.get(this.ontologyToValidate)));
		this.validationRequest = new ValidationRequest();
		this.validationRequest.setContentToValidate(fileStr.replaceAll("[\n]","\r\n")); // still need to json-escape.
		this.validationRequest.setExternalRules(this.ruleSetList);
		} catch (IOException ex) {
		throw new SAREFPipelineException("FormatAsBase64", "Unable to format file as base64: " + ontologyFile);
		throw new SAREFPipelineException("ShaclValidationManager", "Unable to read ontology file: " + this.ontologyToValidate);
		}
		}

		public static HttpRequest.BodyPublisher ofForm(Map<String, Object> data) {
		private HttpRequest.BodyPublisher ofForm(Map<String, Object> data) {
		StringBuilder body = new StringBuilder();
		for (Object dataKey : data.keySet()) {
		if (body.length() > 0) {
		@@ -121,46 +222,33 @@ public class ShaclValidationManager {

		/**
		* POST JSON response format:
		{
		"date": "2024-04-05T10:18:24.197+0000",
		"result": "SUCCESS",
		"overview": {
		"profileID": "Clause_9_4_3_1"
		},
		"counters": {
		"nrOfAssertions": 0,
		"nrOfErrors": 0,
		"nrOfWarnings": 0
		},
		"context": {},
		"reports": {},
		"name": "SHACL Validation"
		}

		* Generic validator option: 1. Replace {contentToValidate} value with the ontology to validate encoded in BASE64. (This can aos be done with URLs.)
		* 2. Configure the user-provided SHACL shape as a RuleSet element of the externalRules array. The content of each RuleSet is as follows:
		* [ "ruleSet": "BASE64-encoded_SHACL_shape", "embeddingMethod":"BASE64", "ruleSyntax": "text/turtle" ] We run one SHACL shape at a time.
		* POST request to Validate a single RDF instance.
		LIST OF: @prefix : <https://saref.etsi.org/saref4auto/> .
		[ rdf:type sh:ValidationReport ;
		sh:conforms true
		] .
		* Generic validator option: 1. Replace {contentToValidate} value with the ontology to validate encoded in JSON. This can also be done with URLs.
		* 2. Configure the user-provided SHACL shape as a RuleSet element of the externalRules array.
		* [ "ruleSet": "JSON-encoded_SHACL_shape", "embeddingMethod":"STRING", "ruleSyntax": TEXT_TURTLE ]
		* POST request to Validate a single RDF ontology.
		* @throws SAREFPipelineException
		* @throws IOException
		* @throws InterruptedException
		*/
		public HttpResponse<String> validateOntologyWithShacl(String shaclFile) throws SAREFPipelineException, InterruptedException {
		public HttpResponse<String> ValidateOntologyWithShacl(List<String> shaclFileList) throws SAREFPipelineException, InterruptedException {
		try {
		String ontologyBase64String = FormatAsBase64(this.ontologyToValidate);
		String shaclBase64String = FormatAsBase64(shaclFile);
		// populate shaclRequestBody json struct to map (no TypeReference)
		String shaclRequestBody = "{ \"contentToValidate\": \"" + ontologyBase64String +
		"\", \"embeddingMethod\": \"BASE64\", \"contentSyntax\": \"text/turtle\", \"reportSyntax\": \"application/json\", \"locale\": \"en\", \"rdfReportSyntax\": \"application/json\", \"externalRules\": [ { \"ruleSet\": \"" +
		shaclBase64String + "\", \"embeddingMethod\": \"BASE64\", \"ruleSyntax\": \"text/turtle\" } ] }";
		byte[] mapData = shaclRequestBody.getBytes();
		Map<String,Object> myMap = objectMapper.readValue(mapData, HashMap.class);
		// now update JSON data
		//myMap.put("contentToValidate", ontologyBase64String);
		//myMap.put("ruleSet", shaclBase64String);
		createValidationRequest(shaclFileList);

		//String json = this.objectMapper.writeValueAsString(this.validationRequest);
		//String escapedJson = StringEscapeUtils.escapeJson(json);
		//Map<String, Object> postMap = this.objectMapper.convertValue(this.validationRequest, new TypeReference<Map<String, Object>>() {});
		Map<String, Object> data = new HashMap<>();
		data.put("fistname", "admin");
		System.out.println("ValidateOntologyWithShacl...");//<<<

		HttpRequest request = HttpRequest.newBuilder()
		.header("Content-Type", "text/turtle") // application/x-www-form-urlencoded
		.uri(URI.create(VALIDATOR_URL + "any/upload"))
		.POST(ofForm(myMap))
		.header("Content-Type", TEXT_TURTLE) // application/x-www-form-urlencoded
		.uri(URI.create(VALIDATOR_URL + "any/upload")) // any/api/validate
		.POST(ofForm(data))
		.build();

		HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
		@@ -168,7 +256,7 @@ public class ShaclValidationManager {
		System.out.println("\n Body: " + response.body());
		return response;
		} catch (IOException ex) {
		throw new SAREFPipelineException("validateOntologyWithShacl", "Unable to process SHACL file: " + shaclFile);
		throw new SAREFPipelineException("ValidateOntologyWithShacl", "Unable to validate ontology: " + this.ontologyToValidate);
		}
		}

Admin message