Worked on ShaclValidationManager. (d20479db) · Commits · SAREF / saref-pipeline

docker.SHACL.validation.configuration.md

0 → 100644

+53 −0

Original line number	Diff line number	Diff line
		docker.SHACL.validation.configuration.txt::

		SHACL Validation server notes: See ShaclValidationManager.java
		Support per validation type of loading the imports defined in the input that can also be set by users at validation time.
		Support for querying SPARQL endpoints to retrieve the content to validate.
		Remote SHACL shape caching: Caching is used to avoid constant lookups of remote SHACL shape files.
		Once loaded, remote SHACL shape files will be automatically refreshed every hour.

		Docker development configuration::https://www.itb.ec.europa.eu/docs/guides/latest/installingTheTestBed/
		These steps are for the specific validator, not the generic one that is in actual use.
		Step 1: Install Docker Community Edition & Docker Compose from https://docs.docker.com/desktop/install/ubuntu/
		docker --version => Docker version 26.0.0, build 2ae903e
		docker compose version => Docker Compose version v2.25.0
		Step 2: Define the test bed’s configuration:
		mkdir ~/dev/testbed && cd ~/dev/testbed && touch docker-compose.yml
		sudo groupadd docker && sudo usermod -aG docker ${USER} && sudo chmod 666 /var/run/docker.sock
		sudo ufw allow 8080/tcp && sudo ufw status verbose (Later: sudo netstat -lnp --tcp --udp)
		Step 3: REST interface: http://localhost:8080/shacl/DOMAIN/api/validate DOMAIN={Commodity\|Ontology}
		Parameter Verb Description Example
		validate POST Validate one RDF instance. Request payload type: application/json. http://localhost:8080/shacl/Ontology/api/validate <payload>
		Step 4: Test your installation: browse to http://localhost:9000/ => test@test.com / test => })cdVZP8>=}wZFa <adminPassword>
		Step 5: Docker commands to manage the test bed:
		docker build -t local/OntologyValidator . # this binds the specific validator to the shapes configuration. NOT USED!
		docker run -d --name SAREFvalidator -p 8080:8080 isaitb/shacl-validator # this runs the general validator.


		Generic Validator Dockerfile::
		FROM isaitb/shacl-validator:latest
		COPY resources /validator/resources/
		ENV validator.resourceRoot /validator/resources/

		Docs for RDF validator: https://www.itb.ec.europa.eu/docs/guides/latest/validatingRDF/index.html
		Base docker image: https://hub.docker.com/r/isaitb/shacl-validator
		Use Swagger UI for testing: http://localhost:8080/shacl/swagger-ui/index.html#/{domain}/api/validate
		http://localhost:8080/shacl/swagger-ui.html

		~/dockerls.sh: sudo docker image ls && sudo docker volume ls && sudo docker container ls && sudo docker network ls
		!!! images !!!
		REPOSITORY TAG IMAGE ID CREATED SIZE
		local/ontologyvalidator latest 8b929e2a947c 3 hours ago 436MB # THIS ONE IS NOT USED!
		isaitb/shacl-validator latest 723eca0b679b 2 weeks ago 436MB # THIS ONE IS USED!
		!!! volumes !!!
		DRIVER VOLUME NAME
		!!! containers !!!
		CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
		d9daf40eed08 local/ontologyvalidator "java -XX:+ExitOnOut…" 3 hours ago Up 3 hours 0.0.0.0:8080->8080/tcp, :::8080->8080/tcp SAREFvalidator
		!!! networks !!!
		NETWORK ID NAME DRIVER SCOPE
		4a98c5692963 bridge bridge local
		f02e9cbf7d1d host host local
		631ce42d72e6 none null local

src/main/java/fr/mines_stetienne/ci/saref/checkers/Clause_10_1_Checker.java

+1 −1

Original line number	Diff line number	Diff line
		@@ -60,7 +60,7 @@ public class Clause_10_1_Checker extends AbstractClauseChecker {
		}
		try {
		if (Files.walk(dir.toPath(), 1).filter(p -> !p.toFile().isFile() && !p.toFile().getName().startsWith("."))
		.count() != 1) {
		.count() < 2) { // count README.md
		logError(getMessage(Clause_10_1_Checker.MESSAGE.one, repository.getProject().getOntologyFileName(Languages.TEXT_TURTLE)));
		}
		} catch (Exception ex) {

src/main/java/fr/mines_stetienne/ci/saref/checkers/Clause_9_4_6_Checker.java

+19 −7

Original line number	Diff line number	Diff line
		@@ -29,8 +29,8 @@ import java.io.File;
		import java.io.IOException;
		import java.util.ArrayList;
		import java.util.List;
		import java.net.http.HttpResponse;
		import java.nio.file.Files;
		import java.util.HashMap;
		import java.util.stream.Collectors;

		import fr.mines_stetienne.ci.saref.SAREFPipelineException;
		@@ -47,7 +47,7 @@ import org.semanticweb.owlapi.model.OWLOntology;
		public class Clause_9_4_6_Checker extends AbstractClauseChecker {

		private enum MESSAGE {
		one, missing, ioexception, server_unavailable
		one, missing, ioexception, server_unavailable, shacl_validation_failed
		}

		public Clause_9_4_6_Checker(RepositoryManager repositoryManager) throws SAREFPipelineException {
		@@ -96,15 +96,27 @@ public class Clause_9_4_6_Checker extends AbstractClauseChecker {
		logError(getMessage(Clause_9_4_6_Checker.MESSAGE.ioexception), e);
		}

		ShaclValidationManager.PingValidationServer();

		//Map<String, HttpResponse<String>> responseMap = new HashMap<String, HttpResponse<String>>();
		try {
		ShaclValidationManager.PingValidationServer();
		ShaclValidationManager shaclValidationManager = new ShaclValidationManager(ontologyToValidate);
		HttpResponse<String> response = shaclValidationManager.ValidateOntologyWithShacl(shaclFileList);
		//responseMap.put(shaclFileName, response); // parse map for errors
		HashMap<String, String> response = shaclValidationManager.ValidateOntologyWithShacl(shaclFileList);
		if (!matchHashMap("200","true",response)) {
		logError(getMessage(Clause_9_4_6_Checker.MESSAGE.shacl_validation_failed, ontologyToValidate));
		throw new SAREFPipelineException(response.get("500"));
		}
		} catch (IOException \| InterruptedException e) {
		logError(getMessage(Clause_9_4_6_Checker.MESSAGE.ioexception, ontologyToValidate));
		}
		}

		private boolean matchHashMap(String key, String value, HashMap<String, String> sG) {
		if (sG != null){
		if (sG.containsKey(key)){
		if ((sG.get(key)).equals(value)){
		return true;
		}
		}
		}
		return false;
		}
		}
		No newline at end of file

src/main/java/fr/mines_stetienne/ci/saref/managers/ShaclValidationManager.java

+41 −34

Original line number	Diff line number	Diff line
		@@ -36,25 +36,26 @@ import java.nio.charset.StandardCharsets;
		import java.nio.file.Paths;
		import java.util.Map;
		import java.util.List;
		import java.util.HashMap;
		import java.util.ArrayList;
		//import java.util.HashMap;
		import org.apache.commons.lang3.StringEscapeUtils;
		//import org.apache.commons.lang3.StringEscapeUtils;
		import com.fasterxml.jackson.databind.ObjectMapper;
		import fr.mines_stetienne.ci.saref.SAREFPipelineException;

		/* docker run -d --name SAREFvalidator -p 8080:8080 isaitb/shacl-validator # runs the generic validator with Dockerfile
		/* docker run -d --name SAREFvalidator -p 8080:8080 isaitb/shacl-validator # runs the generic validator locally given a Dockerfile.
		Docs for RDF validator: https://www.itb.ec.europa.eu/docs/guides/latest/validatingRDF/index.html
		Base docker image: https://hub.docker.com/r/isaitb/shacl-validator (generic validator)
		Use Swagger UI for testing: http://localhost:8080/shacl/swagger-ui/index.html#/{domain}/api/validate
		Use Swagger UI for testing: http://localhost:8080/shacl/swagger-ui/index.html#/{domain}/api/validate or http://193.49.174.57:4050/shacl/swagger-ui/index.html#
		Online validator: https://www.itb.ec.europa.eu/shacl/any/upload
		* */
		public class ShaclValidationManager {

		private static String VALIDATOR_URL = "http://localhost:8080/shacl/";
		private static String VALIDATOR_URL = "http://localhost:8080/shacl/"; //"http://193.49.174.57:4050/shacl/";
		private final String TEXT_TURTLE = "text/turtle";
		private final String JSON_FORMAT = "application/json"; // application/x-www-form-urlencoded
		private final String JSON_FORMAT = "application/json";
		private final String STRING_FORMAT = "STRING";

		private String ontologyToValidate; // file:///home/davidgnabasik/dev/real_saref4auto/ontology/saref4auto.ttl or URL
		private String ontologyToValidate; // can be file or URL.
		private List<String> shaclFileList;
		private List<RuleSet> ruleSetList;
		private HttpClient client;
		@@ -86,7 +87,6 @@ public class ShaclValidationManager {
		class ValidationRequest {
		private String contentToValidate; // JSON-encoded ontology
		private List<RuleSet> externalRules;
		private Map<String, Object> properties; // needed?

		public String getContentToValidate() {
		return contentToValidate;
		@@ -96,12 +96,12 @@ public class ShaclValidationManager {
		this.contentToValidate = contentToValidate;
		}

		public Map<String, Object> getProperties() {
		return properties;
		public List<RuleSet> getExternalRules() {
		return externalRules;
		}

		public void setProperties(Map<String, Object> properties) {
		this.properties = properties;
		public void setExternalRules(List<RuleSet> externalRules) {
		this.externalRules = externalRules;
		}

		// constants for now...
		@@ -114,29 +114,25 @@ public class ShaclValidationManager {
		}

		public String getReportSyntax() {
		return JSON_FORMAT;
		return TEXT_TURTLE;
		}

		public String getRdfReportSyntax() {
		return JSON_FORMAT;
		return TEXT_TURTLE;
		}

		public String getLocale() {
		return "en";
		}

		public List<RuleSet> getExternalRules() {
		return externalRules;
		// TODO: set loadImports to true to allow recursive searches.
		public boolean getLoadImports() {
		return false;
		}

		public void setExternalRules(List<RuleSet> externalRules) {
		this.externalRules = externalRules;
		}

		}

		/**
		* Open a REST interface to the SHACL validator.
		* Open a REST interface to the remote SHACL validator.
		*/
		public ShaclValidationManager(String ontologyToValidate) throws IOException {
		this.client = HttpClient.newHttpClient();
		@@ -144,6 +140,7 @@ public class ShaclValidationManager {
		this.ontologyToValidate = ontologyToValidate;
		String url = System.getenv("VALIDATOR_URL");
		if (url != null) VALIDATOR_URL = url;
		System.out.println("SHACL validation server running at "+VALIDATOR_URL);
		}

		/**
		@@ -160,7 +157,6 @@ public class ShaclValidationManager {
		} catch (IOException e) {
		throw new SAREFPipelineException("response", "Unknown host: " + VALIDATOR_URL);
		}
		//logError(getMessage(ShaclValidationManager.MESSAGE.http_exception, VALIDATOR_URL));
		}

		/**
		@@ -176,7 +172,7 @@ public class ShaclValidationManager {
		try {
		String fileStr = new String(Files.readAllBytes(Paths.get(readingFile)));
		RuleSet ruleSet = new RuleSet();
		ruleSet.setRuleSet(fileStr); //.replaceAll("[\n]","\r\n"));
		ruleSet.setRuleSet(fileStr);
		this.shaclFileList.add(readingFile);
		this.ruleSetList.add(ruleSet);
		} catch (IOException ex) {
		@@ -199,7 +195,7 @@ public class ShaclValidationManager {
		try {
		String fileStr = new String(Files.readAllBytes(Paths.get(this.ontologyToValidate)));
		this.validationRequest = new ValidationRequest();
		this.validationRequest.setContentToValidate(fileStr); // .replaceAll("[\n]","\r\n"));
		this.validationRequest.setContentToValidate(fileStr);
		this.validationRequest.setExternalRules(this.ruleSetList);
		} catch (IOException ex) {
		throw new SAREFPipelineException("ShaclValidationManager", "Unable to read ontology file: " + this.ontologyToValidate);
		@@ -219,8 +215,17 @@ public class ShaclValidationManager {
		return HttpRequest.BodyPublishers.ofString(body.toString());
		}

		// Use 500 Internal server error for failed SHACL validation => value is SHACL file name OR error message.
		private HashMap<String, String> parseHttpResponse(HttpResponse<String> response) {
		String result = response.body().substring(response.body().indexOf("sh:conforms")+13);
		result = result.substring(0,4).replaceAll("\\s+","");
		HashMap<String, String> map = new HashMap<>();
		map.put(String.valueOf(response.statusCode()), result);
		return map;
		}

		/**
		* POST JSON response format:
		* POST JSON successful response format:
		LIST OF: @prefix : <https://saref.etsi.org/saref4auto/> .
		[ rdf:type sh:ValidationReport ;
		sh:conforms true
		@@ -229,28 +234,30 @@ public class ShaclValidationManager {
		* 2. Configure the user-provided SHACL shape as a RuleSet element of the externalRules array.
		* [ "ruleSet": "JSON-encoded_SHACL_shape", "embeddingMethod":"STRING", "ruleSyntax": TEXT_TURTLE ]
		* POST request to Validate a single RDF ontology.
		* Return <HTTP status code, validation result>.
		* @throws SAREFPipelineException
		* @throws IOException
		* @throws InterruptedException
		*/
		public HttpResponse<String> ValidateOntologyWithShacl(List<String> shaclFileList) throws SAREFPipelineException, InterruptedException {
		public HashMap<String, String> ValidateOntologyWithShacl(List<String> shaclFileList) throws SAREFPipelineException, InterruptedException {
		try {
		createValidationRequest(shaclFileList);

		String jsonStr = this.objectMapper.writeValueAsString(this.validationRequest);
		String escapedJson = StringEscapeUtils.escapeJson(jsonStr);
		System.out.println(escapedJson);//<<<
		// Had to format the json manually because escapeJson() also escaped double-quotes. See validation_request_using_strings.txt
		//String escapedJson = StringEscapeUtils.escapeJson(jsonStr);
		jsonStr = jsonStr.replace("text/turtle", "text~turtle").replace("application/json", "application~json");
		jsonStr = jsonStr.replace("/","\\/").replace("\\n","\\r\\n");
		jsonStr = jsonStr.replace("text~turtle", "text/turtle").replace("application~json", "application/json");

		HttpRequest request = HttpRequest.newBuilder()
		.header("Content-Type", JSON_FORMAT)
		.uri(URI.create(VALIDATOR_URL + "any/upload")) // any/api/validate
		.POST(HttpRequest.BodyPublishers.ofString(jsonStr)) // escapedJson
		.uri(URI.create(VALIDATOR_URL + "any/api/validate"))
		.POST(HttpRequest.BodyPublishers.ofString(jsonStr))
		.build();

		HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
		System.out.println("Status code: " + response.statusCode()); //<<<
		System.out.println("\n Body: " + response.body());
		return response;
		return parseHttpResponse(response);
		} catch (IOException ex) {
		throw new SAREFPipelineException("ValidateOntologyWithShacl", "Unable to validate ontology: " + this.ontologyToValidate);
		}

src/main/resources/messages/Clause_9_4_6_Checker.properties

+1 −0

Original line number	Diff line number	Diff line
		server_unavailable=The validation server is unavailable at `%s`
		shacl_validation_failed=SHACL validation failed for `%s`
		ioexception=Error while accessing the local file system
		missing=The shacl directory should contain at least one file. This file shall conform to the pattern specification as defined in clause 9.4.6 in TS 103 673.
		one=If a shacl directory exists, it should contain at least one shapes.ttl file.