Skip to content
Snippets Groups Projects
Commit 1a81703f authored by David Gnabasik's avatar David Gnabasik
Browse files

Worked on ShaclValidationManager.

parent 5ac7c0b1
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 150 additions and 171 deletions
pom.xml
+ 6
− 0
View file @ 1a81703f
......@@ -126,6 +126,12 @@
<version>2.10.1</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.10.1</version>
</dependency>
<dependency>
<!-- see Porcelain https://git-scm.com/book/uz/v2/Appendix-B%3A-Embedding-Git-in-your-Applications-JGit -->
<groupId>org.eclipse.jgit</groupId>
......
src/main/java/fr/mines_stetienne/ci/saref/checkers/Clause_9_4_6_Checker.java
+ 54
− 80
View file @ 1a81703f
......@@ -25,95 +25,33 @@
*/
package fr.mines_stetienne.ci.saref.checkers;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.net.http.HttpResponse;
import java.nio.file.Files;
import java.util.stream.Collectors;
import org.apache.jena.rdf.model.Model;
import org.apache.jena.rdf.model.ResIterator;
import org.apache.jena.rdf.model.Resource;
import org.apache.jena.vocabulary.DCTypes;
import org.apache.jena.vocabulary.RDF;
import fr.mines_stetienne.ci.saref.SAREF;
import fr.mines_stetienne.ci.saref.SAREFPipelineException;
import fr.mines_stetienne.ci.saref.entities.SAREFExample;
//import fr.mines_stetienne.ci.saref.entities.SAREFVersion;
import fr.mines_stetienne.ci.saref.entities.SAREFVersionName;
import fr.mines_stetienne.ci.saref.managers.RepositoryManager;
import fr.mines_stetienne.ci.saref.vocabs.SHACL;
import fr.mines_stetienne.ci.saref.managers.ShaclValidationManager;
import fr.mines_stetienne.ci.saref.utils.Languages;
import org.semanticweb.owlapi.model.OWLOntology;
/**
* Checks TS 103 673 Clause 9.4.6: Conformance to reference ontology patterns.
* The ontology in the ontology document of a SAREF project version shall conform to the SHACL specification of each
* reference ontology pattern defined in this SAREF project version.
*/
public class Clause_9_4_6_Checker extends AbstractShaclChecker {
public class Clause_9_4_6_Checker extends AbstractClauseChecker {
private enum MESSAGE implements MessageResource {
dataset, conformsTo1, conformsToNot, conformsToNot2, conformsTo2, conformsTo3,
title1, title2, title3, abstract1, description1, description2,
license, ioexception
private enum MESSAGE {
one, missing, ioexception, server_unavailable
}
private final SAREFExample example = null;
public Clause_9_4_6_Checker(RepositoryManager repositoryManager) //, SAREFExample shacl
throws SAREFPipelineException {
super(repositoryManager, Clause_9_4_6_Checker.class); // , "example " + example.getName());
//this.example = shacl;
}
@Override
protected Model getModel() {
Model model = example.getModel();
final String defaultVersion = "v0.0.1";
final String regex = "^" + example.getIRI().replace(defaultVersion, SAREF.REGEX_VERSION_NUMBER) + "$";
int onto = 0;
boolean found = false;
for (ResIterator it = model.listSubjectsWithProperty(RDF.type, DCTypes.Dataset); it.hasNext();) {
Resource r = it.next();
if(versionName.equals(SAREFVersionName.DEFAULT)) {
if(r.getURI().matches(regex)) {
found = true;
}
} else {
if(r.getURI().equals( example.getIRI() )) {
found = true;
}
}
onto++;
}
if (onto != 1 || !found) {
final String msg;
if(versionName.equals(SAREFVersionName.DEFAULT)) {
msg = getMessage(MESSAGE.dataset, example.getIRI()).replace("v0.0.1", "<<some SAREF version name>>");
} else {
msg = getMessage(MESSAGE.dataset, example.getIRI());
}
logError(msg);
}
return model;
}
protected final void updateShapeModel() {
if(!versionName.equals(SAREFVersionName.DEFAULT)) {
shapeModel.add(MESSAGE.conformsToNot.asResource(), SHACL.pattern, exactly(version.getIRI()));
add(MESSAGE.conformsTo1, version.getIRI());
}
add(MESSAGE.conformsTo2);
add(MESSAGE.conformsTo3);
add(MESSAGE.title1);
add(MESSAGE.title2);
add(MESSAGE.title3);
add(MESSAGE.abstract1);
add(MESSAGE.description1);
add(MESSAGE.description2);
add(exactly(SAREF.LICENSE), MESSAGE.license);
if(versionName.equals(SAREFVersionName.DEFAULT)) {
remove(MESSAGE.conformsTo1);
}
public Clause_9_4_6_Checker(RepositoryManager repositoryManager) throws SAREFPipelineException {
super(repositoryManager, Clause_9_4_6_Checker.class);
}
/**
......@@ -124,13 +62,49 @@ public class Clause_9_4_6_Checker extends AbstractShaclChecker {
reference ontology pattern defined in this other SAREF project version.
This method calls the generic docker image that runs a specific shacl files.
* */
protected final void validateOntologyPerShacl(String ontologyToValidate, String shaclFile) {
//<<< process every file
@Override
public void checkClause() throws SAREFPipelineException {
OWLOntology ontology = pipeline.getOntologyManager().loadOntology(version, errorLogger);
if (ontology == null) {
return;
}
File dir = new File(repository.getDirectory(), "ontology");
File file = new File(dir, repository.getProject().getOntologyFileName(Languages.TEXT_TURTLE));
String ontologyToValidate = file.getAbsolutePath();
// First get the available directories. If none, not a problem.
File dir2 = new File(repository.getDirectory(), "patterns");
String[] directoryList = new String[]{};
try {
HttpResponse<String> response = new ShaclValidationManager(ontologyToValidate).validateOntologyWithShacl(shaclFile);
} catch (IOException | InterruptedException e) {
logError(getMessage(Clause_9_4_6_Checker.MESSAGE.ioexception));
String directories = Files.walk(dir2.toPath()).filter(p -> {
try {
return p.toFile().isDirectory() && !Files.isSameFile(dir2.toPath(), p);
} catch(IOException ex) {
return false;
}
}).map(p -> p.toString()).collect(Collectors.joining(", "));
if (directories.isEmpty()) {
return; // no SHACL directories found.
}
directoryList = directories.split(",");
} catch (IOException e) {
logError(getMessage(Clause_9_4_6_Checker.MESSAGE.ioexception), e);
}
}
ShaclValidationManager.PingValidationServer();
Map<String, HttpResponse<String>> responseMap = new HashMap<String, HttpResponse<String>>();
// process every shapes.ttl file in the sub-directories of the patterns directory.
for(int i = 0; i < directoryList.length; i++) {
String shaclFileName = directoryList[i] + "/shapes.ttl"; // "file://" +
try {
HttpResponse<String> response = new ShaclValidationManager(ontologyToValidate).validateOntologyWithShacl(shaclFileName);
System.out.println(response); //<<<
responseMap.put(shaclFileName, response); //<<< parse map for errors
} catch (IOException | InterruptedException e) {
logError(getMessage(Clause_9_4_6_Checker.MESSAGE.ioexception, shaclFileName));
}
}
//System.out.println(responseMap); //<<<
}
}
\ No newline at end of file
src/main/java/fr/mines_stetienne/ci/saref/checkers/Clause_9_4_Checker.java
+ 1
− 0
View file @ 1a81703f
......@@ -29,6 +29,7 @@ import java.io.File;
import fr.mines_stetienne.ci.saref.SAREFPipelineException;
import fr.mines_stetienne.ci.saref.managers.RepositoryManager;
import fr.mines_stetienne.ci.saref.utils.Languages;
/**
* Checks TS 103 673 Clause 9.4: The Ontology specification
......
src/main/java/fr/mines_stetienne/ci/saref/checkers/Clause_9_8_1_Checker.java
+ 2
− 2
View file @ 1a81703f
......@@ -66,7 +66,7 @@ public class Clause_9_8_1_Checker extends AbstractClauseChecker {
}
// Step 4. Fetch the list of turtle files in the vocabularies directory.
File[] fileList = null;
File[] fileList = new File[]{};
try {
if (Files.walk(dir.toPath(), 1).filter(p -> !p.toFile().isFile() && !p.toFile().getName().startsWith("."))
.count() != 1) {
......@@ -89,7 +89,7 @@ public class Clause_9_8_1_Checker extends AbstractClauseChecker {
return;
}
// <<< Step 6. Match each imports statements to the presence of each file using the last tag (e.g. "properties" +".ttl")
// TODO: Step 6. Match each imports statements to the presence of each file using the last tag (e.g. "properties" +".ttl")
// Ignore turtle files which do not have a corresponding imports statements.
/*Iterator<OWLImportsDeclaration> itr = this.importDeclarations.iterator();
while(itr.hasNext()) { // <https://saref.etsi.org/saref4auto/v2.1.1/properties/>
......
src/main/java/fr/mines_stetienne/ci/saref/managers/ShaclValidationManager.java
+ 85
− 87
View file @ 1a81703f
......@@ -26,86 +26,101 @@
package fr.mines_stetienne.ci.saref.managers;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.net.URLEncoder;
import java.net.http.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import java.util.Base64;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.fasterxml.jackson.databind.ObjectMapper;
//import fr.mines_stetienne.ci.saref.SAREFErrorLogger;
import fr.mines_stetienne.ci.saref.SAREFPipelineException;
/*
/* docker run -d --name SAREFvalidator -p 8080:8080 isaitb/shacl-validator # runs the generic validator.
Docs for RDF validator: https://www.itb.ec.europa.eu/docs/guides/latest/validatingRDF/index.html
Base docker image: https://hub.docker.com/r/isaitb/shacl-validator
Base docker image: https://hub.docker.com/r/isaitb/shacl-validator (generic validator)
Use Swagger UI for testing: http://localhost:8080/shacl/swagger-ui/index.html#/{domain}/api/validate
BASE64 Encoder/Decoder: https://www.base64encode.org/
docker run -d --name SAREFvalidator -p 8080:8080 local/ontologyvalidator # this runs the general validator.
* */
public class ShaclValidationManager {
private final static String BASE_URL = "http://localhost:8080/shacl/"; // <<<
private static String VALIDATOR_URL = "http://localhost:8080/shacl/";
private String ontologyToValidate; // file:///home/davidgnabasik/dev/real_saref4auto/ontology/saref4auto.ttl or URL
private final HttpClient client;
private final ObjectMapper objectMapper;
private HttpClient client;
private ObjectMapper objectMapper;
//private String shaclRequestBody =
// "{ \"contentToValidate\": \"ONTOLOGY_AS_BASE64_STRING\", \"embeddingMethod\": \"BASE64\", \"contentSyntax\": \"text/turtle\", \"reportSyntax\": \"application/json\", \"locale\": \"en\", \"rdfReportSyntax\": \"application/json\", \"externalRules\": [ { \"ruleSet\": \"SHACL_AS_BASE64_STRING\", \"embeddingMethod\": \"BASE64\", \"ruleSyntax\": \"text/turtle\" } ] }";
private static String encode(Object obj) {
private static String encodeBody(Object obj) {
return URLEncoder.encode(obj.toString(), StandardCharsets.UTF_8);
}
private enum MESSAGE {
private enum MESSAGE { // each of these accepts a single string parameter.
http_exception, ioexception
}
// The full list of available tags: https://www.itb.ec.europa.eu/docs/guides/latest/validatingRDF/
// Replace default values for {contentToValidate, validationType}
private String shaclRequestBody = "{ "+ // generic validator option
"'contentToValidate': 'RDF_CONTENT_AS_BASE64_ENCODED_STRING', "+ // entire content of ontology file (or URI)
"'embeddingMethod': 'BASE64', "+ // {STRING,URL,BASE64}
"'contentSyntax': 'text/turtle', "+
"'validationType': 'string', "+
"'reportSyntax': 'application/json', "+
"'locale': 'en', "+
"'rdfReportSyntax': 'application/json' "+
"}";
/* "contentToValidate": "",
"contentSyntax": "text/turtle",
"embeddingMethod": "BASE64",
"validationType": "Clause_9_4_3_1",
"reportSyntax": "application/json",
"locale": "en" */
JsonObject jsonObject = new JsonParser().parse(shaclRequestBody).getAsJsonObject();
/**
* Open a REST interface to the SHACL validator.
*/
public ShaclValidationManager(String ontologyToValidate) throws IOException {
client = HttpClient.newHttpClient();
objectMapper = new ObjectMapper();
this.ontologyToValidate = ontologyToValidate; //<<< test for ontology file presence.
this.ontologyToValidate = ontologyToValidate;
String url = System.getenv("VALIDATOR_URL");
if (url != null) VALIDATOR_URL = url;
}
/**
* Is the validation server accessible?
* @throws SAREFPipelineException
*/
public static void PingValidationServer() throws SAREFPipelineException {
try {
URL url = new URL(VALIDATOR_URL);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestMethod("GET");
connection.connect();
connection.getResponseCode();
} catch (IOException e) {
throw new SAREFPipelineException("response", "Unknown host: " + VALIDATOR_URL);
}
//logError(getMessage(ShaclValidationManager.MESSAGE.http_exception, VALIDATOR_URL));
}
/**
* Preferred validation format is BASE64. Read file; return single BASE64-encoded string.
* @throws SAREFPipelineException
*/
public static String FormatAsBase64(String ontologyFile) throws SAREFPipelineException {
try {
String value = Files.readString(Path.of(ontologyFile)); // 2Gb file size limit.
return Base64.getUrlEncoder().encodeToString(value.getBytes());
} catch (IOException ex) {
throw new SAREFPipelineException("FormatAsBase64", "Unable to format file as base64: " + ontologyFile);
}
}
public static HttpRequest.BodyPublisher ofForm(Map<String, String> data) {
public static HttpRequest.BodyPublisher ofForm(Map<String, Object> data) {
StringBuilder body = new StringBuilder();
for (String dataKey : data.keySet()) {
for (Object dataKey : data.keySet()) {
if (body.length() > 0) {
body.append("&");
}
body.append(encode(dataKey))
body.append(encodeBody(dataKey))
.append("=")
.append(encode(data.get(dataKey)));
.append(encodeBody(data.get(dataKey)));
}
return HttpRequest.BodyPublishers.ofString(body.toString());
}
/**
* POST request to Validate a single RDF instance: {domain}/api/validate/{requestBody}
<<< This method uses the validator configured to read the Clause_9_4_3_1_3.ttl, Clause_9_4_4_2.ttl, Clause_9_6_3.ttl shacl files.
* Response: HTTP code 200 for successful validation. POST JSON return report format:
* POST JSON response format:
{
"date": "2024-04-05T10:18:24.197+0000",
"result": "SUCCESS",
......@@ -121,57 +136,40 @@ public class ShaclValidationManager {
"reports": {},
"name": "SHACL Validation"
}
*/
public HttpResponse<String> validateOntologyWithShacl(String shaclFile)
throws IOException, InterruptedException {
Map<String, String> data = new HashMap<>();
data = objectMapper.readValue(shaclRequestBody, HashMap.class);
data.put("contentToValidate", ontologyToValidate);
//data.put("validationType", validationType);
System.out.println("Map is: "+data);//<<<
/* "contentToValidate": "RDF_CONTENT_AS_BASE64_ENCODED_STRING",
"contentSyntax": "text/turtle",
"embeddingMethod": "BASE64",
"validationType": "Clause_9_4_3_1",
"reportSyntax": "application/json",
"locale": "en" */
HttpRequest request = HttpRequest.newBuilder()
.header("Content-Type", "text/turtle") // application/x-www-form-urlencoded
.uri(URI.create(BASE_URL + "/api/validate" ))
.POST(ofForm(data))
.build();
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
System.out.println("Status code: " + response.statusCode());
System.out.println("\n Body: " + response.body());
return response;
}
/**
* If shaclDomain is empty, return info for all defined domains. http://localhost:8080/shacl/api/info
* Not needed since a configured validator is not used -- the Clause_9_*.ttl files are augmented (parameterized) so they are only partially useful.
* Generic validator option: 1. Replace {contentToValidate} value with the ontology to validate encoded in BASE64. (This can aos be done with URLs.)
* 2. Configure the user-provided SHACL shape as a RuleSet element of the externalRules array. The content of each RuleSet is as follows:
* [ "ruleSet": "BASE64-encoded_SHACL_shape", "embeddingMethod":"BASE64", "ruleSyntax": "text/turtle" ] We run one SHACL shape at a time.
* POST request to Validate a single RDF instance.
* @throws IOException
* @throws InterruptedException
*/
public HttpResponse<String> getShaclDomainInfo() throws IOException, InterruptedException {
Map<String, String> data = new HashMap<>();
data = objectMapper.readValue(shaclRequestBody, HashMap.class);
String url = BASE_URL + "shacl/api/info";
HttpRequest request = HttpRequest.newBuilder()
.header("Accept", "text/turtle")
.header("Content-Type", "text/turtle")
.uri(URI.create(url))
.GET()
.build();
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
System.out.println("Status code: " + response.statusCode());
System.out.println("\n Body: " + response.body());
return response;
public HttpResponse<String> validateOntologyWithShacl(String shaclFile) throws SAREFPipelineException, InterruptedException {
try {
String ontologyBase64String = FormatAsBase64(this.ontologyToValidate);
String shaclBase64String = FormatAsBase64(shaclFile);
// populate shaclRequestBody json struct to map (no TypeReference)
String shaclRequestBody = "{ \"contentToValidate\": \"" + ontologyBase64String +
"\", \"embeddingMethod\": \"BASE64\", \"contentSyntax\": \"text/turtle\", \"reportSyntax\": \"application/json\", \"locale\": \"en\", \"rdfReportSyntax\": \"application/json\", \"externalRules\": [ { \"ruleSet\": \"" +
shaclBase64String + "\", \"embeddingMethod\": \"BASE64\", \"ruleSyntax\": \"text/turtle\" } ] }";
byte[] mapData = shaclRequestBody.getBytes();
Map<String,Object> myMap = objectMapper.readValue(mapData, HashMap.class);
// now update JSON data
//myMap.put("contentToValidate", ontologyBase64String);
//myMap.put("ruleSet", shaclBase64String);
HttpRequest request = HttpRequest.newBuilder()
.header("Content-Type", "text/turtle") // application/x-www-form-urlencoded
.uri(URI.create(VALIDATOR_URL + "any/upload"))
.POST(ofForm(myMap))
.build();
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
System.out.println("Status code: " + response.statusCode()); //<<<
System.out.println("\n Body: " + response.body());
return response;
} catch (IOException ex) {
throw new SAREFPipelineException("validateOntologyWithShacl", "Unable to process SHACL file: " + shaclFile);
}
}
}
src/main/resources/messages/ShaclValidationManager.properties
+ 2
− 2
View file @ 1a81703f
http_exception=Error while making an HTTP request.
ioexception=Error while accessing the local file system.
http_exception=Error while making an HTTP request to `%s`
ioexception=Error while accessing the local file system
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment