diff --git a/pom.xml b/pom.xml
index 6f0b0c4e6e383d517ef0a299c407435691b9b9f6..9b75880ce02ad79e5f6a73e162b3bdbdf015bcdc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,6 +126,12 @@
<version>2.10.1</version>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.10.1</version>
+ </dependency>
+
<dependency>
<!-- see Porcelain https://git-scm.com/book/uz/v2/Appendix-B%3A-Embedding-Git-in-your-Applications-JGit -->
<groupId>org.eclipse.jgit</groupId>
diff --git a/src/main/java/fr/mines_stetienne/ci/saref/checkers/Clause_9_4_6_Checker.java b/src/main/java/fr/mines_stetienne/ci/saref/checkers/Clause_9_4_6_Checker.java
index 27c8748821b304407ec202a9e27798ade91d9fde..53900596c66fda841e2ec17a303d87c51b873e55 100644
--- a/src/main/java/fr/mines_stetienne/ci/saref/checkers/Clause_9_4_6_Checker.java
+++ b/src/main/java/fr/mines_stetienne/ci/saref/checkers/Clause_9_4_6_Checker.java
@@ -25,95 +25,33 @@
*/
package fr.mines_stetienne.ci.saref.checkers;
+import java.io.File;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
import java.net.http.HttpResponse;
+import java.nio.file.Files;
+import java.util.stream.Collectors;
-import org.apache.jena.rdf.model.Model;
-import org.apache.jena.rdf.model.ResIterator;
-import org.apache.jena.rdf.model.Resource;
-import org.apache.jena.vocabulary.DCTypes;
-import org.apache.jena.vocabulary.RDF;
-
-import fr.mines_stetienne.ci.saref.SAREF;
import fr.mines_stetienne.ci.saref.SAREFPipelineException;
-import fr.mines_stetienne.ci.saref.entities.SAREFExample;
-//import fr.mines_stetienne.ci.saref.entities.SAREFVersion;
-import fr.mines_stetienne.ci.saref.entities.SAREFVersionName;
import fr.mines_stetienne.ci.saref.managers.RepositoryManager;
-import fr.mines_stetienne.ci.saref.vocabs.SHACL;
import fr.mines_stetienne.ci.saref.managers.ShaclValidationManager;
+import fr.mines_stetienne.ci.saref.utils.Languages;
+import org.semanticweb.owlapi.model.OWLOntology;
/**
* Checks TS 103 673 Clause 9.4.6: Conformance to reference ontology patterns.
* The ontology in the ontology document of a SAREF project version shall conform to the SHACL specification of each
* reference ontology pattern defined in this SAREF project version.
*/
-public class Clause_9_4_6_Checker extends AbstractShaclChecker {
+public class Clause_9_4_6_Checker extends AbstractClauseChecker {
- private enum MESSAGE implements MessageResource {
- dataset, conformsTo1, conformsToNot, conformsToNot2, conformsTo2, conformsTo3,
- title1, title2, title3, abstract1, description1, description2,
- license, ioexception
+ private enum MESSAGE {
+ one, missing, ioexception, server_unavailable
}
- private final SAREFExample example = null;
-
- public Clause_9_4_6_Checker(RepositoryManager repositoryManager) //, SAREFExample shacl
- throws SAREFPipelineException {
- super(repositoryManager, Clause_9_4_6_Checker.class); // , "example " + example.getName());
- //this.example = shacl;
- }
-
- @Override
- protected Model getModel() {
- Model model = example.getModel();
- final String defaultVersion = "v0.0.1";
- final String regex = "^" + example.getIRI().replace(defaultVersion, SAREF.REGEX_VERSION_NUMBER) + "$";
- int onto = 0;
- boolean found = false;
- for (ResIterator it = model.listSubjectsWithProperty(RDF.type, DCTypes.Dataset); it.hasNext();) {
- Resource r = it.next();
- if(versionName.equals(SAREFVersionName.DEFAULT)) {
- if(r.getURI().matches(regex)) {
- found = true;
- }
- } else {
- if(r.getURI().equals( example.getIRI() )) {
- found = true;
- }
- }
- onto++;
- }
- if (onto != 1 || !found) {
- final String msg;
- if(versionName.equals(SAREFVersionName.DEFAULT)) {
- msg = getMessage(MESSAGE.dataset, example.getIRI()).replace("v0.0.1", "<<some SAREF version name>>");
- } else {
- msg = getMessage(MESSAGE.dataset, example.getIRI());
- }
- logError(msg);
- }
- return model;
- }
-
- protected final void updateShapeModel() {
- if(!versionName.equals(SAREFVersionName.DEFAULT)) {
- shapeModel.add(MESSAGE.conformsToNot.asResource(), SHACL.pattern, exactly(version.getIRI()));
- add(MESSAGE.conformsTo1, version.getIRI());
- }
- add(MESSAGE.conformsTo2);
- add(MESSAGE.conformsTo3);
- add(MESSAGE.title1);
- add(MESSAGE.title2);
- add(MESSAGE.title3);
- add(MESSAGE.abstract1);
- add(MESSAGE.description1);
- add(MESSAGE.description2);
- add(exactly(SAREF.LICENSE), MESSAGE.license);
-
- if(versionName.equals(SAREFVersionName.DEFAULT)) {
- remove(MESSAGE.conformsTo1);
- }
+ public Clause_9_4_6_Checker(RepositoryManager repositoryManager) throws SAREFPipelineException {
+ super(repositoryManager, Clause_9_4_6_Checker.class);
}
/**
@@ -124,13 +62,49 @@ public class Clause_9_4_6_Checker extends AbstractShaclChecker {
reference ontology pattern defined in this other SAREF project version.
This method calls the generic docker image that runs a specific shacl files.
* */
- protected final void validateOntologyPerShacl(String ontologyToValidate, String shaclFile) {
- //<<< process every file
+ @Override
+ public void checkClause() throws SAREFPipelineException {
+ OWLOntology ontology = pipeline.getOntologyManager().loadOntology(version, errorLogger);
+ if (ontology == null) {
+ return;
+ }
+ File dir = new File(repository.getDirectory(), "ontology");
+ File file = new File(dir, repository.getProject().getOntologyFileName(Languages.TEXT_TURTLE));
+ String ontologyToValidate = file.getAbsolutePath();
+
+ // First get the available directories. If none, not a problem.
+ File dir2 = new File(repository.getDirectory(), "patterns");
+ String[] directoryList = new String[]{};
try {
- HttpResponse<String> response = new ShaclValidationManager(ontologyToValidate).validateOntologyWithShacl(shaclFile);
- } catch (IOException | InterruptedException e) {
- logError(getMessage(Clause_9_4_6_Checker.MESSAGE.ioexception));
+ String directories = Files.walk(dir2.toPath()).filter(p -> {
+ try {
+ return p.toFile().isDirectory() && !Files.isSameFile(dir2.toPath(), p);
+ } catch(IOException ex) {
+ return false;
+ }
+ }).map(p -> p.toString()).collect(Collectors.joining(", "));
+ if (directories.isEmpty()) {
+ return; // no SHACL directories found.
+ }
+ directoryList = directories.split(",");
+ } catch (IOException e) {
+ logError(getMessage(Clause_9_4_6_Checker.MESSAGE.ioexception), e);
}
- }
+ ShaclValidationManager.PingValidationServer();
+
+ Map<String, HttpResponse<String>> responseMap = new HashMap<String, HttpResponse<String>>();
+ // process every shapes.ttl file in the sub-directories of the patterns directory.
+ for(int i = 0; i < directoryList.length; i++) {
+ String shaclFileName = directoryList[i] + "/shapes.ttl"; // "file://" +
+ try {
+ HttpResponse<String> response = new ShaclValidationManager(ontologyToValidate).validateOntologyWithShacl(shaclFileName);
+ System.out.println(response); //<<<
+ responseMap.put(shaclFileName, response); //<<< parse map for errors
+ } catch (IOException | InterruptedException e) {
+ logError(getMessage(Clause_9_4_6_Checker.MESSAGE.ioexception, shaclFileName));
+ }
+ }
+ //System.out.println(responseMap); //<<<
+ }
}
\ No newline at end of file
diff --git a/src/main/java/fr/mines_stetienne/ci/saref/checkers/Clause_9_4_Checker.java b/src/main/java/fr/mines_stetienne/ci/saref/checkers/Clause_9_4_Checker.java
index 868cf6e5814a8037649807e3ac519b4841e9671a..c9e3232537c7a1fb966e599e1540e42255822502 100644
--- a/src/main/java/fr/mines_stetienne/ci/saref/checkers/Clause_9_4_Checker.java
+++ b/src/main/java/fr/mines_stetienne/ci/saref/checkers/Clause_9_4_Checker.java
@@ -29,6 +29,7 @@ import java.io.File;
import fr.mines_stetienne.ci.saref.SAREFPipelineException;
import fr.mines_stetienne.ci.saref.managers.RepositoryManager;
+import fr.mines_stetienne.ci.saref.utils.Languages;
/**
* Checks TS 103 673 Clause 9.4: The Ontology specification
diff --git a/src/main/java/fr/mines_stetienne/ci/saref/checkers/Clause_9_8_1_Checker.java b/src/main/java/fr/mines_stetienne/ci/saref/checkers/Clause_9_8_1_Checker.java
index 1f92aa98c8716c6bace4a94a82ae4a5e562d82ba..6cf183b99313634e50ab66687ee39eebd31054fe 100644
--- a/src/main/java/fr/mines_stetienne/ci/saref/checkers/Clause_9_8_1_Checker.java
+++ b/src/main/java/fr/mines_stetienne/ci/saref/checkers/Clause_9_8_1_Checker.java
@@ -66,7 +66,7 @@ public class Clause_9_8_1_Checker extends AbstractClauseChecker {
}
// Step 4. Fetch the list of turtle files in the vocabularies directory.
- File[] fileList = null;
+ File[] fileList = new File[]{};
try {
if (Files.walk(dir.toPath(), 1).filter(p -> !p.toFile().isFile() && !p.toFile().getName().startsWith("."))
.count() != 1) {
@@ -89,7 +89,7 @@ public class Clause_9_8_1_Checker extends AbstractClauseChecker {
return;
}
- // <<< Step 6. Match each imports statements to the presence of each file using the last tag (e.g. "properties" +".ttl")
+ // TODO: Step 6. Match each imports statements to the presence of each file using the last tag (e.g. "properties" +".ttl")
// Ignore turtle files which do not have a corresponding imports statements.
/*Iterator<OWLImportsDeclaration> itr = this.importDeclarations.iterator();
while(itr.hasNext()) { // <https://saref.etsi.org/saref4auto/v2.1.1/properties/>
diff --git a/src/main/java/fr/mines_stetienne/ci/saref/managers/ShaclValidationManager.java b/src/main/java/fr/mines_stetienne/ci/saref/managers/ShaclValidationManager.java
index 142ca7e6f66ed5d6cccc21d102a590bbcbf08392..1bb641c6a37aa766e94d1ba997f827cefb04f6a3 100644
--- a/src/main/java/fr/mines_stetienne/ci/saref/managers/ShaclValidationManager.java
+++ b/src/main/java/fr/mines_stetienne/ci/saref/managers/ShaclValidationManager.java
@@ -26,86 +26,101 @@
package fr.mines_stetienne.ci.saref.managers;
import java.io.IOException;
+import java.net.HttpURLConnection;
import java.net.URI;
+import java.net.URL;
import java.net.URLEncoder;
import java.net.http.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
+import java.util.Base64;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
import com.fasterxml.jackson.databind.ObjectMapper;
-//import fr.mines_stetienne.ci.saref.SAREFErrorLogger;
+import fr.mines_stetienne.ci.saref.SAREFPipelineException;
-/*
+/* docker run -d --name SAREFvalidator -p 8080:8080 isaitb/shacl-validator # runs the generic validator.
Docs for RDF validator: https://www.itb.ec.europa.eu/docs/guides/latest/validatingRDF/index.html
- Base docker image: https://hub.docker.com/r/isaitb/shacl-validator
+ Base docker image: https://hub.docker.com/r/isaitb/shacl-validator (generic validator)
Use Swagger UI for testing: http://localhost:8080/shacl/swagger-ui/index.html#/{domain}/api/validate
BASE64 Encoder/Decoder: https://www.base64encode.org/
- docker run -d --name SAREFvalidator -p 8080:8080 local/ontologyvalidator # this runs the general validator.
* */
public class ShaclValidationManager {
- private final static String BASE_URL = "http://localhost:8080/shacl/"; // <<<
+ private static String VALIDATOR_URL = "http://localhost:8080/shacl/";
private String ontologyToValidate; // file:///home/davidgnabasik/dev/real_saref4auto/ontology/saref4auto.ttl or URL
- private final HttpClient client;
- private final ObjectMapper objectMapper;
+ private HttpClient client;
+ private ObjectMapper objectMapper;
+
+ //private String shaclRequestBody =
+ // "{ \"contentToValidate\": \"ONTOLOGY_AS_BASE64_STRING\", \"embeddingMethod\": \"BASE64\", \"contentSyntax\": \"text/turtle\", \"reportSyntax\": \"application/json\", \"locale\": \"en\", \"rdfReportSyntax\": \"application/json\", \"externalRules\": [ { \"ruleSet\": \"SHACL_AS_BASE64_STRING\", \"embeddingMethod\": \"BASE64\", \"ruleSyntax\": \"text/turtle\" } ] }";
- private static String encode(Object obj) {
+ private static String encodeBody(Object obj) {
return URLEncoder.encode(obj.toString(), StandardCharsets.UTF_8);
}
- private enum MESSAGE {
+ private enum MESSAGE { // each of these accepts a single string parameter.
http_exception, ioexception
}
- // The full list of available tags: https://www.itb.ec.europa.eu/docs/guides/latest/validatingRDF/
- // Replace default values for {contentToValidate, validationType}
- private String shaclRequestBody = "{ "+ // generic validator option
- "'contentToValidate': 'RDF_CONTENT_AS_BASE64_ENCODED_STRING', "+ // entire content of ontology file (or URI)
- "'embeddingMethod': 'BASE64', "+ // {STRING,URL,BASE64}
- "'contentSyntax': 'text/turtle', "+
- "'validationType': 'string', "+
- "'reportSyntax': 'application/json', "+
- "'locale': 'en', "+
- "'rdfReportSyntax': 'application/json' "+
- "}";
- /* "contentToValidate": "",
- "contentSyntax": "text/turtle",
- "embeddingMethod": "BASE64",
- "validationType": "Clause_9_4_3_1",
- "reportSyntax": "application/json",
- "locale": "en" */
-
- JsonObject jsonObject = new JsonParser().parse(shaclRequestBody).getAsJsonObject();
-
/**
* Open a REST interface to the SHACL validator.
*/
public ShaclValidationManager(String ontologyToValidate) throws IOException {
client = HttpClient.newHttpClient();
objectMapper = new ObjectMapper();
- this.ontologyToValidate = ontologyToValidate; //<<< test for ontology file presence.
+ this.ontologyToValidate = ontologyToValidate;
+ String url = System.getenv("VALIDATOR_URL");
+ if (url != null) VALIDATOR_URL = url;
+ }
+
+ /**
+ * Is the validation server accessible?
+ * @throws SAREFPipelineException
+ */
+ public static void PingValidationServer() throws SAREFPipelineException {
+ try {
+ URL url = new URL(VALIDATOR_URL);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setRequestMethod("GET");
+ connection.connect();
+ connection.getResponseCode();
+ } catch (IOException e) {
+ throw new SAREFPipelineException("response", "Unknown host: " + VALIDATOR_URL);
+ }
+ //logError(getMessage(ShaclValidationManager.MESSAGE.http_exception, VALIDATOR_URL));
+ }
+
+ /**
+ * Preferred validation format is BASE64. Read file; return single BASE64-encoded string.
+ * @throws SAREFPipelineException
+ */
+ public static String FormatAsBase64(String ontologyFile) throws SAREFPipelineException {
+ try {
+ String value = Files.readString(Path.of(ontologyFile)); // 2Gb file size limit.
+ return Base64.getUrlEncoder().encodeToString(value.getBytes());
+ } catch (IOException ex) {
+ throw new SAREFPipelineException("FormatAsBase64", "Unable to format file as base64: " + ontologyFile);
+ }
}
- public static HttpRequest.BodyPublisher ofForm(Map<String, String> data) {
+ public static HttpRequest.BodyPublisher ofForm(Map<String, Object> data) {
StringBuilder body = new StringBuilder();
- for (String dataKey : data.keySet()) {
+ for (Object dataKey : data.keySet()) {
if (body.length() > 0) {
body.append("&");
}
- body.append(encode(dataKey))
+ body.append(encodeBody(dataKey))
.append("=")
- .append(encode(data.get(dataKey)));
+ .append(encodeBody(data.get(dataKey)));
}
return HttpRequest.BodyPublishers.ofString(body.toString());
}
/**
- * POST request to Validate a single RDF instance: {domain}/api/validate/{requestBody}
- <<< This method uses the validator configured to read the Clause_9_4_3_1_3.ttl, Clause_9_4_4_2.ttl, Clause_9_6_3.ttl shacl files.
- * Response: HTTP code 200 for successful validation. POST JSON return report format:
+ * POST JSON response format:
{
"date": "2024-04-05T10:18:24.197+0000",
"result": "SUCCESS",
@@ -121,57 +136,40 @@ public class ShaclValidationManager {
"reports": {},
"name": "SHACL Validation"
}
- */
- public HttpResponse<String> validateOntologyWithShacl(String shaclFile)
- throws IOException, InterruptedException {
- Map<String, String> data = new HashMap<>();
- data = objectMapper.readValue(shaclRequestBody, HashMap.class);
- data.put("contentToValidate", ontologyToValidate);
- //data.put("validationType", validationType);
- System.out.println("Map is: "+data);//<<<
- /* "contentToValidate": "RDF_CONTENT_AS_BASE64_ENCODED_STRING",
- "contentSyntax": "text/turtle",
- "embeddingMethod": "BASE64",
- "validationType": "Clause_9_4_3_1",
- "reportSyntax": "application/json",
- "locale": "en" */
-
- HttpRequest request = HttpRequest.newBuilder()
- .header("Content-Type", "text/turtle") // application/x-www-form-urlencoded
- .uri(URI.create(BASE_URL + "/api/validate" ))
- .POST(ofForm(data))
- .build();
-
- HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
- System.out.println("Status code: " + response.statusCode());
- System.out.println("\n Body: " + response.body());
- return response;
- }
-
- /**
- * If shaclDomain is empty, return info for all defined domains. http://localhost:8080/shacl/api/info
- * Not needed since a configured validator is not used -- the Clause_9_*.ttl files are augmented (parameterized) so they are only partially useful.
+ * Generic validator option: 1. Replace {contentToValidate} value with the ontology to validate encoded in BASE64. (This can aos be done with URLs.)
+ * 2. Configure the user-provided SHACL shape as a RuleSet element of the externalRules array. The content of each RuleSet is as follows:
+ * [ "ruleSet": "BASE64-encoded_SHACL_shape", "embeddingMethod":"BASE64", "ruleSyntax": "text/turtle" ] We run one SHACL shape at a time.
+ * POST request to Validate a single RDF instance.
* @throws IOException
* @throws InterruptedException
*/
- public HttpResponse<String> getShaclDomainInfo() throws IOException, InterruptedException {
- Map<String, String> data = new HashMap<>();
- data = objectMapper.readValue(shaclRequestBody, HashMap.class);
-
- String url = BASE_URL + "shacl/api/info";
-
- HttpRequest request = HttpRequest.newBuilder()
- .header("Accept", "text/turtle")
- .header("Content-Type", "text/turtle")
- .uri(URI.create(url))
- .GET()
- .build();
-
- HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
- System.out.println("Status code: " + response.statusCode());
- System.out.println("\n Body: " + response.body());
-
- return response;
+ public HttpResponse<String> validateOntologyWithShacl(String shaclFile) throws SAREFPipelineException, InterruptedException {
+ try {
+ String ontologyBase64String = FormatAsBase64(this.ontologyToValidate);
+ String shaclBase64String = FormatAsBase64(shaclFile);
+ // populate shaclRequestBody json struct to map (no TypeReference)
+ String shaclRequestBody = "{ \"contentToValidate\": \"" + ontologyBase64String +
+ "\", \"embeddingMethod\": \"BASE64\", \"contentSyntax\": \"text/turtle\", \"reportSyntax\": \"application/json\", \"locale\": \"en\", \"rdfReportSyntax\": \"application/json\", \"externalRules\": [ { \"ruleSet\": \"" +
+ shaclBase64String + "\", \"embeddingMethod\": \"BASE64\", \"ruleSyntax\": \"text/turtle\" } ] }";
+ byte[] mapData = shaclRequestBody.getBytes();
+ Map<String,Object> myMap = objectMapper.readValue(mapData, HashMap.class);
+ // now update JSON data
+ //myMap.put("contentToValidate", ontologyBase64String);
+ //myMap.put("ruleSet", shaclBase64String);
+ HttpRequest request = HttpRequest.newBuilder()
+ .header("Content-Type", "text/turtle") // application/x-www-form-urlencoded
+ .uri(URI.create(VALIDATOR_URL + "any/upload"))
+ .POST(ofForm(myMap))
+ .build();
+
+ HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
+ System.out.println("Status code: " + response.statusCode()); //<<<
+ System.out.println("\n Body: " + response.body());
+ return response;
+ } catch (IOException ex) {
+ throw new SAREFPipelineException("validateOntologyWithShacl", "Unable to process SHACL file: " + shaclFile);
+ }
}
+
}
diff --git a/src/main/resources/messages/ShaclValidationManager.properties b/src/main/resources/messages/ShaclValidationManager.properties
index 9ac8047bb515f6eb91fa0dc4bb8a91e40bd85ce2..aab8431bd940767b46e93f7a7ba1a586b1e89d34 100644
--- a/src/main/resources/messages/ShaclValidationManager.properties
+++ b/src/main/resources/messages/ShaclValidationManager.properties
@@ -1,3 +1,3 @@
-http_exception=Error while making an HTTP request.
-ioexception=Error while accessing the local file system.
+http_exception=Error while making an HTTP request to `%s`
+ioexception=Error while accessing the local file system