Unverified Commit c1b492f9 authored by Maxime Lefrançois's avatar Maxime Lefrançois
Browse files

ldpy pilot: three modules in Linked-Data Python 

- check/TS103673_Clause_9_4_3_1_Checker.ldpy: SHACL shapes as g{ } graphs
  with interpolated f-strings instead of Turtle text built with f-strings
- pipeline.ldpy: generate_metadata writes METADATA with +{ } and tests
  declarations with m{ }
- website/website_generator.ldpy: describe_class and reference_class
  read OWL class expressions with m{ } patterns
- dependency linked-data-python>=0.6.5; import hook in __init__.py;
  .ldpy files shipped as package data

Outputs (check report, metadata.ttl, website) identical to main.
parent b3bcbee6

README.md

+10 −0
Changes for README.md: 10 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -111,6 +111,16 @@ The project is deduced from the name of the project directory (e.g., `saref4abcd 
Command `check` exits with code 1 if some error is reported, so it can be used in continuous integration.





## Linked-Data Python pilot (branch `ldpy-pilot`)



Three modules are written in [Linked-Data Python](https://github.com/linked-data-python/ldpy) (`.ldpy`: Python with IRIs, prefixed names, Turtle graphs and graph patterns as expressions). They are transpiled at import time by the hook installed in `saref_pypeline/__init__.py`:



* `check/TS103673_Clause_9_4_3_1_Checker.ldpy`: the SHACL shapes built from the project version are `g{ … }` graphs with interpolated f-strings, instead of Turtle text assembled with f-strings (and escaped quotes), parsed at run time.

* `pipeline.ldpy`: `generate_metadata` writes the METADATA graph with `+{ {term} rdfs:seeAlso {iri} ; … }` and tests declarations with `m{ {term} a owl:Class }(graph)`.

* `website/website_generator.ldpy`: `describe_class` and `reference_class` (OWL class expressions) read the graph with `m{ {entity} owl:onProperty ?p }(g).first()` instead of `next(g.objects(entity, OWL.onProperty), None)`.



The outputs (check report, metadata.ttl, generated website) are identical to those of `main`.



## About



The SAREF pypeline is part of the SAREF Development Framework. The following projects contributed to the SAREF Pipeline:

pyproject.toml

+4 −0
Changes for pyproject.toml: 4 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -26,6 +26,7 @@ dependencies = [ 
  "beautifulsoup4",

  "markdown",

  "pyyaml",

  "linked-data-python>=0.6.5",

]



[project.urls]
@@ -74,6 +75,9 @@ all = [ 
]



[tool.setuptools.package-data]

"saref_pypeline" = ["*.ldpy"]

"saref_pypeline.check" = ["*.ldpy"]

"saref_pypeline.website" = ["*.ldpy"]

"saref_pypeline.check.resources" = ["**/*"]

"saref_pypeline.ts.resources" = ["**/*"]

"saref_pypeline.website.resources" = ["**/*"]

src/saref_pypeline/__init__.py

+5 −0
Changes for src/saref_pypeline/__init__.py: 5 added lines, 0 removed lines.
Original line number Diff line number Diff line 
import ldpy



# modules written in Linked-Data Python (`.ldpy`) are transpiled at import time

ldpy.install()



from saref_pypeline.pipeline import SAREFPipeline

src/saref_pypeline/check/TS103673_Clause_9_4_3_1_Checker.py→src/saref_pypeline/check/TS103673_Clause_9_4_3_1_Checker.ldpy

+37 −45
Changes for src/saref_pypeline/check/TS103673_Clause_9_4_3_1_Checker.ldpy: 37 added lines, 45 removed lines.
Original line number Diff line number Diff line 
@prefix sh: <http://www.w3.org/ns/shacl#> .

@prefix owl: <http://www.w3.org/2002/07/owl#> .

@prefix sarefsh: <https://saref.etsi.org/shape#> .



from saref_pypeline.check.BaseGraphDocumentSHACLChecker import BaseGraphDocumentSHACLChecker

from rdflib import Graph, RDF, OWL

from rdflib import Graph

from saref_pypeline.entities import SAREFGraphDocument, SAREFGraphDocumentType



from typing import TYPE_CHECKING
@@ -21,65 +25,53 @@ class Checker(BaseGraphDocumentSHACLChecker): 
        # document name, the Ontology IRI, and the Ontology Version IRI

        doc_name = self.document.type.display_name

        version_iri_name = self.document.type.version_iri_name

        shapes_graph += Graph(namespace_manager=shapes_graph.namespace_manager).parse(

            data=f"""

@prefix sh: <http://www.w3.org/ns/shacl#> .

@prefix sarefsh: <https://saref.etsi.org/shape#> .



sarefsh:versionInfo sh:pattern "^{self.version}$" ;

  sh:message "The {doc_name} shall contain exactly one metadata `owl:versionInfo`, with value `\\"{self.version}\\"`" .



sarefsh:versionIRI sh:pattern "^{self.document.version_iri}$" ;

  sh:message "The {doc_name} shall contain exactly one metadata `owl:versionIRI`, with value `<{self.document.version_iri}>` (the {version_iri_name})" .



sarefsh:preferredNamespacePrefix sh:pattern "^{self.project.prefix}$" ;

  sh:message "The {doc_name} shall contain exactly one metadata `vann:preferredNamespacePrefix`, with value `\\"{self.project.prefix}\\"`" .

        version = str(self.version)

        version_iri = str(self.document.version_iri)

        prefix = self.project.prefix

        namespace = self.project.namespace



sarefsh:preferredNamespaceUri sh:pattern "^{self.project.namespace}$" ;

  sh:message "The {doc_name} shall contain exactly one metadata `vann:preferredNamespaceUri`, with value `\\"{self.project.namespace}\\"`" .

""",

            format="turtle",

        )

        shapes_graph += g{

            sarefsh:versionInfo sh:pattern f"^{version}$" ;

                sh:message f"The {doc_name} shall contain exactly one metadata `owl:versionInfo`, with value `\"{version}\"`" .

            sarefsh:versionIRI sh:pattern f"^{version_iri}$" ;

                sh:message f"The {doc_name} shall contain exactly one metadata `owl:versionIRI`, with value `<{version_iri}>` (the {version_iri_name})" .

            sarefsh:preferredNamespacePrefix sh:pattern f"^{prefix}$" ;

                sh:message f"The {doc_name} shall contain exactly one metadata `vann:preferredNamespacePrefix`, with value `\"{prefix}\"`" .

            sarefsh:preferredNamespaceUri sh:pattern f"^{namespace}$" ;

                sh:message f"The {doc_name} shall contain exactly one metadata `vann:preferredNamespaceUri`, with value `\"{namespace}\"`" .

        }



        prior_version = self.pipeline.get_prior_version(self.project, self.version)

        if prior_version:

            prior_version_iri = str(self.document.version_iri).replace(

                f"{self.project.namespace}{self.version}/",

                f"{self.project.namespace}{prior_version.version}/",

                1,

            )

        if not prior_version:

            shape = f"""

            shapes_graph += g{

                sarefsh:priorVersion sh:maxCount 0 ;

    sh:message "There exists no prior version to {self.version}, so the {doc_name} shall not contain a metadata `owl:priorVersion`" .

"""

        elif self.document.type == SAREFGraphDocumentType.ONTOLOGY:

            shape = f"""

                    sh:message f"There exists no prior version to {version}, so the {doc_name} shall not contain a metadata `owl:priorVersion`" .

            }

            return



        prior_version_iri = version_iri.replace(

            f"{namespace}{version}/", f"{namespace}{prior_version.version}/", 1

        )

        if self.document.type == SAREFGraphDocumentType.ONTOLOGY:

            shapes_graph += g{

                sarefsh:priorVersion sh:minCount 1 ;

                    sh:maxCount 1 ;

    sh:pattern "^{prior_version_iri}$" ;

    sh:message "There exists a prior version {prior_version.version}, so the {doc_name} shall contain exactly one metadata `owl:priorVersion`, with value `<{prior_version_iri}>`" .

"""

                    sh:pattern f"^{prior_version_iri}$" ;

                    sh:message f"There exists a prior version {prior_version.version}, so the {doc_name} shall contain exactly one metadata `owl:priorVersion`, with value `<{prior_version_iri}>`" .

            }

        else:

            # the document may not exist in the prior version

            shape = f"""

            shapes_graph += g{

                sarefsh:priorVersion sh:maxCount 1 ;

    sh:pattern "^{prior_version_iri}$" ;

    sh:message "If the {doc_name} contains a metadata `owl:priorVersion`, then it shall contain exactly one, with value `<{prior_version_iri}>`" .

"""

        shapes_graph += Graph(namespace_manager=shapes_graph.namespace_manager).parse(

            data=f"""

@prefix sh: <http://www.w3.org/ns/shacl#> .

@prefix sarefsh: <https://saref.etsi.org/shape#> .

{shape}""",

            format="turtle",

        )

                    sh:pattern f"^{prior_version_iri}$" ;

                    sh:message f"If the {doc_name} contains a metadata `owl:priorVersion`, then it shall contain exactly one, with value `<{prior_version_iri}>`" .

            }



    @filter_document

    @filter_clause

    def check_clause(self):

        graph = self.pipeline.dataset.graph(self.document.version_iri)

        ontologies = list(graph.subjects(RDF.type, OWL.Ontology))

        ontologies = list(m{ ?o a owl:Ontology }(graph))

        declaration_iri = self.document.declaration_iri

        if len(ontologies) != 1 or ontologies[0] != declaration_iri:

            self.logger.error(

src/saref_pypeline/pipeline.py→src/saref_pypeline/pipeline.ldpy

+22 −16
Changes for src/saref_pypeline/pipeline.ldpy: 22 added lines, 16 removed lines.
Original line number Diff line number Diff line 
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .

@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .

@prefix owl: <http://www.w3.org/2002/07/owl#> .

@prefix skos: <http://www.w3.org/2004/02/skos/core#> .

@prefix dcterms: <http://purl.org/dc/terms/> .



from functools import cache

import os

import shutil
@@ -1105,6 +1111,7 @@ class SAREFPipeline: 
        dataset = self.dataset

        graph = dataset.graph(version_iri)

        metadata = dataset.graph(METADATA)

        @graph metadata

        ns = self.dataset.namespace_manager

        visited = set()

        for term in chain(graph.subjects(), graph.predicates(), graph.objects()):
@@ -1114,22 +1121,21 @@ class SAREFPipeline: 
                continue

            visited.add(term)



            metadata.add((term, RDFS.seeAlso, ontology_version_iri))

            +{ {term} rdfs:seeAlso {ontology_version_iri} }

            # add metadata if the term is used or subject of this version

            if term.startswith(BASE) and not PATTERN_SAREF_GRAPHS.match(term):

                hidden_label = re.match(r"(.*)/([^/#]*)(#[^#/]*)?", term).group(2)

                metadata.add((term, SKOS.hiddenLabel, Literal(hidden_label)))



                metadata.add((version_iri, DCTERMS.references, term))

                metadata.add((term, DCTERMS.isReferencedBy, version_iri))

                +{ {term} skos:hiddenLabel {hidden_label} ;

                          dcterms:isReferencedBy {version_iri} .

                   {version_iri} dcterms:references {term} }

                # documents of a project version share the project namespace:

                # skip terms only typed in an imported document (e.g., another vocabulary)

                if term.startswith(namespace) and not (

                    (term, RDF.type, None) not in graph

                    and (term, RDF.type, None) in graph_with_imports

                ):

                    metadata.add((version_iri, DCTERMS.subject, term))

                    metadata.add((term, RDFS.isDefinedBy, version_iri))

                    +{ {version_iri} dcterms:subject {term} .

                       {term} rdfs:isDefinedBy {version_iri} }



            # skip if the term is declared in a well-known namespace

            if any(term.startswith(ns) for ns in [OWL, RDF, RDFS, XSD]):
@@ -1151,9 +1157,9 @@ class SAREFPipeline: 
                continue



            # the term is declared as a named individual

            if (term, RDF.type, OWL.NamedIndividual) in graph_with_imports:

                graph_with_imports.remove((term, RDF.type, OWL.NamedIndividual))

                graph_with_imports.add((term, RDF.type, OWL.Thing))

            if m{ {term} a owl:NamedIndividual }(graph_with_imports):

                -{ {term} a owl:NamedIndividual }(graph_with_imports)

                +{ {term} a owl:Thing }(graph_with_imports)

                continue



            # skip if the term is a value of ontology metadata
@@ -1168,20 +1174,20 @@ class SAREFPipeline: 


            # the term is declared as an instance of something declared as a class

            if any(

                (typ, RDF.type, OWL.Class) in graph_with_imports

                for typ in graph.objects(subject=term, predicate=RDF.type)

                m{ {typ} a owl:Class }(graph_with_imports)

                for typ in m{ {term} a ?typ }(graph)

            ):

                graph_with_imports.add((term, RDF.type, OWL.Thing))

                +{ {term} a owl:Thing }(graph_with_imports)

                continue



            # the term is used as a subject or object of an ObjectProperty

            if any(

                (op, RDF.type, OWL.ObjectProperty) in graph_with_imports

                m{ {op} a owl:ObjectProperty }(graph_with_imports)

                for op in chain(

                    graph.predicates(subject=term), graph.predicates(object=term)

                )

            ):

                graph_with_imports.add((term, RDF.type, OWL.Thing))

                +{ {term} a owl:Thing }(graph_with_imports)

                continue



            # the term is object of some well known predicates
@@ -1200,7 +1206,7 @@ class SAREFPipeline: 
            logger.warning(

                f"{document} declares {term.n3(ns)} but this is not declared as an ontology, class, property, or individual in the graph with imports. Assuming it's an individual."

            )

            graph_with_imports.add((term, RDF.type, OWL.Thing))

            +{ {term} a owl:Thing }(graph_with_imports)



    def fetch_saref_patterns(self):

        basename = os.path.basename(os.path.abspath(self.directory))