Unverified Commit 30f8d43e authored by Maxime Lefrançois's avatar Maxime Lefrançois
Browse files

website: fix missing term pages, crashes, and HTML escaping 

- term pages were generated once per hidden label across all project
  versions and projects (e.g. no core/v4.1.1/Device.html, no
  saref4ener/*/Device.html): key the visited sets by version and project
- the "primary" term of a term page was overwritten by the loop over the
  secondary terms; `terms.remove(term)` raised when the primary term is
  not defined; `desc` was undefined in the fallback branch
- NameError on anonymous inverse properties; AttributeError on IRI
  values of `schema:workExample`; owl:maxCardinality 0 and
  owl:hasValue false were dropped (Literal truthiness)
- owl:AllDifferent with owl:distinctMembers printed nothing
- links were emitted as raw `<iri>` text; authored heading ids were lost
- escape plain literals, titles, and literal values of rdfs:comment
- user-defined datatypes got the CSS class `type-unkown`
- a project with patterns raised TypeError (assignment into a tuple)
- a project without ETSI work item crashed the ontology page
- files are written as UTF-8; .htaccess Content-disposition typo
- `describe()` shared `visited` sets were never used (mutable default)
parent 0d6533e8

src/saref_pypeline/entities.py

+11 −10
Changes for src/saref_pypeline/entities.py: 11 added lines, 10 removed lines.
Original line number Diff line number Diff line
@@ -145,19 +145,21 @@ class SAREFProject: 
        if not isinstance(name, str):

            raise TypeError("Project name must be a string")



        parts = re.split("[- _4]", name.lower().strip())

        name = name.lower().strip()



        if any("pattern" in part for part in parts):

        if "pattern" in name:

            return "saref-patterns"



        for part in parts:

            if match := re.match(r"^(\w{4})$", part):

        # SAREF4ABCD, saref4abcd, s4abcd, or the bare acronym abcd

        if match := re.fullmatch(r"(?:saref ?4|s4)([a-z]{4})", name):

            return match.group(1)

        if re.fullmatch(r"[a-z]{4}", name) and name not in {"core", "etsi"}:

            return name



        if any(hint in part for (hint, part) in product(parts, {"saref", "core"})):

        if name in {"saref", "core", "saref-core", "saref_core", "saref core"}:

            return "core"



        raise ValueError("Illegal project name")

        raise ValueError(f"Illegal project name: `{name}`")



    def __str__(self):

        return self.name
@@ -442,12 +444,12 @@ class SAREFGraphDocument: 
    def _register_project_version_pattern_ontology(

        self, project_version: "SAREFProjectVersion"

    ):

        project_version.patterns.setdefault(self._name, (None, None))[0] = self

        project_version.patterns.setdefault(self._name, [None, None])[0] = self



    def _register_project_version_pattern_shacl_specification(

        self, project_version: "SAREFProjectVersion"

    ):

        project_version.patterns.setdefault(self._name, (None, None))[1] = self

        project_version.patterns.setdefault(self._name, [None, None])[1] = self



    def _register_project_version_example(self, project_version: "SAREFProjectVersion"):

        project_version.examples[self.version_iri] = self
@@ -697,8 +699,7 @@ class EntityDescription: 
                return next(

                    filter(

                        lambda x: isinstance(x, Literal)

                        and x.datatype == None

                        or x.datatype == XSD.string,

                        and (x.datatype == None or x.datatype == XSD.string),

                        labels,

                    )

                )

src/saref_pypeline/website/site_manager.py

+18 −14
Changes for src/saref_pypeline/website/site_manager.py: 18 added lines, 14 removed lines.
Original line number Diff line number Diff line
@@ -299,21 +299,25 @@ class SiteManager: 


        # html documentation

        html = docgen.document_ontology(project_version.ontology)

        if html is not None:  # None if the document is filtered out

            dest = Path(target_dir, project_version.ontology.name + ".html")

        dest.write_text(html)

            dest.write_text(html, encoding="utf-8")

            logger.debug(f"HTML page for {project_version.ontology}: {dest}")



        if project_version.examples:

            os.makedirs(os.path.join(target_dir, "example"), exist_ok=True)

        for example in project_version.examples.values():

            html = docgen.document_document(example)

            Path(self.site_dir, example.site_path + ".html").write_text(html)

            if html is not None:  # None if the document is filtered out

                Path(self.site_dir, example.site_path + ".html").write_text(

                    html, encoding="utf-8"

                )



        for doc in chain(

            [project_version.ontology],

            project_version.examples.values(),

            project_version.vocabularies.values(),

            (item for pair in project_version.patterns.values() for item in pair),

            (item for pair in project_version.patterns.values() for item in pair if item),

        ):



            os.makedirs(
@@ -357,12 +361,12 @@ class SiteManager: 
            desc = docgen.description(term, onto_graph)



            # document term in project version directory

            if desc.hidden_label in self.visited_vhlabels:

            if (project_version.ontology.version_iri, desc.hidden_label) in self.visited_vhlabels:

                continue

            self.visited_vhlabels.add(desc.hidden_label)

            self.visited_vhlabels.add((project_version.ontology.version_iri, desc.hidden_label))



            html, graph = docgen.document_vhlabel(project_version, desc.hidden_label)

            Path(pv_dir, desc.hidden_label + ".html").write_text(html)

            Path(pv_dir, desc.hidden_label + ".html").write_text(html, encoding="utf-8")

            for format, ext in VARIANTS_W_TTL:

                graph.serialize(

                    os.path.join(pv_dir, desc.hidden_label + ext),
@@ -371,20 +375,20 @@ class SiteManager: 
                )



            # document term in project directory

            if term in self.visited_phlabels:

            if (project_version.project, desc.hidden_label) in self.visited_phlabels:

                continue

            self.visited_phlabels.add(term)

            self.visited_phlabels.add((project_version.project, desc.hidden_label))



            html = docgen.document_phlabel(project_version.project, desc.hidden_label)

            Path(p_dir, desc.hidden_label + ".html").write_text(html)

            Path(p_dir, desc.hidden_label + ".html").write_text(html, encoding="utf-8")



            # document term in root directory

            if term in self.visited_hlabels:

            if desc.hidden_label in self.visited_hlabels:

                continue

            self.visited_hlabels.add(term)

            self.visited_hlabels.add(desc.hidden_label)



            html = docgen.document_hlabel(desc.hidden_label)

            Path(self.site_dir, desc.hidden_label + ".html").write_text(html)

            Path(self.site_dir, desc.hidden_label + ".html").write_text(html, encoding="utf-8")



    def generate_htaccess(self):

        htaccess_path = os.path.join(self.site_dir, ".htaccess")
@@ -394,10 +398,10 @@ class SiteManager: 
            )



            writer.write(

                'Header set Content-disposition "expr=attachement; filename=%{resp:Content-Location}" "expr=%{resp:Content-Location} =~ /.(rdf|nt|n3)$/"\n'

                'Header set Content-disposition "expr=attachment; filename=%{resp:Content-Location}" "expr=%{resp:Content-Location} =~ /\\.(rdf|nt|n3)$/"\n'

            )

            writer.write(

                'Header set Content-disposition "expr=inline; filename=%{resp:Content-Location}" "expr=%{resp:Content-Location} =~ /.+/ && !( %{resp:Content-Location} =~ /.(rdf|nt|n3)$/ )"\n\n'

                'Header set Content-disposition "expr=inline; filename=%{resp:Content-Location}" "expr=%{resp:Content-Location} =~ /.+/ && !( %{resp:Content-Location} =~ /\\.(rdf|nt|n3)$/ )"\n\n'

            )



            for project in self.pipeline.projects_fetched:

src/saref_pypeline/website/website_generator.py

+66 −60
Changes for src/saref_pypeline/website/website_generator.py: 66 added lines, 60 removed lines.
Original line number Diff line number Diff line
@@ -2,6 +2,7 @@ from concurrent.futures import ThreadPoolExecutor 
from contextvars import ContextVar

from datetime import date

from functools import cache, cached_property

from html import escape as html_escape

import os

from pathlib import Path

import logging
@@ -134,7 +135,7 @@ class HTMLEntityDescription(EntityDescription): 
            return "type-ap"

        elif OWL.Thing in self.types:

            return "type-ni"

        elif OWL.Thing in self.types or self.uri.startswith(XSD):

        elif RDFS.Datatype in self.types or self.uri.startswith(XSD):

            return "type-dt"

        else:

            return "type-unkown"
@@ -210,7 +211,7 @@ def add_to_toc(toc: html_tag, h: Tag, title="unknown"): 


    # make room for id="clause-x.y.z" attribute

    if h.get("id"):

        h.contents.insert(0, BeautifulSoup(f'<a id="{h.get("id")}"'))

        h.insert(0, Tag(name="a", attrs={"id": h.get("id")}))



    h_id = slugify(f"clause-{'.'.join([str(i) for i in id])}")

    h["id"] = h_id
@@ -286,7 +287,14 @@ class WebsiteGenerator: 
        ):

            raw(markdown(literal, extensions=["extra", "admonition"]))

        else:

            raw(literal.replace("\r", "").replace("\n", "<br/>"))

            raw(html_escape(str(literal)).replace("\r", "").replace("\n", "<br/>"))



    def _br_links(self, pairs):

        """Links separated by line breaks. `pairs` yields (text, href)."""

        for _, last, (label, href) in with_flags(list(pairs)):

            a(label, href=href)

            if not last:

                br()



    def _dt_dd(

        self,
@@ -481,7 +489,11 @@ class WebsiteGenerator: 
        toc = ul()



        with div(_class="container") as contents:

            doc_ref = f"{self.context[WK_FIELD.ONE_DD]} {self.context[WK_FIELD.DDD]} (V{self.context[WK_FIELD.mte]})"

            doc_ref = (

                f"{self.context[WK_FIELD.ONE_DD]} {self.context[WK_FIELD.DDD]} (V{self.context[WK_FIELD.mte]})"

                if self.context

                else "(not yet published)"

            )

            div(

                f"NOTE: The text in this section is extracted from ETSI TS {doc_ref} ",

                a("[0]", href="#ref-0"),
@@ -789,13 +801,14 @@ class WebsiteGenerator: 
                terms,

            )

        )

        if term in terms:

            terms.remove(term)

        for term in terms:

            self.describe(term, onto_graph, g, visited_onto_graph)

            self.describe(term, self.metadata, g, visited_metadata)

        for secondary in terms:

            self.describe(secondary, onto_graph, g, visited_onto_graph)

            self.describe(secondary, self.metadata, g, visited_metadata)



        if (term, RDFS.isDefinedBy, version_iri) in self.metadata:

        desc = self.description(term, onto_graph)

        if (term, RDFS.isDefinedBy, version_iri) in self.metadata:

            with h1() as doc_h1:

                a(desc.curie, href=term)

                text(f" — {desc.label}")
@@ -810,29 +823,20 @@ class WebsiteGenerator: 
                    if objects:

                        dt("Is defined by")

                        with dd():

                            raw(

                                "<br/>".join(

                                    [f'<a href="{o}">{o}</a>' for o in objects]

                                )

                            )

                            self._br_links((o, o) for o in objects)



                    objects = sorted(g.objects(term, DCTERMS.isReferencedBy))

                    if objects:

                        dt("Is referenced by")

                        with dd():

                            raw(

                                "<br/>".join(

                                    [f'<a href="{o}">{o}</a>' for o in objects]

                                )

                            )

                            self._br_links((o, o) for o in objects)



                    if terms:

                        dt("Secondary terms")

                        with dd():

                            raw(

                                "<br/>".join(

                                    [f'<a href="#{t.n3()}">{t.n3()}</a>' for t in terms]

                                )

                            self._br_links(

                                (d.curie, f"#{d.curie_id}")

                                for d in (self.description(t, onto_graph) for t in terms)

                            )



                    dt("Download serialization")
@@ -908,10 +912,9 @@ class WebsiteGenerator: 
                    if terms:

                        dt("Secondary terms")

                        with dd():

                            raw(

                                "<br/>".join(

                                    [f'<a href="#{t.n3()}">{t.n3()}</a>' for t in terms]

                                )

                            self._br_links(

                                (d.curie, f"#{d.curie_id}")

                                for d in (self.description(t, onto_graph) for t in terms)

                            )



                    dt("Download serialization")
@@ -1003,7 +1006,7 @@ class WebsiteGenerator: 
            )



        context = {

            "title": f"{desc.curie} — {desc.label}",

            "title": html_escape(f"{desc.curie} — {desc.label}"),

            "extensions": self.extensions,

            "yyyy": date.today().year,

            "pathToRoot": "../../",
@@ -1022,7 +1025,9 @@ class WebsiteGenerator: 


        # secondary terms (with fragment)

        terms = list(self.metadata.subjects(SKOS.hiddenLabel, Literal(hlabel)))

        terms = sorted(filter(lambda term: term.startswith(project.namespace), terms))

        pattern = re.compile(re.escape(project.namespace + hlabel) + r"(#.*)?")

        terms = sorted(filter(lambda t: pattern.fullmatch(t), terms))

        if term in terms:

            terms.remove(term)



        if version_iris := list(self.metadata.objects(term, RDFS.isDefinedBy)):
@@ -1048,11 +1053,7 @@ class WebsiteGenerator: 
                    if objects:

                        dt("Is defined by")

                        with dd():

                            raw(

                                "<br/>".join(

                                    [f'<a href="{o}">{o}</a>' for o in objects]

                                )

                            )

                            self._br_links((o, o) for o in objects)



                    objects = sorted(

                        self.metadata.objects(term, DCTERMS.isReferencedBy)
@@ -1060,16 +1061,15 @@ class WebsiteGenerator: 
                    if objects:

                        dt("Is referenced by")

                        with dd():

                            raw(

                                "<br/>".join(

                                    [f'<a href="{o}">{o}</a>' for o in objects]

                                )

                            )

                            self._br_links((o, o) for o in objects)



                    if terms:

                        dt("Secondary terms")

                        with dd():

                            raw("<br/>".join([t.n3() for t in terms]))

                            for _, last, t in with_flags(terms):

                                text(t.n3(self.nm))

                                if not last:

                                    br()



                    dt("License")

                    dd(
@@ -1095,7 +1095,10 @@ class WebsiteGenerator: 
                    if terms:

                        dt("Secondary terms")

                        with dd():

                            raw("<br/>".join([t.n3() for t in terms]))

                            for _, last, t in with_flags(terms):

                                text(t.n3(self.nm))

                                if not last:

                                    br()



                    dt("License")

                    dd(
@@ -1112,8 +1115,8 @@ class WebsiteGenerator: 


        with div(_class="container") as contents:

            version_iris = set()

            for term in chain([term], terms):

                version_iris.update(self.metadata.objects(term, RDFS.isDefinedBy))

            for t in chain([term], terms):

                version_iris.update(self.metadata.objects(t, RDFS.isDefinedBy))

            ontologies = sorted(

                [self.pipeline.graph_documents[iri] for iri in version_iris],

                key=lambda x: x.version,
@@ -1217,7 +1220,7 @@ class WebsiteGenerator: 
                        describe_func(ontology, t)



        context = {

            "title": f"{project.path}/{hlabel}",

            "title": hlabel,

            "extensions": self.extensions,

            "yyyy": date.today().year,

            "pathToRoot": "./",
@@ -1282,8 +1285,7 @@ class WebsiteGenerator: 
            value = value or next(

                filter(

                    lambda x: isinstance(x, Literal)

                    and x.datatype == None

                    or x.datatype == XSD.string,

                    and (x.datatype == None or x.datatype == XSD.string),

                    g.objects(document.declaration_iri, property),

                ),

                None,
@@ -1291,6 +1293,9 @@ class WebsiteGenerator: 
        if not value or not value.strip():

            return

        if not toc:

            if isinstance(value, Literal):

                self._format_literal(value)

            else:

                raw(value)

            return

        soup = BeautifulSoup(value, "html.parser")
@@ -1427,9 +1432,10 @@ class WebsiteGenerator: 
        t: IdentifiedNode,

        source: Graph,

        target: Graph,

        visited: Set | None = set(),

    ) -> Graph:

        visited = visited or set()

        visited: Set | None = None,

    ) -> None:

        if visited is None:

            visited = set()

        if t in visited:

            return

        visited.add(t)
@@ -1565,12 +1571,6 @@ class WebsiteGenerator: 
                        OWL.members,

                        self.reference_property,

                    ),

                    (

                        OWL.AllDifferent,

                        "All Different Individuals",

                        OWL.members,

                        self.reference_individual_or_literal,

                    ),

                    (

                        OWL.AllDifferent,

                        "All Different Individuals",
@@ -1583,6 +1583,9 @@ class WebsiteGenerator: 
                            dt(label)

                            ctx = dd()

                            ctx.__enter__()

                        # owl:AllDifferent may use owl:distinctMembers or owl:members

                        l = next(g.objects(ga, p), None)

                        if l is None:

                            l = next(g.objects(ga, OWL.members), RDF.nil)

                        while l != RDF.nil:

                            first = next(g.objects(l, RDF.first), None)
@@ -2146,7 +2149,8 @@ class WebsiteGenerator: 
                            self.reference_individual_or_literal(document, o)



                # individual value

                if value := next(g.objects(entity, OWL.hasValue), None):

                value = next(g.objects(entity, OWL.hasValue), None)

                if value is not None:

                    raw(" <span class='logic'>value</span> ")

                    self.reference_individual_or_literal(document, value)
@@ -2156,7 +2160,8 @@ class WebsiteGenerator: 
                    (OWL.minCardinality, "min"),

                    (OWL.maxCardinality, "max"),

                ]:

                    if cardinality := next(g.objects(entity, p), None):

                    cardinality = next(g.objects(entity, p), None)

                    if cardinality is not None:

                        raw(f" <span class='logic'>{connector}</span> {cardinality} ")



                for p, connector in [
@@ -2164,7 +2169,8 @@ class WebsiteGenerator: 
                    (OWL.minQualifiedCardinality, "min"),

                    (OWL.maxQualifiedCardinality, "max"),

                ]:

                    if cardinality := next(g.objects(entity, p), None):

                    cardinality = next(g.objects(entity, p), None)

                    if cardinality is not None:

                        raw(f" <span class='logic'>{connector}</span> {cardinality} ")

                        if c := next(g.objects(entity, OWL.onClass), None):

                            self.reference_class(document, c, False)
@@ -2256,7 +2262,7 @@ class WebsiteGenerator: 
            description.sup()



        elif isinstance(entity, Literal):

            raw(entity.n3(self.nm))

            text(entity.n3(self.nm))



        else:  # fallback

            raw(
@@ -2270,9 +2276,9 @@ class WebsiteGenerator: 
            description.href_title(document.namespace)

            # description.sup()



        elif (entity, OWL.inverseOf, None) in graph:

        elif (entity, OWL.inverseOf, None) in g:

            for first, last, inverse in with_flags(

                graph.objects(entity, OWL.inverseOf)

                g.objects(entity, OWL.inverseOf)

            ):

                if first:

                    raw("(<span class='logic'>inverse of</span> ")