Unverified Commit 58c879e5 authored by Maxime Lefrançois's avatar Maxime Lefrançois
Browse files

ts: optional documentation/ts.yaml for document metadata, clauses and annexes 

- document: TS number/part, version, date, titles, work item, keywords,
  overriding the ETSI work item or the draft placeholders
- clauses: ordered documentation sources with the headings to add when
  a source has none; any extra source becomes a clause
- annexes: documentation sources (normative or informative) and the
  ontology reference (title, normativity, sections), in any order
- defaults unchanged when the file is absent
parent b3bcbee6

README.md

+37 −0
Changes for README.md: 37 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -111,6 +111,43 @@ The project is deduced from the name of the project directory (e.g., `saref4abcd 
Command `check` exits with code 1 if some error is reported, so it can be used in continuous integration.





## Technical Specification generation



`saref-dev ts` fills the ETSI TS skeleton with the documentation sources of the project (`documentation/*.md`) and an annex describing the ontology. The ETSI work item (document number, dates, titles) is read from the ETSI work programme; a project without work item is generated as a draft `TS 103 XXX`.



By default, `abstract.md` becomes clause 4 (with 4.1 *Introduction and overview*), `description.md` becomes 4.2 and `examples.md` becomes 4.3 *Instantiating <NAME>*; headings missing in the sources are added. The optional file `documentation/ts.yaml` controls the document metadata, the clauses and the annexes:



```yaml

document:                 # overrides the ETSI work item (or the draft placeholders)

  number: "103 410-11"    # TS number, with the part if any

  version: "1.1.1"

  date: "2026-10"         # publication date (yyyy-mm); leave empty for a draft

  title: "SmartM2M"

  part_title: "Extension to SAREF"

  subpart_title: "Part 11: Mechanical Systems Domain"

  short_title: "SAREF4MECH"

  workitem: "DTS/SmartM2M-103410-11"

  keywords: "ontology, SAREF, Semantic"

clauses:                  # documentation sources after clause 3, in order

  - file: abstract.md

    headings:             # headings added when the source has none at these levels

      - [1, "SAREF4MECH ontology and semantics"]

      - [2, "Introduction and overview"]

  - file: description.md

  - file: examples.md

    optional: true

  - file: mapping.md      # any other source becomes a clause

annexes:                  # in order

  - file: annexes.md      # each `#` heading starts an annex

    optional: true

    normative: false

  - ontology_reference:

      title: "Ontology Reference"

      normative: true

      sections: [Classes, Object Properties, Data Properties, Named Individuals, Annotation Properties]

  - file: bibliography.md

```



## About



The SAREF pypeline is part of the SAREF Development Framework. The following projects contributed to the SAREF Pipeline:

src/saref_pypeline/ts/ts_generator.py

+185 −13
Changes for src/saref_pypeline/ts/ts_generator.py: 185 added lines, 13 removed lines.
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ import importlib.resources as resources 
import os

import re

import shutil

import yaml

from pathlib import Path

from functools import cache

from typing import Callable, List
@@ -221,6 +222,7 @@ class TSGenerator: 
        self.dataset = pipeline.dataset

        self.project_version = project_version

        self.project = project_version.project

        self.config = self.load_config()

        self.context = self.pipeline.context(project_version)

        if not self.context:

            logger.warning(
@@ -245,6 +247,7 @@ class TSGenerator: 
                WK_FIELD.short_techbody: "SmartM2M",

                WK_FIELD.long_techbody: "Smart Machine-to-Machine communications",

            }

        self.context = self.apply_document_config(dict(self.context))

        self.ts_dir = os.path.join(pipeline.target_dir, NAME_TS)



        self.nm = NamespaceManager(Graph(), bind_namespaces="none")
@@ -265,6 +268,101 @@ class TSGenerator: 
        self.rstyling_funcs: List[Callable[[Run], None]] = []





    CONFIG_FILE = "ts.yaml"



    def load_config(self) -> dict:

        """Read the optional `documentation/ts.yaml` of the project version.



        Keys (all optional):



        document:               # overrides the ETSI work item (or the draft placeholders)

          number: "103 410-11"  # TS number, with the part if any

          version: "1.1.1"

          date: "2026-10"       # publication date (yyyy-mm); none for a draft

          title: "SmartM2M"

          part_title: "Extension to SAREF"

          subpart_title: "Part 11: Mechanical Systems Domain"

          short_title: "SAREF4MECH"

          workitem: "DTS/SmartM2M-103410-11"

          keywords: "ontology, SAREF, Semantic"

        clauses:                # documentation sources of the clauses after clause 3, in order

          - file: abstract.md   # default: abstract.md, description.md, examples.md

            headings:           # headings added when the source has none at these levels

              - [1, "SAREF4MECH ontology and semantics"]

              - [2, "Introduction and overview"]

          - file: description.md

            headings: [[2, "SAREF4MECH"]]

          - file: examples.md

            optional: true

          - file: mapping.md

        annexes:                # in order; default: annexes.md then the ontology reference

          - file: annexes.md

            optional: true

            normative: false

          - ontology_reference:

              title: "Ontology Reference"

              normative: true

              sections: [Classes, Object Properties, Data Properties, Named Individuals, Annotation Properties]

        """

        path = Path(

            self.project.directory,

            self.project_version.ontology.doc_folder,

            self.CONFIG_FILE,

        )

        if not path.exists():

            return {}

        try:

            with open(path, encoding="utf-8") as f:

                config = yaml.safe_load(f) or {}

        except Exception as e:

            logger.error(f"Cannot read {path}: {e}")

            return {}

        if not isinstance(config, dict):

            logger.error(f"{path} shall contain a mapping")

            return {}

        logger.info(f"Using TS configuration {path}")

        return config



    def apply_document_config(self, context: dict) -> dict:

        document = self.config.get("document") or {}

        if not isinstance(document, dict):

            logger.error(f"{self.CONFIG_FILE}: `document` shall be a mapping")

            return context

        if number := document.get("number"):

            match = re.fullmatch(

                r"(?:TS\s*)?(\d{3})\s*(\d{3}(?:-(\d+))?)", str(number).strip()

            )

            if not match:

                logger.error(

                    f"{self.CONFIG_FILE}: `document.number` shall look like `103 410-11`, got `{number}`"

                )

            else:

                context[WK_FIELD.ONE_DD] = match.group(1)

                context[WK_FIELD.DDD] = match.group(2)

                context[WK_FIELD.PART] = match.group(3)

        if version := document.get("version"):

            context[WK_FIELD.mte] = str(version).lstrip("vV")

        if "date" in document:

            date = document.get("date")

            match = re.fullmatch(r"(\d{4})-(\d{2})", str(date)) if date else None

            if date and not match:

                logger.error(

                    f"{self.CONFIG_FILE}: `document.date` shall look like `2026-10`, got `{date}`"

                )

            context[WK_FIELD.yyyy] = match.group(1) if match else None

            context[WK_FIELD.mm] = match.group(2) if match else None

        for key, field in [

            ("title", WK_FIELD.Title),

            ("part_title", WK_FIELD.Title_Part),

            ("subpart_title", WK_FIELD.Title_Sub_Part),

            ("short_title", WK_FIELD.Short_Title),

            ("workitem", WK_FIELD.Workitem),

            ("keywords", WK_FIELD.Keywords),

        ]:

            if key in document:

                context[field] = document[key]

        return context



    def generate_ts(self):

        """main method"""

        document = self.render_document()
@@ -1083,20 +1181,69 @@ class TSGenerator: 
    def insert_description1(self):

        name = self.project.name

        self.delete_section("User defined")

        self.insert_soup_for_file(

            "abstract",

            headings=[

                (1, f"{name} ontology and semantics"),

                (2, "Introduction and overview"),



        clauses = self.config.get("clauses") or [

            {

                "file": "abstract.md",

                "headings": [

                    [1, f"{name} ontology and semantics"],

                    [2, "Introduction and overview"],

                ],

        )

        self.insert_soup_for_file("description", headings=[(2, name)])

            },

            {"file": "description.md", "headings": [[2, name]]},

            {

                "file": "examples.md",

                "optional": True,

                "headings": [[2, f"Instantiating {name}"]],

            },

        ]

        for clause in clauses:

            if isinstance(clause, str):

                clause = {"file": clause}

            if not isinstance(clause, dict) or "file" not in clause:

                logger.error(f"{self.CONFIG_FILE}: each clause shall have a `file`")

                continue

            headings = [

                (int(level), str(title))

                for level, title in (clause.get("headings") or [])

            ]

            self.insert_soup_for_file(

            "examples", mandatory=False, headings=[(2, f"Instantiating {name}")]

                Path(clause["file"]).stem,

                mandatory=not clause.get("optional", False),

                headings=headings,

            )



        self._is_appendix = True

        self.insert_soup_for_file("annexes", mandatory=False)

        self.describe_ontology()

        annexes = self.config.get("annexes") or [

            {"file": "annexes.md", "optional": True},

            {"ontology_reference": {}},

        ]

        for annex in annexes:

            if annex == "ontology_reference":

                annex = {"ontology_reference": {}}

            elif isinstance(annex, str):

                annex = {"file": annex}

            if not isinstance(annex, dict):

                logger.error(f"{self.CONFIG_FILE}: invalid annex {annex!r}")

                continue

            if "ontology_reference" in annex:

                options = annex["ontology_reference"] or {}

                self.describe_ontology(

                    title=options.get("title", "Ontology Reference"),

                    normative=options.get("normative", True),

                    sections=options.get("sections"),

                )

            elif "file" in annex:

                self._annex_normative = bool(annex.get("normative", False))

                self.insert_soup_for_file(

                    Path(annex["file"]).stem,

                    mandatory=not annex.get("optional", False),

                )

                self._annex_normative = False

            else:

                logger.error(

                    f"{self.CONFIG_FILE}: each annex shall have a `file` or be `ontology_reference`"

                )



    def insert_soup_for_file(self, file: str, mandatory=True, headings=None):

        soup = self.get_soup(file)
@@ -1319,7 +1466,11 @@ class TSGenerator: 
        self.new_heading(p_style, increment)

        self.insert_soup_children(el)



    def insert_soup_h1(self, el: Tag, normative: bool = False) -> None:

    _annex_normative: bool = False



    def insert_soup_h1(self, el: Tag, normative: bool = None) -> None:

        if normative is None:

            normative = self._annex_normative

        if not self._is_appendix:

            self.insert_soup_h(el)

        else:
@@ -1806,10 +1957,29 @@ class TSGenerator: 
            else:

                self.new_run(last_function())



    def describe_ontology(self):

    ONTOLOGY_REFERENCE_SECTIONS = [

        "Classes",

        "Object Properties",

        "Data Properties",

        "Named Individuals",

        "Annotation Properties",

    ]



    def describe_ontology(

        self, title="Ontology Reference", normative=True, sections=None

    ):

        if sections is None:

            sections = self.ONTOLOGY_REFERENCE_SECTIONS

        for section in sections:

            if section not in self.ONTOLOGY_REFERENCE_SECTIONS:

                logger.error(

                    f"{self.CONFIG_FILE}: unknown ontology reference section `{section}`, expected one of {self.ONTOLOGY_REFERENCE_SECTIONS}"

                )

        self._enabled_sections = set(sections)

        annex_id = self.get_next_annex_number()

        self.new_paragraph(

            f"Annex {annex_id} (normative):\nOntology Reference", P_STYLE.Heading_8

            f"Annex {annex_id} ({'normative' if normative else 'informative'}):\n{title}",

            P_STYLE.Heading_8,

        )



        self.describe_entities(
@@ -1848,6 +2018,8 @@ class TSGenerator: 
    def describe_entities(

        self, title: str, entity_type: URIRef, print_function: Callable

    ) -> None:

        if title not in getattr(self, "_enabled_sections", {title}):

            return

        ontology = self.project_version.ontology

        g = self.dataset.graph(ontology.version_iri_with_imports)

        all_entities: List[URIRef] = sorted(