ts: clause structure for website-style sources, crash fixes (b3bcbee6) · Commits · SAREF / saref-pypeline

README.md

+9 −3

Changes for README.md: 9 added lines, 3 removed lines.
Original line number	Diff line number	Diff line
		@@ -135,9 +135,15 @@ The content of this repository is licensed under the terms of the BSD-3-Clause L

		## TODO

		Known limitations:

		* [] ts generation: missing general axioms
		Known limitations (audit of 2026-10-10):

		* ts generation: general axioms, `owl:hasKey`, `owl:disjointUnionOf`, datatype declarations and ontology header metadata are not documented in the ontology annex; vocabularies, patterns and examples are not documented.
		* ts generation: markdown images `![]()` need `<figure>` HTML, markdown tables cannot carry captions, definition lists, footnotes, blockquotes and nested ordered lists are rendered poorly; `{{field}}` placeholders in page headers and footers of the skeleton are not replaced.
		* ts generation: documentation sources written for the website (headings starting at `###`) get default clause headings `4 <NAME> ontology and semantics`, `4.1 Introduction and overview`, `4.2 <NAME>`, `4.3 Instantiating <NAME>`; write the headings in `abstract.md`, `description.md` and `examples.md` to control them.
		* check: clause 9.4.6 does not check vocabularies against the SHACL specifications of patterns; `owl:priorVersion` is derived from the `develop-v*` branches, not from the versions published on the portal.
		* website: no pages are generated for vocabularies and patterns, although they are linked from the ontology page.
		* `tests/` is out of date (imports `saref_pypeline.dataset`, `BranchType.PRERELEASE`, `SAREFPipeline(mode=...)`) and does not run.
		* on exit, Python 3.13 sometimes prints `Error in sys.excepthook: / Original exception was:` after a `ts` run; the output and the exit code are not affected.

		## For maintainers

src/saref_pypeline/ts/ts_generator.py

+136 −54

Changes for src/saref_pypeline/ts/ts_generator.py: 136 added lines, 54 removed lines.
Original line number	Diff line number	Diff line
		@@ -3,6 +3,7 @@ from itertools import chain
		import importlib.resources as resources
		import os
		import re
		import shutil
		from pathlib import Path
		from functools import cache
		from typing import Callable, List
		@@ -11,7 +12,7 @@ from datetime import datetime
		from functools import cache
		import logging

		from bs4 import BeautifulSoup, Tag, NavigableString, PageElement
		from bs4 import BeautifulSoup, Comment, Tag, NavigableString, PageElement
		from docx import Document
		from docx.section import Section
		from docx.shared import Cm
		@@ -221,6 +222,29 @@ class TSGenerator:
		self.project_version = project_version
		self.project = project_version.project
		self.context = self.pipeline.context(project_version)
		if not self.context:
		logger.warning(
		f"No ETSI work item found for {project_version}: the TS is generated as a draft with placeholder document number TS 103 XXX."
		)
		self.context = {
		WK_FIELD.WK_ID: None,
		WK_FIELD.Short_Title: self.project.name,
		WK_FIELD.ONE_DD: "103",
		WK_FIELD.DDD: "XXX",
		WK_FIELD.PART: None,
		WK_FIELD.mte: str(project_version.version)[1:],
		WK_FIELD.yyyy: None,
		WK_FIELD.mm: None,
		WK_FIELD.Title: "SmartM2M;",
		WK_FIELD.Title_Part: f"Extension to SAREF;",
		WK_FIELD.Title_Sub_Part: f"{self.project.name}",
		WK_FIELD.Title_Release: None,
		WK_FIELD.Workitem: "DTS/SmartM2M-103XXX",
		WK_FIELD.Keywords: "ontology, SAREF, Semantic",
		WK_FIELD.TB_type: "ETSI Technical Committee",
		WK_FIELD.short_techbody: "SmartM2M",
		WK_FIELD.long_techbody: "Smart Machine-to-Machine communications",
		}
		self.ts_dir = os.path.join(pipeline.target_dir, NAME_TS)

		self.nm = NamespaceManager(Graph(), bind_namespaces="none")
		@@ -262,11 +286,8 @@ class TSGenerator:
		document.save(dest)
		logger.info(f"TS for {self.project_version} generated at {dest}")

		try:
		os.system(f'cmd.exe /C start "{self.ts_dir}/{document_name}"')
		except Exception as ex:
		logger.warning(f'Error while opening file "{self.ts_dir}/{document_name}": {ex}')
		pass
		if shutil.which("cmd.exe"): # WSL: open the document in Word
		os.system(f'cmd.exe /C start "" "{dest}"')

		@staticmethod
		def _add_gridspan(cell: _Cell, val: int):
		@@ -699,7 +720,7 @@ class TSGenerator:
		for prev in reversed(all_block_items[:idx]):
		if prev.style.name in [P_STYLE.Heading_8, P_STYLE.Heading_9]:
		# Match something like "Annex A\tsomething"
		match = re.match(r"^Annex ([A-Z+])", prev.text)
		match = re.match(r"^Annex ([A-Z]+)", prev.text)
		if match:
		previous_annex_id = match.group(1)
		break
		@@ -778,6 +799,8 @@ class TSGenerator:
		def replace_fields(self):
		"""we assume run fields, and paragraph fields"""
		for paragraph in self.document.paragraphs:
		if paragraph.style is not None and paragraph.style.name == P_STYLE.PL:
		continue # code: braces are not fields
		in_field = False
		field_parts = []
		original_paragraph = paragraph
		@@ -796,7 +819,10 @@ class TSGenerator:
		f"cannot embed field in field: {run.text}"
		)
		if not in_field and text == "}}":
		raise Exception(f"no field to close: {run.text}")
		paragraph, index = self.replace_with_text(
		text, paragraph, run, index
		)
		continue
		if text == "":
		continue
		elif text == "{{":
		@@ -902,15 +928,18 @@ class TSGenerator:
		)
		return

		# split normative and non normative
		# split normative and non normative, at the heading "Informative references"
		# (whatever its level: documentation sources written for the website use h2 or h3)
		informative_soup = BeautifulSoup()
		normative_h2 = normative_soup.find("h2")
		if normative_h2:
		informative_h2 = normative_h2.find_next("h2")
		headings = normative_soup.find_all(self.HEADING_RE)
		informative_h2 = next(
		(h for h in headings if re.search("informative", h.get_text(), re.I)), None
		)
		if informative_h2:
		for el in informative_h2.find_next_siblings():
		el.extract()
		informative_soup.append(el)
		for h2 in normative_soup.find_all(re.compile("h1\|h2")):
		for h2 in headings:
		h2.extract()

		self.insert_references("Normative references", normative_soup)
		@@ -926,6 +955,8 @@ class TSGenerator:
		for li in soup.find_all("li"):
		li["data-docx-pstyle"] = P_STYLE.EX
		a = li.find("a")
		if a is None:
		continue
		if isinstance(a.next_sibling, NavigableString):
		a.next_sibling.replace_with("\t" + a.next_sibling.string.lstrip(" "))
		else:
		@@ -965,7 +996,7 @@ class TSGenerator:
		return

		for li in soup.find_all("li"):
		li.string = re.sub(r" : ", "\t", li.string, 1)
		li.string = re.sub(r" : ", "\t", li.get_text(), count=1)
		li["data-docx-pstyle"] = P_STYLE.EW

		self.insert_soup(soup)
		@@ -987,7 +1018,7 @@ class TSGenerator:
		for li in soup.find_all("li"):
		for desc in li.descendants:
		if isinstance(desc, NavigableString):
		new_text, n = re.subn(r"\s*:\s+", "\t", str(desc), 1)
		new_text, n = re.subn(r"\s*:\s+", "\t", str(desc), count=1)
		if n: # replaced once
		desc.replace_with(new_text)
		break
		@@ -1031,6 +1062,11 @@ class TSGenerator:
		date = datetime.strptime(
		str(yyyy) + "-" + str(mm).zfill(2), "%Y-%m"
		)
		if i + 1 >= len(rows):
		logger.warning(
		"The History table of the skeleton has too few rows for all prior versions."
		)
		break
		rows[i + 1].cells[0].paragraphs[0].text = f"V{mte}"
		rows[i + 1].cells[1].paragraphs[0].text = date.strftime("%b %Y")
		rows[i + 1].cells[2].paragraphs[0].text = "Publication"
		@@ -1045,15 +1081,24 @@ class TSGenerator:
		self.insert_description1()

		def insert_description1(self):
		name = self.project.name
		self.delete_section("User defined")
		self.insert_soup_for_file("abstract")
		self.insert_soup_for_file("description")
		self.insert_soup_for_file("examples", mandatory=False)
		self.insert_soup_for_file(
		"abstract",
		headings=[
		(1, f"{name} ontology and semantics"),
		(2, "Introduction and overview"),
		],
		)
		self.insert_soup_for_file("description", headings=[(2, name)])
		self.insert_soup_for_file(
		"examples", mandatory=False, headings=[(2, f"Instantiating {name}")]
		)
		self._is_appendix = True
		self.insert_soup_for_file("annexes", mandatory=False)
		self.describe_ontology()

		def insert_soup_for_file(self, file: str, mandatory=True):
		def insert_soup_for_file(self, file: str, mandatory=True, headings=None):
		soup = self.get_soup(file)
		if not soup:
		if mandatory:
		@@ -1063,8 +1108,44 @@ class TSGenerator:
		style=C_STYLE.Guidance,
		)
		else:
		if headings:
		self.normalize_headings(soup, headings)
		self.insert_soup(soup)

		HEADING_RE = re.compile(r"^h[1-6]$")

		def normalize_headings(self, soup: BeautifulSoup, headings) -> None:
		"""Give a documentation source the clause structure expected in the TS.

		`headings` lists the (level, title) headings this source is expected to
		start with, e.g. [(1, "SAREF4ABCD ontology and semantics"), (2, "Introduction")].
		Documentation sources written for the website often start at a deeper
		level (the page title provides the upper levels): the missing headings are
		added, and the existing headings are shifted so that the content nests
		below them. A source that already has them is left unchanged.
		"""
		existing = soup.find_all(self.HEADING_RE)
		min_level = min((int(h.name[1]) for h in existing), default=None)
		required_max = max(level for level, _ in headings)
		if min_level is not None and min_level > required_max + 1:
		delta = min_level - (required_max + 1)
		for h in existing:
		h.name = f"h{min(6, int(h.name[1]) + delta)}"
		min_level = required_max + 1
		added = []
		for level, title in headings:
		if min_level is None or min_level > level:
		h = soup.new_tag(f"h{level}")
		h.string = title
		added.append(h)
		for h in reversed(added):
		soup.insert(0, h)
		if added:
		logger.info(
		f"Documentation source has no heading of level <= {required_max}: added "
		+ ", ".join(f"`{h.get_text()}`" for h in added)
		)

		# ---------------------------------------------------------------------
		# Methods for inserting soup
		# ---------------------------------------------------------------------
		@@ -1126,7 +1207,7 @@ class TSGenerator:
		self.insert_soup_children(el)

		elif isinstance(el, NavigableString):
		if not el or el == "\n":
		if isinstance(el, Comment) or not el or el == "\n":
		return
		self.ensure_cursor_paragraph()
		self.new_run(el)
		@@ -1167,7 +1248,7 @@ class TSGenerator:

		def insert_soup_div(self, el: Tag) -> None:
		# only a few special cases of admonition are supported
		if "admonition" in el.get("class"):
		if "admonition" in (el.get("class") or []):
		title = el.find("p", class_="admonition-title")
		body_parts = [c for c in el.find_all(recursive=False) if c is not title]
		if len(body_parts) >= 1 and body_parts[0].name == "p":
		@@ -1185,6 +1266,7 @@ class TSGenerator:
		style = P_STYLE.EX

		self.pstyling_funcs.append(make_pstyling_func(style))
		if child is not None:
		self.insert_soup(child)
		self.pstyling_funcs.pop()

		@@ -1226,7 +1308,7 @@ class TSGenerator:
		if add_tab:
		for child in el.children:
		if isinstance(child, NavigableString) and ":" in child.text:
		child.replace_with(re.sub(r":\w*", ":\t", child.text, 1))
		child.replace_with(re.sub(r":\s*", ":\t", child.text, count=1))
		break

		self.insert_soup_children(el)
		@@ -1418,11 +1500,13 @@ class TSGenerator:
		section.page_width = first_section.page_width
		section.page_height = first_section.page_height

		def insert_soup_img(self, el: Tag, text_width) -> None:
		def insert_soup_img(self, el: Tag, text_width=None) -> None:
		"""
		Insert an image from <img> into the Word document.
		Supports data-docx-width as percentage (e.g., '80%') or cm (e.g., '15cm').
		"""
		if text_width is None:
		text_width = self.get_maximal_textwidth()
		src = el.get("src")
		if not src:
		self.new_paragraph()
		@@ -1558,9 +1642,11 @@ class TSGenerator:
		if el.caption:
		self.insert_soup_caption(el.caption)
		rows = el.find_all("tr")
		nb_cols = len(rows[0].find_all(["td", "th"]))
		if not rows:
		return
		nb_cols = sum(
		int(c.get("colspan", "1")) for c in rows[0].find_all(["td", "th"])
		)
		self.new_table(
		rows=0, cols=nb_cols, alignment=WD_TABLE_ALIGNMENT.CENTER, autofit=True
		)
		@@ -1619,7 +1705,7 @@ class TSGenerator:
		if colspan > 1:
		self._add_gridspan(cells[cell_idx], colspan)
		for i in range(colspan - 1):
		to_remove = cells[cell_idx + 1]._tc
		to_remove = cells[cell_idx + i + 1]._tc
		to_remove.getparent().remove(to_remove)
		cell_idx += 1
		continue
		@@ -1811,18 +1897,18 @@ class TSGenerator:
		def describe_annotation_property(self, ap: URIRef):
		ontology = self.project_version.ontology
		g = self.dataset.graph(ontology.version_iri_with_imports)
		self._dt_dd(
		self._EW(
		"super-properties",
		g.objects(ap, RDFS.subPropertyOf),
		self.reference_property,
		)
		self._dt_dd(
		self._EW(
		"sub-properties",
		g.subjects(RDFS.subPropertyOf, ap),
		self.reference_property,
		)
		self._dt_dd("domain", g.objects(ap, RDFS.domain), self.reference_class)
		self._dt_dd(
		self._EW("domain", g.objects(ap, RDFS.domain), self.reference_class)
		self._EW(
		"domain includes",
		g.objects(ap, SCHEMA.domainIncludes),
		self.reference_class,
		@@ -1834,8 +1920,8 @@ class TSGenerator:
		else:
		self.reference_data_range(thing)

		self._dt_dd("range", g.objects(ap, RDFS.range), reference_class_or_data_range)
		self._dt_dd(
		self._EW("range", g.objects(ap, RDFS.range), reference_class_or_data_range)
		self._EW(
		"range includes",
		g.objects(ap, SCHEMA.rangeIncludes),
		reference_class_or_data_range,
		@@ -1915,9 +2001,9 @@ class TSGenerator:
		def label(other):
		labels = {OWL.FunctionalProperty: "functional"}
		if other in labels:
		labels[other]
		self.new_run(labels[other])
		else:
		other.n3(self.nm)
		self.new_run(other.n3(self.nm))

		self._EW(
		"characteristics",
		@@ -2009,9 +2095,9 @@ class TSGenerator:
		OWL.TransitiveProperty: "transitive",
		}
		if other in labels:
		labels[other]
		self.new_run(labels[other])
		else:
		other.n3(self.nm)
		self.new_run(other.n3(self.nm))

		self._EW(
		"characteristics",
		@@ -2125,8 +2211,8 @@ class TSGenerator:
		) in [
		(OWL.intersectionOf, None, " ", "and", " ", None, True),
		(OWL.unionOf, None, " ", "or", " ", None, True),
		(OWL.disjointUnionOf, " ", "disjoint union of", " ", " , ", None, True),
		(OWL.oneOf, " ", "one of", " { ", ", ", " } ", False),
		(OWL.disjointUnionOf, "disjoint union of ", "", ",", " ", None, True),
		(OWL.oneOf, "one of { ", "", ",", " ", " }", False),
		]:
		if list := next(g.objects(entity, property), None):
		if first:
		@@ -2186,10 +2272,11 @@ class TSGenerator:
		if is_op:
		self.reference_class(o, False)
		else:
		self.reference_individual_or_literal(o)
		self.reference_data_range(o)

		# individual value
		if value := next(g.objects(entity, OWL.hasValue), None):
		value = next(g.objects(entity, OWL.hasValue), None)
		if value is not None:
		self.new_run(" ")
		self.new_run("value", bold=True)
		self.new_run(" ")
		@@ -2201,7 +2288,8 @@ class TSGenerator:
		(OWL.minCardinality, "min"),
		(OWL.maxCardinality, "max"),
		]:
		if cardinality := next(g.objects(entity, p), None):
		cardinality = next(g.objects(entity, p), None)
		if cardinality is not None:
		self.new_run(" ")
		self.new_run(connector, bold=True)
		self.new_run(f" {cardinality} ")
		@@ -2211,7 +2299,8 @@ class TSGenerator:
		(OWL.minQualifiedCardinality, "min"),
		(OWL.maxQualifiedCardinality, "max"),
		]:
		if cardinality := next(g.objects(entity, p), None):
		cardinality = next(g.objects(entity, p), None)
		if cardinality is not None:
		self.new_run(" ")
		self.new_run(connector, bold=True)
		self.new_run(f" {cardinality} ")
		@@ -2239,19 +2328,12 @@ class TSGenerator:
		self.new_run("not", bold=True)
		self.new_run(" ")
		self.reference_data_range(d)
		self.new_run(")")

		# data range intersection, union, and literal enumeration
		for property, first, connector, last, is_ni in [
		(
		OWL.intersectionOf,
		"(",
		" <span class='logic'>and</span> ",
		")",
		False,
		),
		(OWL.unionOf, "(", " <span class='logic'>or</span> ", ")", False),
		(OWL.oneOf, "<span class='logic'>one of</span> { ", ", ", " }", True),
		(OWL.intersectionOf, "(", " and ", ")", False),
		(OWL.unionOf, "(", " or ", ")", False),
		(OWL.oneOf, "one of { ", ", ", " }", True),
		]:
		if l := next(g.objects(entity, property), None):
		self.new_run(first)
		@@ -2322,12 +2404,12 @@ class TSGenerator:
		description = self.description(entity, g)
		self.new_run(description.curie)

		elif (entity, OWL.inverseOf, None) in graph:
		elif (entity, OWL.inverseOf, None) in g:
		for first, last, inverse in with_flags(
		graph.objects(entity, OWL.inverseOf)
		g.objects(entity, OWL.inverseOf)
		):
		if first:
		self.new_run("()")
		self.new_run("(")
		self.new_run("inverse of", bold=True)
		self.new_run(" ")
		self.reference_property(inverse)