page for terms at project and root level (f5730aa9) · Commits · SAREF / saref-pypeline

src/saref_pypeline/docgen/site_manager.py

+57 −29

Original line number	Diff line number	Diff line
		@@ -29,6 +29,16 @@ if TYPE_CHECKING:

		MESSAGES = Localization(__file__)

		VARIANTS_WO_TTL = [
		("json-ld", ".jsonld"),
		("n3", ".n3"),
		("nt", ".nt"),
		("xml", ".rdf"),
		]
		VARIANTS_WO_TTL = list()
		VARIANTS_W_TTL = list(VARIANTS_WO_TTL)
		VARIANTS_W_TTL.append(("ttl", ".ttl"))


		class SiteManager:

		@@ -38,6 +48,9 @@ class SiteManager:
		self.ts_dir = os.path.join(pipeline.target_dir, NAME_TS)
		self.report_file_html = os.path.join(pipeline.sources_dir, NAME_REPORT_HTML)
		self.dataset = self.pipeline.dataset
		self.visited_vhlabels = set()
		self.visited_phlabels = set()
		self.visited_hlabels = set()

		@cache
		def context(self, project_version: SAREFProjectVersion) -> Dict[WK_FIELD, str]:
		@@ -136,7 +149,7 @@ class SiteManager:
		docgen = WebsiteGenerator(self, project_version)

		# html documentation
		html = docgen.render_ontology_documentation(project_version.ontology)
		html = docgen.document_ontology(project_version.ontology)
		dest = Path(target_dir, project_version.ontology.name + ".html")
		dest.write_text(html)
		logger.debug(f"HTML page for {project_version.ontology}: {dest}")
		@@ -144,7 +157,7 @@ class SiteManager:
		if project_version.examples:
		os.makedirs(os.path.join(target_dir, "example"), exist_ok=True)
		for example in project_version.examples.values():
		html = docgen.render_example_documentation(example)
		html = docgen.document_document(example)
		Path(self.site_dir, example.site_path + ".html").write_text(html)

		for doc in chain(
		@@ -166,40 +179,55 @@ class SiteManager:
		)

		# variants
		# graph = self.dataset.graph(doc.version_iri)
		# for format, ext in [
		# ("json-ld", ".jsonld"),
		# ("n3", ".n3"),
		# ("nt", ".nt"),
		# ("xml", ".rdf"),
		# ]:
		# graph.serialize(
		# os.path.join(self.site_dir, doc.site_path + ext),
		# format=format,
		# encoding="utf-8",
		# )
		graph = self.dataset.graph(doc.version_iri)
		for format, ext in VARIANTS_WO_TTL:
		graph.serialize(
		os.path.join(self.site_dir, doc.site_path + ext),
		format=format,
		encoding="utf-8",
		)

		metadata = self.pipeline.dataset.graph(METADATA)
		pv_dir = Path(self.site_dir, project_version.project.path, str(project_version.version))
		for term in metadata.subjects(RDFS.isDefinedBy, project_version.ontology.version_iri):
		html, graph = docgen.render_term_documentation(project_version, term)
		local_name = term[len(project_version.project.namespace):]
		Path(pv_dir, local_name + ".html").write_text(html)
		onto_graph = self.dataset.graph(project_version.ontology.version_iri)

		# variants
		for format, ext in [
		("ttl", ".ttl"),
		# ("json-ld", ".jsonld"),
		# ("n3", ".n3"),
		# ("nt", ".nt"),
		# ("xml", ".rdf"),
		]:
		# document term in project version directory, project directory, and root directory
		p_dir = Path(self.site_dir, project_version.project.path)
		pv_dir = Path(p_dir, str(project_version.version))
		for term in metadata.subjects(
		RDFS.isDefinedBy, project_version.ontology.version_iri
		):
		logger.debug(f"Printing html documentation for {term.n3(self.dataset.namespace_manager)}")
		desc = docgen.description(term, onto_graph)

		# document term in project version directory
		if desc.hidden_label in self.visited_vhlabels:
		continue
		self.visited_vhlabels.add(desc.hidden_label)

		html, graph = docgen.document_vhlabel(project_version, desc.hidden_label)
		Path(pv_dir, desc.hidden_label + ".html").write_text(html)
		for format, ext in VARIANTS_W_TTL:
		graph.serialize(
		os.path.join(pv_dir, local_name + ext),
		os.path.join(pv_dir, desc.hidden_label + ext),
		format=format,
		encoding="utf-8",
		)

		# document term in project directory
		if term in self.visited_phlabels:
		continue
		self.visited_phlabels.add(term)

		html = docgen.document_phlabel(project_version.project, desc.hidden_label)
		Path(p_dir, desc.hidden_label + ".html").write_text(html)

		# document term in root directory
		if term in self.visited_hlabels:
		continue
		self.visited_hlabels.add(term)

		html = docgen.document_hlabel(desc.hidden_label)
		Path(self.site_dir, desc.hidden_label + ".html").write_text(html)

		def generate_ts(self, project_version: SAREFProjectVersion):
		project = project_version.project

src/saref_pypeline/docgen/website_generator.py

+399 −122

Original line number	Diff line number	Diff line
		@@ -10,7 +10,7 @@ from markdown import markdown
		from itertools import chain
		from bs4 import BeautifulSoup
		from rdflib.term import URIRef, BNode, Literal, IdentifiedNode, Node
		from rdflib import Graph, RDF, RDFS, OWL, DC, DCTERMS, XSD
		from rdflib import SKOS, Graph, RDF, RDFS, OWL, DC, DCTERMS, XSD
		from rdflib.namespace import NamespaceManager
		from jinja2 import Environment, FileSystemLoader
		from saref_pypeline.entities import (
		@@ -47,8 +47,21 @@ class HTMLEntityDescription(EntityDescription):
		super().__init__(uri, graph, namespace_manager)

		@cached_property
		def onto_hlabel_fragment(self):
		return re.match(r"(.)/([^/#])(#[^#/]*)?", self.uri).groups()

		@property
		def local_name(self):
		return re.search(r"([^/#]+)$", self.uri).group(1)
		onto, hlabel, fragment = self.onto_hlabel_fragment
		return hlabel + fragment if fragment else hlabel

		@property
		def hidden_label(self):
		return self.onto_hlabel_fragment[1]

		@property
		def fragment(self):
		return self.onto_hlabel_fragment[2]

		def sup(self):
		if OWL.Class in self.types:
		@@ -102,8 +115,17 @@ class WebsiteGenerator:
		self.description(OWL.bottomObjectProperty, OWL_GRAPH)
		self.description(OWL.topObjectProperty, OWL_GRAPH)

		self.template = jinja2_env.get_template("html.j2")
		self.describe_funcs = [
		("Description as a Class", OWL.Class, self.describe_class),
		("Description as an Object Property", OWL.ObjectProperty, self.describe_object_property),
		("Definition as a Datatype Property", OWL.DatatypeProperty, self.describe_data_property),
		("Definition as an Annotation Property", OWL.AnnotationProperty, self.describe_annotation_property),
		("Definition as a Named Individual", OWL.Thing, self.describe_named_individual),
		]

		@cache
		def description(self, uri: URIRef, graph) -> HTMLEntityDescription:
		def description(self, uri: URIRef, graph:Graph) -> HTMLEntityDescription:
		return HTMLEntityDescription(uri, graph, self.nm)

		def _format_literal(self, literal):
		@@ -144,7 +166,7 @@ class WebsiteGenerator:
		ctx.__exit__(None, None, None)

		@filter_document("document")
		def render_ontology_documentation(self, document: SAREFGraphDocument):
		def document_ontology(self, document: SAREFGraphDocument):
		if document.project_version != self.project_version:
		raise ValueError()
		if document.type not in [
		@@ -155,7 +177,6 @@ class WebsiteGenerator:
		raise ValueError()
		logger.debug(f"Printing html documentation for {document}")

		template = jinja2_env.get_template("html.j2")
		g = self.dataset.graph(document.version_iri)

		with h1() as doc_h1:
		@@ -475,20 +496,16 @@ class WebsiteGenerator:
		"title": self.title(document),
		"pathToStatic": "../../static",
		"isOntologyDocumentation": True,
		"project": self.project,
		"project_version": self.project_version,
		"version": self.version,
		"document": document,
		"h1": doc_h1,
		"dls": dls,
		"content": contents,
		"my_tab_content": my_tab_content,
		}

		return template.render(context)
		return self.template.render(context)

		@filter_document("document")
		def render_example_documentation(self, document: SAREFGraphDocument):
		def document_document(self, document: SAREFGraphDocument):
		if (
		document.type != SAREFGraphDocumentType.EXAMPLE
		or document.project_version != self.project_version
		@@ -496,7 +513,6 @@ class WebsiteGenerator:
		raise ValueError()
		logger.debug(f"Printing html documentation for {document}")

		template = jinja2_env.get_template("html.j2")
		g = self.dataset.graph(document.version_iri)

		with h1() as doc_h1:
		@@ -615,41 +631,40 @@ class WebsiteGenerator:
		"title": self.title(document),
		"pathToStatic": "../../../static",
		"isOntologyDocumentation": False,
		"project": self.project,
		"project_version": self.project_version,
		"version": self.version,
		"document": document,
		"dls": dls,
		"h1": doc_h1,
		"content": contents,
		}

		return template.render(context)
		return self.template.render(context)

		def render_term_documentation(
		self, project_version: SAREFProjectVersion, term: URIRef
		def document_vhlabel(
		self, project_version: SAREFProjectVersion, hlabel: str
		):
		logger.debug(f"Printing html documentation for {term.n3(self.dataset.namespace_manager)}")
		ontology = project_version.ontology
		version_iri = ontology.version_iri
		onto_graph = self.dataset.graph(ontology.version_iri_with_imports)

		g = Graph(bind_namespaces="none")
		for prefix, namespace in ontology.namespaces:
		g.bind(prefix, namespace)

		onto_graph = self.dataset.graph(ontology.version_iri_with_imports)
		self.describe(term, onto_graph, g)
		self.describe(term, self.metadata, g)

		describe_funcs = [
		("Description as a Class", OWL.Class, self.describe_class),
		("Description as an Object Property", OWL.ObjectProperty, self.describe_object_property),
		("Definition as a Datatype Property", OWL.DatatypeProperty, self.describe_data_property),
		("Definition as an Annotation Property", OWL.AnnotationProperty, self.describe_annotation_property),
		("Definition as a Named Individual", OWL.Thing, self.describe_named_individual),
		]

		template = jinja2_env.get_template("html.j2")

		# primary term
		term = URIRef(project_version.project.namespace + hlabel)
		visited_onto_graph = set()
		self.describe(term, onto_graph, g, visited_onto_graph)
		visited_metadata = set()
		self.describe(term, self.metadata, g, visited_metadata)

		# secondary terms (with fragment)
		terms = list(self.metadata.subjects(SKOS.hiddenLabel, Literal(hlabel)))
		terms = sorted(filter(lambda term: (term, RDFS.isDefinedBy, version_iri) in self.metadata, terms))
		terms.remove(term)
		for term in terms:
		self.describe(term, onto_graph, g, visited_onto_graph)
		self.describe(term, self.metadata, g, visited_metadata)

		if (term, RDFS.isDefinedBy, version_iri) in self.metadata:
		desc = self.description(term, onto_graph)
		with h1() as doc_h1:
		a(desc.curie, href=term)
		@@ -675,6 +690,82 @@ class WebsiteGenerator:
		raw("<br/>".join([f'<a href="{o}">{o}</a>' for o in objects])
		)

		if terms:
		dt("Secondary terms")
		with dd():
		raw("<br/>".join([f'<a href="#{t.n3()}">{t.n3()}</a>' for t in terms])
		)

		dt("Download serialization")
		dd(
		span(
		a(
		img(
		src="https://img.shields.io/badge/Format-JSON--LD-blue.svg"
		),
		href=f"{ desc.local_name }.jsonld",
		target="_blank",
		)
		),
		span(
		a(
		img(src="https://img.shields.io/badge/Format-N3-blue.svg"),
		href=f"{ desc.local_name }.n3",
		target="_blank",
		)
		),
		span(
		a(
		img(
		src="https://img.shields.io/badge/Format-N--Triples-blue.svg"
		),
		href=f"{ desc.local_name }.nt",
		target="_blank",
		)
		),
		span(
		a(
		img(src="https://img.shields.io/badge/Format-RDF/XML-blue.svg"),
		href=f"{ desc.local_name }.rdf",
		target="_blank",
		)
		),
		span(
		a(
		img(src="https://img.shields.io/badge/Format-Turtle-blue.svg"),
		href=f"{ desc.local_name }.ttl",
		target="_blank",
		)
		),
		)

		dt("License")
		dd(
		span(
		a(
		img(
		src="https://img.shields.io/badge/License-https://forge.etsi.org/etsi--software--license-blue.svg"
		),
		href="https://forge.etsi.org/etsi-software-license",
		target="_blank",
		)
		)
		)

		else:
		# primary_term not in terms (should not happen)
		logger.warning(f"{term} is not defined in {project_version}")
		with h1() as doc_h1:
		a(hlabel, href=term)

		with div() as dls:
		with dl():
		if terms:
		dt("Secondary terms")
		with dd():
		raw("<br/>".join([f'<a href="#{t.n3()}">{t.n3()}</a>' for t in terms])
		)

		dt("Download serialization")
		dd(
		span(
		@@ -731,21 +822,24 @@ class WebsiteGenerator:
		)
		)


		with div(_class="container") as contents:

		h2("Definition")
		for t in chain([term], terms):
		desc = self.description(t, onto_graph)
		h2(f"Definition of {desc.curie}", id=desc.curie)
		with p():
		with em():
		text("Defined in ")
		a(ontology.declaration_iri, href=ontology.declaration_iri)
		a(ontology.version_iri, href=ontology.version_iri)
		text(" as:")
		q(strong(desc.comment))

		for title, typ, describe_func in describe_funcs:
		if (term, RDF.type, typ) not in g:
		for title, typ, describe_func in self.describe_funcs:
		if (t, RDF.type, typ) not in g:
		continue
		h2(title)
		describe_func(ontology, term)
		h3(title)
		describe_func(ontology, t)

		h2("Turtle")
		pre(
		@@ -757,15 +851,198 @@ class WebsiteGenerator:
		"title": f"{desc.curie} — {desc.label}",
		"pathToStatic": "../../static",
		"isOntologyDocumentation": False,
		"project": self.project,
		"project_version": self.project_version,
		"version": self.version,
		"h1": doc_h1,
		"dls": dls,
		"content": contents,
		}

		return template.render(context), g
		return self.template.render(context), g


		def document_phlabel(
		self, project: SAREFProject, hlabel: str
		):

		# primary term
		term = URIRef(project.namespace + hlabel)

		# secondary terms (with fragment)
		terms = list(self.metadata.subjects(SKOS.hiddenLabel, Literal(hlabel)))
		terms = sorted(filter(lambda term: term.startswith(project.namespace), terms))
		terms.remove(term)

		if version_iris:=list(self.metadata.objects(term, RDFS.isDefinedBy)):
		ontologies = sorted([self.pipeline.graph_documents[iri] for iri in version_iris], key=lambda x: x.version, reverse=True)
		latest_ontology = ontologies[0]
		onto_graph = self.dataset.graph(latest_ontology.version_iri_with_imports)
		desc = self.description(term, onto_graph)
		with h1() as doc_h1:
		a(desc.curie, href=term)
		text(f" — {desc.label}")

		with div() as dls:
		with dl():

		dt(f"IRI")
		dd(a(term, href=term))

		objects = sorted(self.metadata.objects(term, RDFS.isDefinedBy))
		if objects:
		dt("Is defined by")
		with dd():
		raw("<br/>".join([f'<a href="{o}">{o}</a>' for o in objects])
		)

		objects = sorted(self.metadata.objects(term, DCTERMS.isReferencedBy))
		if objects:
		dt("Is referenced by")
		with dd():
		raw("<br/>".join([f'<a href="{o}">{o}</a>' for o in objects])
		)

		if terms:
		dt("Secondary terms")
		with dd():
		raw("<br/>".join([t.n3() for t in terms])
		)

		dt("License")
		dd(
		span(
		a(
		img(
		src="https://img.shields.io/badge/License-https://forge.etsi.org/etsi--software--license-blue.svg"
		),
		href="https://forge.etsi.org/etsi-software-license",
		target="_blank",
		)
		)
		)

		else:
		# primary_term not in terms (should not happen)
		logger.warning(f"{term} is not defined in any version of {project}")
		with h1() as doc_h1:
		a(hlabel, href=term)

		with div() as dls:
		with dl():
		if terms:
		dt("Secondary terms")
		with dd():
		raw("<br/>".join([t.n3() for t in terms])
		)

		dt("License")
		dd(
		span(
		a(
		img(
		src="https://img.shields.io/badge/License-https://forge.etsi.org/etsi--software--license-blue.svg"
		),
		href="https://forge.etsi.org/etsi-software-license",
		target="_blank",
		)
		)
		)


		with div(_class="container") as contents:
		version_iris = set()
		for term in chain([term], terms):
		version_iris.update(self.metadata.objects(term, RDFS.isDefinedBy))
		ontologies = sorted([self.pipeline.graph_documents[iri] for iri in version_iris], key=lambda x: x.version, reverse=True)
		for ontology in ontologies:
		onto_graph = self.dataset.graph(ontology.version_iri_with_imports)
		h2(f"Definition in {ontology}")
		for t in chain([term], terms):
		if (t, RDFS.isDefinedBy, ontology.version_iri) not in self.metadata:
		continue
		desc = self.description(t, onto_graph)
		h2(f"{desc.curie} — {desc.label}")
		with dl():
		canonical_iri = ontology.version_iri + hlabel
		dt("canonical IRI")
		dd(a(canonical_iri, href=canonical_iri))
		dt("label")
		dd(desc.label)
		dt("definition")
		dd(desc.comment)
		for title, typ, describe_func in self.describe_funcs:
		if (t, RDF.type, typ) not in onto_graph:
		continue
		h3(title)
		describe_func(ontology, t)


		context = {
		"title": f"{project.path}/{hlabel}",
		"pathToStatic": "../static",
		"isOntologyDocumentation": False,
		"h1": doc_h1,
		"dls": dls,
		"content": contents,
		}

		return self.template.render(context)

		def document_hlabel(
		self, hlabel: str
		):

		doc_h1 = h1(hlabel)

		terms = list(self.metadata.subjects(SKOS.hiddenLabel, Literal(hlabel)))
		version_iris = set()
		for term in terms:
		version_iris.update(self.metadata.objects(term, RDFS.isDefinedBy))
		all_project_versions = { self.pipeline.graph_documents[version_iri].project_version for version_iri in version_iris}
		projects = { project_version.project for project_version in all_project_versions }

		project_versions_by_project = { project: sorted(filter(lambda x: x.project == project, all_project_versions), key=lambda x: x.version) for project in projects}

		with dl() as dls:
		dt("Is defined by")
		with dl():
		for first, last, (project, project_versions) in with_flags(project_versions_by_project.items()):
		a(str(project), href=project.path)
		text(" in ")
		for first2, last2, project_version in with_flags(project_versions):
		path = f"{project_version.project.path}/{project_version.version}"
		a(str(project_version.version), href=path)
		if not last2:
		text(", ")
		if not last:
		br()

		with div(_class="container") as contents:
		for project, project_versions in project_versions_by_project.items():
		last_project_version = project_versions[-1]
		ontology = last_project_version.ontology
		onto_graph = self.dataset.graph(ontology.version_iri_with_imports)
		h2(f"Definition in {last_project_version}")
		for t in sorted(filter(lambda x: (x, RDFS.isDefinedBy, ontology.version_iri) in self.metadata, terms)):
		desc = self.description(t, onto_graph)
		h3(f"{desc.curie} — {desc.label}", id=desc.curie)
		q(strong(desc.comment))
		for title, typ, describe_func in self.describe_funcs:
		if (t, RDF.type, typ) not in onto_graph:
		continue
		h4(title)
		describe_func(ontology, t)


		context = {
		"title": f"{project.path}/{hlabel}",
		"pathToStatic": "static",
		"isOntologyDocumentation": False,
		"h1": doc_h1,
		"dls": dls,
		"content": contents,
		}

		return self.template.render(context)


		def title(self, document: SAREFGraphDocument):
		g = self.dataset.graph(document.version_iri)

src/saref_pypeline/pipeline.py

+4 −0

Original line number	Diff line number	Diff line
		@@ -10,6 +10,7 @@ from typing import Set, Dict, List, Tuple

		from git import Git, Repo, GitCommandError
		from rdflib import (
		SKOS,
		BNode,
		Dataset,
		Graph,
		@@ -917,6 +918,9 @@ class SAREFPipeline:
		metadata.add((term, RDFS.seeAlso, ontology_version_iri))
		# add metadata if the term is used or subject of this version
		if term.startswith(BASE) and not PATTERN_SAREF_GRAPHS.match(term):
		hidden_label = re.match(r"(.)/([^/#])(#[^#/]*)?", term).group(2)
		metadata.add((term, SKOS.hiddenLabel, Literal(hidden_label)))

		metadata.add((version_iri, DCTERMS.references, term))
		metadata.add((term, DCTERMS.isReferencedBy, version_iri))
		if term.startswith(namespace):