documentation generation as in saref-pipeline for ontologies (843cd6ac) · Commits · SAREF / saref-pypeline

Pipfile

+5 −0

Original line number	Diff line number	Diff line
		@@ -12,6 +12,11 @@ pyyaml = "*"
		pyshacl = "*"
		requests = "*"
		dotenv = "*"
		python-gitlab = "*"
		jinja2 = "*"
		dominate = "*"
		markdown = "*"
		pygments = "*"

		[dev-packages]
		pytest = "*"

Pipfile.lock

+124 −4

Original line number	Diff line number	Diff line
		{
		"_meta": {
		"hash": {
		"sha256": "75b617610b81155b6d63810169212d552e7d100372170be40c36c69dc930f935"
		"sha256": "fcc4505ca4139dc762204a1d5d5b1b2c93fb4ba7f7f8458e3d1fbc907544e8bf"
		},
		"pipfile-spec": 6,
		"requires": {},
		@@ -120,6 +120,15 @@
		"markers": "python_version >= '3.7'",
		"version": "==3.4.2"
		},
		"dominate": {
		"hashes": [
		"sha256:558284687d9b8aae1904e3d6051ad132dd4a8c0cf551b37ea4e7e42a31d19dc4",
		"sha256:cb7b6b79d33b15ae0a6e87856b984879927c7c2ebb29522df4c75b28ffd9b989"
		],
		"index": "pypi",
		"markers": "python_version >= '3.4'",
		"version": "==2.9.1"
		},
		"dotenv": {
		"hashes": [
		"sha256:29cf74a087b31dafdb5a446b6d7e11cbce8ed2741540e2339c69fbef92c94ce9"
		@@ -159,6 +168,91 @@
		],
		"version": "==0.6.1"
		},
		"jinja2": {
		"hashes": [
		"sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d",
		"sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"
		],
		"index": "pypi",
		"markers": "python_version >= '3.7'",
		"version": "==3.1.6"
		},
		"markdown": {
		"hashes": [
		"sha256:247b9a70dd12e27f67431ce62523e675b866d254f900c4fe75ce3dda62237c45",
		"sha256:5c83764dbd4e00bdd94d85a19b8d55ccca20fe35b2e678a1422b380324dd5f24"
		],
		"index": "pypi",
		"markers": "python_version >= '3.9'",
		"version": "==3.8.2"
		},
		"markupsafe": {
		"hashes": [
		"sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4",
		"sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30",
		"sha256:1225beacc926f536dc82e45f8a4d68502949dc67eea90eab715dea3a21c1b5f0",
		"sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9",
		"sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396",
		"sha256:1a9d3f5f0901fdec14d8d2f66ef7d035f2157240a433441719ac9a3fba440b13",
		"sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028",
		"sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca",
		"sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557",
		"sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832",
		"sha256:3169b1eefae027567d1ce6ee7cae382c57fe26e82775f460f0b2778beaad66c0",
		"sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b",
		"sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579",
		"sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a",
		"sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c",
		"sha256:48032821bbdf20f5799ff537c7ac3d1fba0ba032cfc06194faffa8cda8b560ff",
		"sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c",
		"sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22",
		"sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094",
		"sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb",
		"sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e",
		"sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5",
		"sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a",
		"sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d",
		"sha256:6e296a513ca3d94054c2c881cc913116e90fd030ad1c656b3869762b754f5f8a",
		"sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b",
		"sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8",
		"sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225",
		"sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c",
		"sha256:88b49a3b9ff31e19998750c38e030fc7bb937398b1f78cfa599aaef92d693144",
		"sha256:8c4e8c3ce11e1f92f6536ff07154f9d49677ebaaafc32db9db4620bc11ed480f",
		"sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87",
		"sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d",
		"sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93",
		"sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf",
		"sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158",
		"sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84",
		"sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb",
		"sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48",
		"sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171",
		"sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c",
		"sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6",
		"sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd",
		"sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d",
		"sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1",
		"sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d",
		"sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca",
		"sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a",
		"sha256:cfad01eed2c2e0c01fd0ecd2ef42c492f7f93902e39a42fc9ee1692961443a29",
		"sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe",
		"sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798",
		"sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c",
		"sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8",
		"sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f",
		"sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f",
		"sha256:eaa0a10b7f72326f1372a713e73c3f739b524b3af41feb43e4921cb529f5929a",
		"sha256:eb7972a85c54febfb25b5c4b4f3af4dcc731994c7da0d8a0b4a6eb0640e1d178",
		"sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0",
		"sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79",
		"sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430",
		"sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50"
		],
		"markers": "python_version >= '3.9'",
		"version": "==3.0.2"
		},
		"owlready2": {
		"hashes": [
		"sha256:86b4d8500d769a674c524b54397fdd738ff5d0a96878432b69f4d606d6a7a4d8"
		@@ -202,6 +296,15 @@
		"markers": "python_version >= '3.9'",
		"version": "==3.16.0"
		},
		"pygments": {
		"hashes": [
		"sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887",
		"sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"
		],
		"index": "pypi",
		"markers": "python_version >= '3.8'",
		"version": "==2.19.2"
		},
		"pyparsing": {
		"hashes": [
		"sha256:a749938e02d6fd0b59b356ca504a24982314bb090c383e3cf201c95ef7e2bfcf",
		@@ -227,6 +330,15 @@
		"markers": "python_version >= '3.9'",
		"version": "==1.1.1"
		},
		"python-gitlab": {
		"hashes": [
		"sha256:062d6943699b09bce40f727f0d0906a0b254ca455593f81745bb1a352535ee77",
		"sha256:d36fb714cfde7773dcd6c19dd714bdb6e4dfb00a7ef694c8da176335da171b23"
		],
		"index": "pypi",
		"markers": "python_full_version >= '3.9.0'",
		"version": "==6.1.0"
		},
		"pyyaml": {
		"hashes": [
		"sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff",
		@@ -312,6 +424,14 @@
		"markers": "python_version >= '3.8'",
		"version": "==2.32.4"
		},
		"requests-toolbelt": {
		"hashes": [
		"sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6",
		"sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06"
		],
		"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
		"version": "==1.0.0"
		},
		"setuptools": {
		"hashes": [
		"sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922",
		@@ -379,11 +499,11 @@
		},
		"pygments": {
		"hashes": [
		"sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f",
		"sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c"
		"sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887",
		"sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"
		],
		"markers": "python_version >= '3.8'",
		"version": "==2.19.1"
		"version": "==2.19.2"
		},
		"pytest": {
		"hashes": [

saref_pypeline/main.py

+26 −4

Original line number	Diff line number	Diff line
		import sys
		import argparse
		import os
		from cProfile import Profile
		from pstats import SortKey, Stats

		try:
		from saref_pypeline.pipeline import SAREFPipeline
		from saref_pypeline.entities import PipelineMode
		from saref_pypeline.constants import PipelineMode
		except ModuleNotFoundError as e:
		if __name__ == "__main__":
		import sys
		from pathlib import Path
		sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
		from saref_pypeline.pipeline import SAREFPipeline
		from saref_pypeline.entities import PipelineMode
		from saref_pypeline.constants import PipelineMode
		else:
		raise

		@@ -45,6 +47,21 @@ def main():
		action="store_true",
		help="Do not generate the static portal for terms."
		)
		parser.add_argument(
		"-f", "--no-fetch",
		action="store_true",
		help="Do not fetch the changes for the existing sources (git fetch)."
		)
		parser.add_argument(
		"-i", "--include",
		action="append",
		help="Include specific project versions (can be provided multiple times)."
		)
		parser.add_argument(
		"-x", "--exclude",
		action="append",
		help="Exclude specific project versions (can be provided multiple times)."
		)
		try:
		args = parser.parse_args()
		args.mode = PipelineMode(args.mode)
		@@ -62,9 +79,14 @@ def main():
		directory=args.directory,
		no_examples=args.no_examples,
		no_terms=args.no_terms,
		no_site=args.no_site
		no_site=args.no_site,
		no_fetch=args.no_fetch,
		include=args.include,
		exclude=args.exclude
		)
		with Profile() as profile:
		pipeline.run()
		# Stats(profile).sort_stats(SortKey.CUMULATIVE).print_stats(0.1)

		if __name__ == "__main__":
		main()

saref_pypeline/checkers/BaseChecker.py

+39 −40

Original line number	Diff line number	Diff line
		import os
		from saref_pypeline.localization import Localization
		from saref_pypeline.entities import PipelineConfig, SAREFProject, SAREFVersionName, SAREFVersion
		from packaging.version import Version
		from saref_pypeline.entities import SAREFProject, SAREFProjectVersion, SAREFVersionName
		import logging
		from rdflib import Dataset, Graph

		from typing import TYPE_CHECKING
		if TYPE_CHECKING:
		from saref_pypeline.pipeline import SAREFPipeline

		CHECKER_SUFFIX = "_Checker.py"

		class FirstEventHandler(logging.Handler):
		def __init__(self, title):
		super().__init__()
		self.title = title
		self.first_event_logged = False

		def emit(self, record):
		if not self.first_event_logged:
		self.first_event_logged = True
		logger = logging.getLogger(record.name)
		logger.info(self.title)

		class BaseChecker:

		def __init__(self, caller_file: str, config: PipelineConfig, project_version: SAREFVersion):
		def __init__(self, caller_file: str, pipeline: "SAREFPipeline", project_version: SAREFProjectVersion, logger_name=None):
		self.file = caller_file
		self.config = config
		self.project_version = project_version
		self.pipeline = pipeline
		self.project_version:SAREFProjectVersion = project_version
		self.project:SAREFProject = project_version.project
		self.version:SAREFVersionName = project_version.version
		if not isinstance(logger_name, None \| str):
		raise ValueError("logger_name shall be None or a string")
		if logger_name == None:
		basename = os.path.basename(caller_file)
		self.ts, _, *clause, _ = basename.split('_')
		self.clause = ".".join(clause)
		self.display_name = f"{self.ts} Clause {self.clause}"
		logger_name = f"saref_pypeline.checkers.{os.path.splitext(basename)[0]}"
		self._logger = logging.getLogger(logger_name)
		# self.localization = Localization(caller_file)
		self._logged_title = False

		def check_clause(self, dataset: Dataset=None):
		pass

		def _log_title_if_needed(self):
		if self._logged_title:
		return
		self._logged_title = True
		self._logger.info(f"""## Issues in {self.project.name} {self.version} w.r.t. {self.display_name}""")

		def logWarning(self, args, *kwargs):
		self._log_title_if_needed()
		self._logger.warning(args, *kwargs)

		def logError(self, args, *kwargs):
		self._log_title_if_needed()
		self._logger.error(args, *kwargs)

		def logInfo(self, args, *kwargs):
		self._log_title_if_needed()
		self._logger.info(args, *kwargs)

		def logDebug(self, args, *kwargs):
		self._log_title_if_needed()
		self._logger.debug(args, *kwargs)
		self.ts = basename[:basename.index('_')]
		if 'Clause' in basename:
		_, _, *clauses, _ = basename.split('_')
		clause = '.'.join(clauses)
		self.display_name = f"{self.ts} Clause {clause}"
		logger_name = f"{project_version.logger_name}.{self.ts}.{clause}"
		else:
		self.display_name = f"{self.ts}"
		logger_name = f"{project_version.logger_name}.{self.ts}"
		nb_hash = logger_name.count('.')
		self.logger = logging.getLogger(logger_name)
		if not any(isinstance(h, FirstEventHandler) for h in self.logger.handlers):
		self.logger.addHandler(FirstEventHandler(f"""{'#'*nb_hash} Issues in {self.project_version} w.r.t. {self.display_name}"""))

saref_pypeline/checkers/BaseExampleChecker.py

deleted100644 → 0

+0 −18

Original line number	Diff line number	Diff line
		from saref_pypeline.checkers import BaseChecker
		from saref_pypeline.entities import PipelineConfig, SAREFExample
		from rdflib import Dataset

		class BaseExampleChecker(BaseChecker):

		def __init__(self, caller_file: str, config: PipelineConfig, example: SAREFExample):
		super().__init__(caller_file, config, example.project_version)
		self.example = example

		def _log_title_if_needed(self):
		if self._logged_title:
		return
		self._logged_title = True
		self._logger.info(f"""## Issues in example {self.example.name} of {self.project.name} {self.version} w.r.t. {self.display_name}""")

		def check_clause(self, dataset: Dataset=None):
		pass