Unverified Commit a19fd9f2 authored by Maxime Lefrançois's avatar Maxime Lefrançois
Browse files

add lunr search index

parent f5730aa9

pyproject.toml

+1 −0
Original line number Diff line number Diff line
@@ -26,6 +26,7 @@ dependencies = [ 
  "beautifulsoup4",

  "python-docx",

  "python-dotenv",

  "lunr>=0.8.0",

]



[tool.setuptools.packages.find]

src/saref_pypeline/docgen/site_manager.py

+49 −12
Original line number Diff line number Diff line 
from functools import cache, cached_property

import json

import logging

import os

from pathlib import Path

import shutil

import logging

from git import Repo, GitCommandError

from saref_pypeline.docgen.ts_generator import TSGenerator

from functools import cache, cached_property

from itertools import chain

from pathlib import Path

from typing import TYPE_CHECKING, Dict, TypeVar



from bs4 import BeautifulSoup

from git import GitCommandError, Repo

from lunr import lunr



from saref_pypeline.constants import *

from saref_pypeline.docgen.ts2md_extractor import TS2MDExtractor

from saref_pypeline.docgen.ts_generator import TSGenerator

from saref_pypeline.docgen.utils import *

from saref_pypeline.docgen.website_generator import WebsiteGenerator

from saref_pypeline.entities import (

    SAREFCore,

    SAREFPatterns,
@@ -14,11 +24,9 @@ from saref_pypeline.entities import ( 
)

from saref_pypeline.etsi import WK_FIELD

from saref_pypeline.localization import Localization

from saref_pypeline.constants import *

from typing import TYPE_CHECKING, Dict, TypeVar

from itertools import chain

from saref_pypeline.docgen.utils import *

from saref_pypeline.docgen.website_generator import WebsiteGenerator









T = TypeVar("T")
@@ -193,9 +201,9 @@ class SiteManager: 
        # document term in project version directory, project directory, and root directory

        p_dir = Path(self.site_dir, project_version.project.path)

        pv_dir = Path(p_dir, str(project_version.version))

        for term in metadata.subjects(

        for term in sorted(metadata.subjects(

            RDFS.isDefinedBy, project_version.ontology.version_iri

        ):

        ), key= lambda x: x.lower()):

            logger.debug(f"Printing html documentation for {term.n3(self.dataset.namespace_manager)}")

            desc = docgen.description(term, onto_graph)
@@ -330,3 +338,32 @@ class SiteManager: 
            writer.write("\n")



        writer.write("\n")



    def generate_index(self):

        logger.info("Building search index ...")

        documents = []

        site_dir = self.site_dir



        for root, _, files in os.walk(site_dir):

            for f in files:

                if f.endswith(".html"):

                    path = os.path.join(root, f)

                    url = os.path.relpath(path, site_dir)  # relative URL

                    with open(path, encoding="utf-8") as fp:

                        soup = BeautifulSoup(fp, "html.parser")

                        title = soup.title.string if soup.title else url

                        content = soup.get_text(" ", strip=True)

                        documents.append({"url": url, "title": title, "content": content})



        index = lunr(

            ref="url",

            fields=("title", "content"),

            documents=documents

        )



        # 3. Dump both the raw documents and the index

        with open(os.path.join(site_dir, "search_index.json"), "w", encoding="utf-8") as f:

            json.dump({

                "docs": documents,

                "index": index.serialize()

            }, f, ensure_ascii=False)

src/saref_pypeline/docgen/website_generator.py

+12 −6
Original line number Diff line number Diff line 
from datetime import date

from functools import cache

import os

from pathlib import Path
@@ -494,7 +495,8 @@ class WebsiteGenerator: 


        context = {

            "title": self.title(document),

            "pathToStatic": "../../static",

            "yyyy": date.today().year,

            "pathToRoot": "../../",

            "isOntologyDocumentation": True,

            "h1": doc_h1,

            "dls": dls,
@@ -629,10 +631,11 @@ class WebsiteGenerator: 


        context = {

            "title": self.title(document),

            "pathToStatic": "../../../static",

            "yyyy": date.today().year,

            "pathToRoot": "../../../",

            "isOntologyDocumentation": False,

            "dls": dls,

            "h1": doc_h1,

            "dls": dls,

            "content": contents,

        }
@@ -849,7 +852,8 @@ class WebsiteGenerator: 


        context = {

            "title": f"{desc.curie} — {desc.label}",

            "pathToStatic": "../../static",

            "yyyy": date.today().year,

            "pathToRoot": "../../",

            "isOntologyDocumentation": False,

            "h1": doc_h1,

            "dls": dls,
@@ -977,7 +981,8 @@ class WebsiteGenerator: 


        context = {

            "title": f"{project.path}/{hlabel}",

            "pathToStatic": "../static",

            "yyyy": date.today().year,

            "pathToRoot": "../",

            "isOntologyDocumentation": False,

            "h1": doc_h1,

            "dls": dls,
@@ -1034,7 +1039,8 @@ class WebsiteGenerator: 


        context = {

            "title": f"{project.path}/{hlabel}",

            "pathToStatic": "static",

            "yyyy": date.today().year,

            "pathToRoot": "./",

            "isOntologyDocumentation": False,

            "h1": doc_h1,

            "dls": dls,

src/saref_pypeline/pipeline.py

+2 −0
Original line number Diff line number Diff line
@@ -292,6 +292,7 @@ class SAREFPipeline: 


        if self.mode == PipelineMode.WEBSITE:

            self.site_manager.generate_htaccess()

            self.site_manager.generate_index()



    def run_on_project(self, basename):

        try:
@@ -332,6 +333,7 @@ class SAREFPipeline: 
            self.site_manager.copy_static_files()

            self.site_manager.generate_website(project_version)

            self.site_manager.generate_htaccess()

            self.site_manager.generate_index()

        elif self.mode == PipelineMode.TS:

            self.site_manager.generate_ts(project_version)

        elif self.mode == PipelineMode.TS2MD:

uv.lock

+11 −0
Original line number Diff line number Diff line
@@ -134,6 +134,15 @@ wheels = [ 
    { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" },

]



[[package]]

name = "lunr"

version = "0.8.0"

source = { registry = "https://pypi.org/simple" }

sdist = { url = "https://files.pythonhosted.org/packages/43/e9/b3dee02312eaa2a1b3212b6e20a90a81adba489b404d4f0ffbbe8258b761/lunr-0.8.0.tar.gz", hash = "sha256:b46cf5059578d277a14bfc901bb3d5666d013bf73c035331ac0222fdac358228", size = 1147598, upload-time = "2025-03-08T13:31:40.907Z" }

wheels = [

    { url = "https://files.pythonhosted.org/packages/6c/8b/bf975fabd26195915ebdf3e4252baa936f1863bcd9eb49598b705638f5d5/lunr-0.8.0-py3-none-any.whl", hash = "sha256:a2bc4e08dbb35b32723006bf2edbe6dc1f4f4b95955eea0d23165a184d276ce8", size = 35211, upload-time = "2025-03-08T13:31:38.657Z" },

]



[[package]]

name = "lxml"

version = "6.0.1"
@@ -409,6 +418,7 @@ dependencies = [ 
    { name = "dominate" },

    { name = "gitpython" },

    { name = "jinja2" },

    { name = "lunr" },

    { name = "markdown" },

    { name = "owlready2" },

    { name = "pygments" },
@@ -428,6 +438,7 @@ requires-dist = [ 
    { name = "dominate" },

    { name = "gitpython" },

    { name = "jinja2" },

    { name = "lunr", specifier = ">=0.8.0" },

    { name = "markdown" },

    { name = "owlready2" },

    { name = "pygments" },