Unverified Commit 82a0ba1d authored by Maxime Lefrançois's avatar Maxime Lefrançois
Browse files

package for PyPI: make rdflib-leveldb optional 

PyPI does not accept direct URL dependencies. The on-disk LevelDB cache
is now used only if rdflib-leveldb is installed, otherwise an in-memory
dataset is used. The docker image still installs it.

Also add PyPI metadata (readme, license, urls, classifiers), and fix the
install commands in the README (package saref-pypeline).
parent 60134aef

README.md

+25 −7
Changes for README.md: 25 added lines, 7 removed lines.
Original line number Diff line number Diff line
@@ -13,22 +13,34 @@ NOTE: SAREF Pypeline has been tested on Linux and Windows WSL2. It's not a goal 
**Prerequisites:** 



* [`uv`](https://docs.astral.sh/uv/) (recommended) or `pipx` with Python >= 3.13

* [`leveldb`](https://github.com/google/leveldb) (package `libleveldb-dev` on ubuntu)

* for command `check`: Java >= 11





**With uv:** 



uv tool install saref-pipeline

```

uv tool install 'saref-pypeline[check]'

```



**With pipx:** 



pipx install saref-pipeline

```

pipx install 'saref-pypeline[check]'

```



**For local development:** 



```

git clone https://labs.etsi.org/rep/saref/saref-pypeline.git

cd saref-pipeline

cd saref-pypeline

pip install -e .[check,website,ts]

```



**Optional cache:** if the package [`rdflib-leveldb`](https://github.com/RDFLib/rdflib-leveldb) is installed, the documents loaded are cached in the `target/dataset` folder between runs. It requires [`leveldb`](https://github.com/google/leveldb) (package `libleveldb-dev` on ubuntu), and is not available on PyPI:



```

uv tool install 'saref-pypeline[check]' --with 'rdflib-leveldb @ git+https://github.com/RDFLib/rdflib-leveldb'

```



## Basic usage
@@ -86,11 +98,18 @@ This command require extra 'check', install with uv tool install 'saref-pypeline 
```



Optional features require extras, e.g.:

uv tool install 'saref-pipeline[check]'

uv tool install 'saref-pipeline[website]'



```

uv tool install 'saref-pypeline[check]'

uv tool install 'saref-pypeline[website]'

```



Option `-c` is only available for `check`.



The project is deduced from the name of the project directory (e.g., `saref4abcd`), or else from the name of the repository of its git remote `origin`.



Command `check` exits with code 1 if some error is reported, so it can be used in continuous integration.





## About
@@ -119,7 +138,6 @@ The content of this repository is licensed under the terms of the BSD-3-Clause L 
Known limitations:



* [] ts generation: missing general axioms

* [] publish to pypi.



## For maintainers

dockerfile

+1 −1
Changes for dockerfile: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -20,7 +20,7 @@ COPY pyproject.toml uv.lock .python-version ./ 
COPY src/ src/



ENV SETUPTOOLS_SCM_PRETEND_VERSION_FOR_SAREF_PYPELINE=${PSEUDO_VERSION}

RUN uv pip install --system -e .

RUN uv pip install --system -e . "rdflib-leveldb @ git+https://github.com/RDFLib/rdflib-leveldb"

CMD ["saref-dev"]



# ---- Website ----

pyproject.toml

+14 −2
Changes for pyproject.toml: 14 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -5,22 +5,34 @@ build-backend = "setuptools.build_meta" 
[project]

name = "saref-pypeline"

dynamic = ["version"]

description = "SAREF pipeline tools"

description = "SAREF pipeline tools: check SAREF projects against ETSI TS 103 673, generate their documentation website and Technical Specifications"

readme = "README.md"

license = "BSD-3-Clause"

license-files = ["LICENSE"]

authors = [

  { name = "Maxime Lefrançois", email = "maxime.lefrancois@emse.fr" }

]

requires-python = ">= 3.13"

classifiers = [

  "Programming Language :: Python :: 3",

  "Operating System :: OS Independent",

  "Topic :: Scientific/Engineering",

]

dependencies = [

  "python-gitlab",

  "requests",

  "rdflib ~=7.0.0",

  "rdflib-leveldb @ git+https://github.com/RDFLib/rdflib-leveldb",

  "gitpython",

  "beautifulsoup4",

  "markdown",

  "pyyaml",

]



[project.urls]

Homepage = "https://saref.etsi.org/"

Repository = "https://labs.etsi.org/rep/saref/saref-pypeline"

Issues = "https://labs.etsi.org/rep/saref/saref-pypeline/-/issues"



[tool.setuptools.packages.find]

where = ["src"]

src/saref_pypeline/pipeline.py

+23 −1
Changes for src/saref_pypeline/pipeline.py: 23 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -140,6 +140,28 @@ class LevelDBDataset(Dataset): 
        self.close()





class InMemoryDataset(Dataset):

    """Fallback for the on-disk dataset when the optional package rdflib-leveldb is not installed."""



    def __enter__(self):

        return self



    def __exit__(self, exc_type, exc_value, traceback):

        pass





def open_dataset_on_disk(target_dir: str) -> Dataset:

    """On-disk dataset that caches the loaded documents between runs, if rdflib-leveldb is installed."""

    try:

        plugin.get("LevelDB", Store)

    except plugin.PluginException:

        logger.debug(

            "Optional package rdflib-leveldb is not installed: documents are not cached between runs."

        )

        return InMemoryDataset()

    return LevelDBDataset(target_dir)





class SAREFPipeline:



    def __init__(
@@ -252,7 +274,7 @@ class SAREFPipeline: 
        self.fetch_projects_metadata()



        self.dataset = Dataset()  # in memory dataset

        with LevelDBDataset(self.target_dir) as self.dataset_on_disk:  # on-disk dataset

        with open_dataset_on_disk(self.target_dir) as self.dataset_on_disk:



            basename = os.path.basename(os.path.abspath(self.directory))

            if "portal" in basename:

uv.lock

+0 −27
Changes for uv.lock: 0 added lines, 27 removed lines.
Original line number Diff line number Diff line
@@ -251,12 +251,6 @@ wheels = [ 
    { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" },

]



[[package]]

name = "plyvel"

version = "1.5.1"

source = { registry = "https://pypi.org/simple" }

sdist = { url = "https://files.pythonhosted.org/packages/4b/b0/23f0cc21c943355fc5d49f1933e97812269d7a865daef5b9758b62902d46/plyvel-1.5.1.tar.gz", hash = "sha256:3caf6009e4fc24fbf8712d3f5ef3da7e56495c26a488df2161218fc34e46019c", size = 200128, upload-time = "2024-01-15T17:33:55.681Z" }



[[package]]

name = "prettytable"

version = "3.16.0"
@@ -383,16 +377,6 @@ html = [ 
    { name = "html5lib" },

]



[[package]]

name = "rdflib-leveldb"

version = "0.2"

source = { git = "https://github.com/RDFLib/rdflib-leveldb#a0f3386c71e6b1cfbd09c257e29400f5fde43ed3" }

dependencies = [

    { name = "plyvel" },

    { name = "rdflib" },

    { name = "setuptools" },

]



[[package]]

name = "requests"

version = "2.32.5"
@@ -450,7 +434,6 @@ dependencies = [ 
    { name = "python-gitlab" },

    { name = "pyyaml" },

    { name = "rdflib" },

    { name = "rdflib-leveldb" },

    { name = "requests" },

]
@@ -507,7 +490,6 @@ requires-dist = [ 
    { name = "pyyaml" },

    { name = "rcssmin", marker = "extra == 'website'", specifier = ">=1.2.1" },

    { name = "rdflib", specifier = "~=7.0.0" },

    { name = "rdflib-leveldb", git = "https://github.com/RDFLib/rdflib-leveldb" },

    { name = "requests" },

    { name = "rjsmin", marker = "extra == 'website'", specifier = ">=1.2.4" },

    { name = "saref-pypeline", extras = ["check"], marker = "extra == 'all'" },
@@ -517,15 +499,6 @@ requires-dist = [ 
]

provides-extras = ["check", "website", "ts", "ts2md", "all"]



[[package]]

name = "setuptools"

version = "80.9.0"

source = { registry = "https://pypi.org/simple" }

sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958, upload-time = "2025-05-27T00:56:51.443Z" }

wheels = [

    { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" },

]



[[package]]

name = "six"

version = "1.17.0"