test: create a disposable platform lab for local and GitLab checks

  stage: test

  script:

    - python3 -m py_compile ansible/roles/oop-platform/files/fm_partners.py ansible/tests/r2_e2e/*.py ci/*.py

    - sh -n ci/run_platform_lab.sh

platform-smoke-checks:

  stage: test

    reports:

      junit: .cache/platform-smoke/junit.xml

    expire_in: 1 week

# Disposable core platform: create -> existing deploy -> existing health -> delete.

# Requires a trusted DinD-capable runner with shared TLS client certificates.

platform-lab:

  stage: test

  image: docker:29.5.1-cli@sha256:b40b3737eb3bf588d25bb856d3564dd3f9fdb32ac2fc19ebe85cc58d761692a5

  services:

    - name: docker:29.5.1-dind@sha256:eed377f3ce14afda560e742c5b3ae446df1ce00a16e1beab2ac6c78c1477fe85

      alias: docker

      variables:

        HEALTHCHECK_TCP_PORT: "2376"

  cache: []

  timeout: 40 minutes

  variables:

    FF_NETWORK_PER_BUILD: "true"

    DOCKER_HOST: tcp://docker:2376

    DOCKER_TLS_CERTDIR: /certs

    DOCKER_TLS_VERIFY: "1"

    DOCKER_CERT_PATH: /certs/client

  rules:

    - if: '$RUN_PLATFORM_LAB == "1" && $CI_COMMIT_REF_PROTECTED == "true"'

    - when: never

  script:

    - sh ci/run_platform_lab.sh "$OOP_LAB_PROFILE" .cache/platform-lab

  artifacts:

    when: always

    paths:

      - .cache/platform-lab/result.json

      - .cache/platform-lab/pods.txt

      - .cache/platform-lab/events.txt

      - .cache/platform-lab/health/result.json

    reports:

      junit: .cache/platform-lab/health/junit.xml

    expire_in: 1 week

"""Qualify lab ownership, stage failures and cleanup at the command boundary."""

import importlib.util

import json

from pathlib import Path

import subprocess

import pytest

ROOT = Path(__file__).resolve().parents[3]

spec = importlib.util.spec_from_file_location("platform_lab", ROOT / "ci/platform_lab.py")

lab = importlib.util.module_from_spec(spec)

spec.loader.exec_module(lab)

PROFILE = {"chart_revision": "a" * 40,

           "images": {name: f"example/{name}:candidate@sha256:" + "b" * 64

                      for name in ("oeg", "fm", "srm")}}

def test_ci_rejects_unpinned_images_and_local_exceptions():

    profile = {**PROFILE, "images": {**PROFILE["images"], "srm": "local/srm:candidate"},

               "local_images": ["local/srm:candidate"]}

    with pytest.raises(ValueError):

        lab.validate_profile(profile)

    lab.validate_profile(profile, allow_local=True)

    with pytest.raises(ValueError):

        lab.validate_profile({**profile, "local_images": []})

@pytest.mark.parametrize("failure", [None, "cluster", "deploy", "identity", "health", "cleanup", "existing"])

def test_owned_lab_lifecycle_propagates_failure_and_preserves_existing_clusters(tmp_path, failure):

    name = "oop-ci-123-1234abcd"

    (tmp_path / "private").mkdir()

    commands = []

    def command(argv, **kwargs):

        commands.append(argv)

        output = ""

        if argv[:3] == ["k3d", "cluster", "list"]:

            output = json.dumps([{"name": name if failure == "existing" else "shared-lab"}])

        elif argv[:3] == ["k3d", "cluster", "create"] and failure == "cluster":

            raise subprocess.CalledProcessError(1, argv)

        elif argv[:2] == ["docker", "inspect"]:

            output = json.dumps([{"NetworkSettings": {"Networks": {

                "k3d-" + name: {"IPAddress": "172.19.0.2"}}}}])

        elif argv[:2] == ["docker", "exec"]:

            output = '{"clusters":[{"cluster":{"server":"https://old:6443"}}]}'

        elif argv[:3] == ["docker", "image", "inspect"]:

            output = "sha256:" + "a" * 64

        elif "pods" in argv and "json" in argv:

            output = json.dumps({"items": [{"spec": {"containers": [

                {"name": c, "image": "wrong:candidate" if failure == "identity" else f"oop-ci-images/{c}:{name}"}]}, "status": {

                    "containerStatuses": [{"name": c, "ready": True,

                        "imageID": "sha256:" + "a" * 64}]}}

                for c in ("oeg", "fm", "srm")]})

        elif "ansible/playbooks/scenarios/r2_existing_cluster/deploy.yml" in argv:

            if failure == "deploy":

                raise subprocess.CalledProcessError(2, argv)

            inventory = json.loads((tmp_path / "private/inventory.json").read_text())

            operator = inventory["all"]["children"]["oop_platform_operators"]["hosts"]["ci_lab"]

            assert operator["oop_platform_public_host"] == "172.19.0.2"

            assert operator["ansible_connection"] == "local"

        elif "ci/platform_smoke.py" in argv:

            if failure == "health":

                raise subprocess.CalledProcessError(1, argv)

            (tmp_path / "health").mkdir()

            (tmp_path / "health/result.json").write_text('{"status":"passed","tests":1}')

        elif argv[:3] == ["k3d", "cluster", "delete"] and failure == "cleanup":

            raise subprocess.CalledProcessError(1, argv)

        return subprocess.CompletedProcess(argv, 0, stdout=output)

    result = lab.exercise(PROFILE, tmp_path, name, Path("/chart"), "controller", call=command)

    summary = json.loads((tmp_path / "result.json").read_text())

    assert result == (0 if failure is None else 1)

    assert summary["status"] == ("passed" if failure is None else "failed")

    removed = [argv[-1] for argv in commands if argv[:3] == ["k3d", "cluster", "delete"]]

    assert removed == ([] if failure == "existing" else [name])

    if failure in ("deploy", "identity", "health"):

        assert summary["stage"] == failure

        assert (tmp_path / "pods.txt").exists()

        assert (tmp_path / "events.txt").exists()

    if failure == "existing":

        assert len(commands) == 1

    if failure == "cleanup":

        assert summary["cleanup"] == "failed"

FROM docker:29.5.1-cli@sha256:b40b3737eb3bf588d25bb856d3564dd3f9fdb32ac2fc19ebe85cc58d761692a5 AS docker

FROM python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1

COPY --from=docker /usr/local/bin/docker /usr/local/bin/docker

RUN apt-get update && apt-get install -y --no-install-recommends bash ca-certificates git \

    && rm -rf /var/lib/apt/lists/*

WORKDIR /work

ENV TMPDIR=/work/.cache/tmp PYTHONDONTWRITEBYTECODE=1 \

    ANSIBLE_COLLECTIONS_PATH=/opt/collections

COPY ci/requirements-smoke.txt ci/requirements-smoke.txt

RUN mkdir -p "$TMPDIR" \

    && pip install --no-cache-dir -r ci/requirements-smoke.txt kubernetes==34.1.0 \

    && ansible-galaxy collection install kubernetes.core:6.6.0 -p /opt/collections

COPY ci/lab/install_tools.py ci/lab/install_tools.py

RUN python ci/lab/install_tools.py

COPY ansible.cfg ansible.cfg

COPY ansible/playbooks ansible/playbooks

COPY ansible/roles ansible/roles

COPY ansible/tests/r2_e2e ansible/tests/r2_e2e

COPY ci/platform_smoke.py ci/platform_smoke.py

COPY ci/platform_lab.py ci/platform_lab.py

CMD ["python", "ci/platform_lab.py", "--profile", "/work/profile.json"]

**

!ansible.cfg

!ci/

!ci/**

!ansible/

!ansible/playbooks/

!ansible/playbooks/**

!ansible/roles/

!ansible/roles/**

!ansible/tests/

!ansible/tests/r2_e2e/

!ansible/tests/r2_e2e/**

**/__pycache__/

**/*.pyc

**/.cache/

**/*.local.*

# Disposable platform health lab

Run the same controller locally and in GitLab CI:

```sh

sh ci/run_platform_lab.sh /path/to/reviewed-profile.json .cache/lab-run-001

```

Supply an empty report directory and a Docker daemon that can run privileged

k3d nodes. The command creates a unique cluster, installs the core platform

through the existing `r2_existing_cluster/deploy.yml`, and invokes the existing

`ci/platform_smoke.py` / `e2e.yml` health case. It removes its owned cluster,

network, private image tags, controller and configuration volume afterward.

It refuses existing cluster names and reports cleanup failures as failures.

The default kubeconfig stays unchanged. Application NodePorts stay inside the

temporary Docker network; the Kubernetes API mapping binds only to loopback.

The controller joins that network.

The core profile enables OEG, SRM, FM, Keycloak, PostgreSQL and NATS. Portal,

AI2, Data Lab, federation peers and external network providers remain outside

this first scenario. The existing deployment also waits for chart readiness

and checks public OIDC issuers. The selected pytest case proves OEG/FM HTTP

health only: no authentication transaction, SDK workload execution or app

endpoint acceptance is inferred.

## Candidate inputs

Copy `profile.example.json` outside tracked source and replace the SRM

placeholder with a reviewed `repository:tag@sha256:digest`. OEG/FM defaults

are inspected published candidates, not moving tags. The controller embeds

the stated Helm revision and refuses a different profile revision. Change

both through review when selecting another chart candidate. The pinned Linux

amd64 tools are checksum-verified at image build; apt/Python transitive

packages are not a fully locked operating-system supply chain.

All three component images must be pullable by the controller's Docker

client. It loads their exact contents under unique private tags and sets

`imagePullPolicy: Never`. This avoids changing current charts, which also

use image tags as Kubernetes labels. It requires ready Pods referencing the

owned imported tags and records Docker and Kubernetes image IDs separately;

the classic and containerd Docker stores expose different ID kinds. It does

not prove the SDK package contents inside an arbitrary SRM image.

An unpublished local SRM test image can be imported with an explicit local

exception: list its tag in `local_images` and set

`OOP_LAB_ALLOW_LOCAL_IMAGES=1`. This exception is rejected when `CI` is set.

Do not silently substitute a published older SRM/SDK to make the profile green.

Prepare a new reviewed immutable SRM image containing the intended SDK before

using this scenario as its CI acceptance gate. No image/package publication

is performed by this command.

## GitLab job

`platform-lab` is opt-in: a protected ref plus `RUN_PLATFORM_LAB=1`.

`OOP_LAB_PROFILE` is a protected File variable containing the reviewed JSON.

The Docker-executor runner must permit DinD and share its TLS client

certificates at `/certs/client`, as documented by GitLab. An online runner

or its description alone proves neither capability nor available capacity.

The job uses a per-job DinD service and network; its controller socket is the

service's Docker socket, not a mount of the runner host's Docker daemon.

Check runner configuration/capacity before enabling the job. The same wrapper

can run against a developer's local Docker daemon.

K3d's generated files use a private Docker volume mounted at the same absolute

path in the controller and daemon. This matters for a remote daemon: binding

the CI workspace path would refer to the DinD filesystem instead of the job.

The socket mounts directly under `/run` to avoid a `/var/run` symlink issue

observed with the containerd image store. Reports are copied through Docker;

no shared CI-workspace mount is assumed.

Only `result.json`, bounded Pod/event summaries and the health JUnit/result

are copied out. Kubeconfig, inventory, Ansible caches and generated platform

credentials remain private to the disposable controller/cluster. On ordinary

failure the same cleanup runs. Forced runner/process termination can prevent

local cleanup; destroying the CI job's DinD service removes the remaining lab.

No host-wide Docker prune or shared-cluster cleanup is used.

## Verification

Default CI keeps its existing behavior and adds offline controller checks.

`ansible/tests/unit/test_platform_lab.py` covers ownership, missing image pins,

wrong running candidate, stage failures and cleanup failure. The existing

smoke-command tests exercise the actual Ansible/pytest path against controlled

HTTP endpoints. A local DinD proof does not replace execution on the selected

GitLab runner or qualification of release-specific images.