test: extend the shared lab through application lifecycle checkpoints

    - sh -n ci/run_platform_lab.sh

    - python3 -m unittest discover -s ansible/tests -p test_tfsdk_bootstrap.py

# Pure gate/observation checks; no inventory, cluster access or deployment secrets.

sdk-acceptance-checks:

  rules:

    - if: '$RUN_SDK_ACCEPTANCE == "1"'

      when: never

    - when: on_success

  stage: test

  variables:

    TMPDIR: "$CI_PROJECT_DIR/.cache/sdk-tmp"

    PYTHONDONTWRITEBYTECODE: "1"

  script:

    - mkdir -p "$TMPDIR"

    - pip install pytest==8.3.2 requests==2.33.0

    - python -m pytest --confcutdir=ansible/tests/unit -q -p no:cacheprovider ansible/tests/unit/test_sdk_acceptance.py

    - python -m pytest --collect-only -q -p no:cacheprovider ansible/tests/r2_e2e

# Explicit live mode leaves all default component tests and deployment jobs alone.

sdk-live-acceptance:

  stage: test

  image: python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1

  tags: [sdk-lab]

  interruptible: false

  timeout: 30 minutes

  resource_group: dedicated-sdk-lab

  environment:

    name: sdk-acceptance

    action: verify

  cache: []

  variables:

    TMPDIR: "$CI_PROJECT_DIR/.cache/sdk-tmp"

    PYTHONDONTWRITEBYTECODE: "1"

  rules:

    - if: '$RUN_SDK_ACCEPTANCE == "1" && $CI_COMMIT_REF_PROTECTED == "true"'

    - when: never

  before_script:

    - mkdir -p "$TMPDIR"

    - pip install pytest==8.3.2 requests==2.33.0

    - python ci/install_kubectl.py

    - export PATH="$CI_PROJECT_DIR/.cache/bin:$PATH"

    - kubectl version --client

  script:

    - python ci/sdk_live.py

  artifacts:

    when: always

    paths:

      - .cache/sdk-live/qualification.json

      - .cache/sdk-e2e/*/run.json

    reports:

      junit: .cache/sdk-live/junit.xml

    expire_in: 90 days

platform-smoke-checks:

  stage: test

  image: python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1

    - mkdir -p "$TMPDIR"

    - pip install -r ci/requirements-smoke.txt

    - python -m pytest --confcutdir=ansible/tests/unit -q -p no:cacheprovider ansible/tests/unit --basetemp="$TMPDIR/pytest"

    - python -m pytest --collect-only -q -p no:cacheprovider ansible/tests/r2_e2e

# Read-only opt-in check of an already deployed platform, through its public URLs.

platform-smoke:

    DOCKER_TLS_VERIFY: "1"

    DOCKER_CERT_PATH: /certs/client

  rules:

    - if: '$CI_COMMIT_BRANCH == "test/platform-application-checkpoints" || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "test/platform-application-checkpoints"'

      when: manual

      allow_failure: false

      variables:

        OOP_LAB_PROFILE: ci/lab/profile.release-2.0.json

    - if: '$CI_COMMIT_BRANCH == "test/platform-lab"'

      when: manual

      allow_failure: false

    paths:

      - .cache/platform-lab/result.json

      - .cache/platform-lab/junit.xml

      - .cache/platform-lab/chart.json

      - .cache/platform-lab/wrapper-junit.xml

      - .cache/platform-lab/pods.txt

      - .cache/platform-lab/events.txt

      - .cache/platform-lab/health/result.json

      - .cache/platform-lab/health/junit.xml

      - .cache/platform-lab/health/acceptance/

    reports:

      junit:

        - .cache/platform-lab/junit.xml

        - .cache/platform-lab/wrapper-junit.xml

    expire_in: 1 week

# Explicit live mode leaves all default component tests and deployment jobs alone.

sdk-live-acceptance:

  stage: test

  image: python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1

  tags: [sdk-lab]

  interruptible: false

  timeout: 30 minutes

  resource_group: dedicated-sdk-lab

  environment:

    name: sdk-acceptance

    action: verify

  cache: []

  variables:

    TMPDIR: "$CI_PROJECT_DIR/.cache/sdk-tmp"

    PYTHONDONTWRITEBYTECODE: "1"

  rules:

    - if: '$RUN_SDK_ACCEPTANCE == "1" && $CI_COMMIT_REF_PROTECTED == "true"'

    - when: never

  before_script:

    - mkdir -p "$TMPDIR"

    - pip install pytest==8.3.2 requests==2.33.0

    - python ci/install_kubectl.py

    - export PATH="$CI_PROJECT_DIR/.cache/bin:$PATH"

    - kubectl version --client

  script:

    - python ci/sdk_live.py

  artifacts:

    when: always

    paths:

      - .cache/sdk-live/qualification.json

      - .cache/sdk-e2e/*/run.json

    reports:

      junit: .cache/sdk-live/junit.xml

    expire_in: 90 days

## Step 1: deploy one platform per operator

### Independent application onboarding

After deploying the platform, select the existing positive catalogue tests without

running federation, workload deployment or the other component scenarios:

```bash

ansible-playbook -i inventory/r2-sdk-lab.local.yml \

  playbooks/scenarios/r2_existing_cluster/e2e.yml \

  -e e2e_suite=onboarding \

  -e e2e_application_image=traefik/whoami@sha256:200689790a0a0ea48ca45992e0450bc26ccab5307375b41c84dfc4f2475937ab \

  -e '{"e2e_pytest_args":"--junitxml=.cache/onboarding.xml -p no:cacheprovider"}'

```

Create `.cache` first. Run this same command locally or in a GitLab job whose

runner can reach the inventory's public URLs and Kubernetes API. Install

`tests/r2_e2e/requirements.txt` in the controller's Python environment; set

`e2e_python` to a different test interpreter when needed. Retain the JUnit file

as a CI artifact. This change adds no live runner or automatic cluster access.

The two selected cases POST a unique caller-supplied application ID, read its

name/version/package/image, CPU/memory intent and external TCP interface, check

list membership, then delete and verify absence. OEG currently projects a

CONTAINER catalogue entry back to `ContainerResources`; CPU/memory comparisons

account for that representation. This suite does not certify full manifest

round-trip, topology-field retention, image pulls or resource enforcement.

Cleanup also runs after partial registration or a lost response. It deletes only

this run's known IDs and fails when deletion cannot be verified. Temporary

operator settings are removed even if pytest fails. The existing default suite

keeps its original 23 cases, and the full SDK endpoint/TI cases remain opt-in.

Generic suites cannot be combined with SDK checkpoint/profile options.

For a fresh local lab, create a separate cluster and kubeconfig, then use the

same deploy and E2E playbooks above. Do not replace an existing Kubernetes

context. Select a reviewed Release 2 revision of `refactor/helm-2.0` explicitly;

the chart repository's `develop` branch may still contain the earlier SRM/OEG

charts. Record chart/source revisions and actual running image IDs with every

live result. The SDK scenarios share this suite's supported Kubernetes manifest

builder rather than submitting the unsupported `infraKind: container` input.

Use controller temporary and fact-cache directories on disk when required:

`TMPDIR`, `ANSIBLE_LOCAL_TEMP`, `ANSIBLE_REMOTE_TEMP`,

`ANSIBLE_CACHE_PLUGIN_CONNECTION`, and the inventory's `e2e_temp_dir`.

### Cumulative SDK workload checkpoint

With the dedicated SDK inventory and completed artifact profile, pass these

options in an extra-vars YAML file (for example, an ignored `acceptance.local.yml`):

```yaml

e2e_suite: ""

e2e_pytest_args: >-

  --sdk-checkpoint=deployed

  --sdk-profile=profiles/lab.local.json

  --junitxml=.cache/deployed.xml

  -p no:cacheprovider

```

Run `e2e.yml -e @acceptance.local.yml`. The test checks the actual component image

IDs and installed SDK fingerprint before mutation, registers a fresh application,

selects the dedicated lab's single active zone, instantiates through OEG and waits

for public ready state. It then verifies the immutable workload image, matching

Service selector, allocated TCP NodePort and ready Pod. Teardown removes its

instance and waits for Services, Deployments and Pods to disappear. Sanitized

request status/correlation and workload identity evidence is saved under

`.cache/sdk-e2e/<run-id>/run.json` relative to the test process's working directory.

Select `endpoints` to extend the same flow with the original public endpoint,

allocated port and HTTP-to-this-Pod assertions. Missing endpoint output remains a

failure there even when `deployed` passes. The protected GitLab SDK lab job accepts

`SDK_CHECKPOINT=deployed` with its existing exact-artifact/profile requirements;

it still needs a configured dedicated runner and lab credentials. The current TI

case retains its earlier application-wide provider gate; device-scoped TI requires

a separate test update before it can qualify the newer public route.

The `oop-platform` role runs on the operator host, where the kubeconfig lives. A remote

host works the same way as `ansible_connection: local`. In order, it:

One known gap is marked `xfail(strict=True)`: OEG lets any App Provider read or delete

another tenant's app. Strict means the run fails once that's fixed, as a reminder to drop

the mark. The last run on the three test operators gave 22 passed and 1 xfailed. Pass

pytest arguments with `-e e2e_pytest_args='-k federation'`.

pytest arguments with `-e '{"e2e_pytest_args":"-k federation"}'`.

### First scenario: public HTTP health

#   For an operator reached over SSH, the tests use a copy of its kubeconfig on the

#   controller, pointed at the host's address (e2e_kube_api_host) when it names loopback.

# Usage: ansible-playbook -i inventory/r2.local.yml playbooks/scenarios/r2_existing_cluster/e2e.yml

#   Extra pytest arguments: -e e2e_pytest_args='-k federation'

#   Extra pytest arguments: -e '{"e2e_pytest_args":"-k federation"}'

#   Catalogue only: -e e2e_suite=onboarding -e e2e_application_image=IMAGE@sha256:DIGEST

- name: Collect the operators' settings

  hosts: oop_platform_operators

  gather_facts: true

  tasks:

    - name: Validate an independently selected acceptance suite

      ansible.builtin.assert:

        that:

          - e2e_suite | default('') in ['', 'onboarding']

          - >-

            e2e_suite | default('') != 'onboarding' or

            (e2e_application_image | default('') | regex_search('^[^\s@]+@sha256:[0-9a-f]{64}$')) is not none

          - >-

            e2e_suite | default('') | length == 0 or

            '--sdk-' not in (e2e_pytest_args | default(''))

        fail_msg: Select onboarding with an application image pinned by sha256 digest; do not combine SDK selectors.

    - name: Carry the inventory's test options to the controller play

      ansible.builtin.set_fact:

        e2e_run_options:

          suite: "{{ e2e_suite | default('') }}"

          application_image: "{{ e2e_application_image | default('') }}"

          python: "{{ e2e_python | default(ansible_playbook_python) }}"

          pytest_args: "{{ e2e_pytest_args | default('') }}"

          pytest_argv: "{{ e2e_pytest_argv | default([]) }}"

          targets: "{{ e2e_pytest_targets | default([]) }}"

          temp_dir: "{{ e2e_temp_dir | default('') }}"

          report_dir: "{{ e2e_report_dir | default('') }}"

    - name: Prepare an explicitly configured controller temporary directory

      ansible.builtin.file:

        path: "{{ e2e_temp_dir }}"

  connection: local

  gather_facts: false

  vars:

    e2e_pytest_args: ""

    e2e_options: "{{ hostvars[groups['oop_platform_operators'] | first].e2e_run_options }}"

    e2e_operators: >-

      {{ groups['oop_platform_operators'] | map('extract', hostvars, 'oop_platform_settings') | list }}

  tasks:

    - name: Require one consistent test selection across the operator inventory

      ansible.builtin.assert:

        that: hostvars[item].e2e_run_options == e2e_options

        fail_msg: Configure the same E2E suite, image, interpreter and report arguments for every operator.

      loop: "{{ groups['oop_platform_operators'] }}"

    - name: Write the settings for the tests

      ansible.builtin.tempfile:

        path: "{{ e2e_temp_dir | default(omit) }}"

        path: "{{ e2e_options.temp_dir | default(omit, true) }}"

        suffix: .json

      register: e2e_settings_file

    - name: Run tests with temporary operator settings

      block:

        - name: Fill in the settings file

          ansible.builtin.copy:

            content: "{{ {'operators': e2e_operators} | to_nice_json }}"

        - name: Run pytest

          ansible.builtin.command:

            argv: >-

          {{ [ansible_playbook_python, '-m', 'pytest', playbook_dir ~ '/../../../tests/r2_e2e', '-v', '-rsxX']

             + (e2e_pytest_argv if e2e_pytest_argv is defined

                else (e2e_pytest_args | split if e2e_pytest_args | length > 0 else [])) }}

              {{ [e2e_options.python, '-m', 'pytest']

                 + (e2e_options.targets if e2e_options.targets | length > 0

                    else [playbook_dir ~ '/../../../tests/r2_e2e'])

                 + ['-v', '-rsxX']

                 + (['--e2e-suite', e2e_options.suite, '--e2e-app-image', e2e_options.application_image]

                    if e2e_options.suite | length > 0 else [])

                 + (e2e_options.pytest_argv if e2e_options.pytest_argv | length > 0

                     else (e2e_options.pytest_args | split if e2e_options.pytest_args | length > 0 else [])) }}

          environment:

            OOP_E2E_SETTINGS: "{{ e2e_settings_file.path }}"

            OOP_E2E_REPORT_DIR: "{{ e2e_options.report_dir }}"

          register: e2e_result

          changed_when: false

          failed_when: false

      always:

        - name: Remove the settings file and the kubeconfig copies

          ansible.builtin.file:

            path: "{{ item }}"

    - name: Show the test report

      ansible.builtin.debug:

        msg: "{{ e2e_result.stdout_lines }}"

        msg: "{{ e2e_result.stdout_lines + e2e_result.stderr_lines }}"

    - name: Fail when a test failed

      ansible.builtin.fail:

   The same script runs read-only in the selected SRM pod. Python/chart contents

   and package version must match; the version alone cannot identify a commit.

   Replace all three SDK fields with the selected candidate; the command shows

   the published rc3 filename as an example, not the final release candidate.

   Replace all SDK fields with the selected candidate; rc3 is an example.

   Record the reviewed source SHA alongside the wheel fingerprint. The test

   verifies both SRM/OEG running image digests before creating a test client.

then removes that client. Cleanup failures are reported alongside the test

failure, and no namespace, platform release or unrelated resource is deleted.

The release SDK can return NodePort endpoints. This scenario still requires

a compatible SRM/OEG image set, explicit control-path configuration and a real

lab run; offline success does not qualify the selected live candidate.

The release SDK can return NodePort endpoints. Verify each fresh candidate

with its reviewed SRM/OEG images and explicit control-path configuration.

## Complete the TI checkpoint separately

settings/profile are exported as CI artifacts. A passing endpoints checkpoint

cannot substitute for Traffic Influence or other release acceptance scenarios.

## Resource quantities

## Public container request

Catalogue and SDK acceptance use the same container manifest builder. The

public Kubernetes shape is requiredResources.infraKind=kubernetes with

isStandalone=false and applicationResources.cpuPool containing integer numCPU,

memory in MB and the required topology. Canonical SDK values such as100m belong

at the SDK boundary, not in this public request.

The default is one CPU core and256MB. An optional resource_requests profile

object can select {"cpu":2,"memory_mb":257}; CPU must be an integer1–256 and

memory_mb an integer1–16384. Invalid values fail during profile loading, before

creating lab clients. This bounded repair fixes request construction; observing

applied Deployment/Pod resources belongs to the next cumulative checkpoint.

The existing `resources` platform checkpoint compares the public application

request with both Deployment and Pod quantities. Set

`acceptance.resource_requests` to `{"cpu": 2, "memory_mb": 257}` in the

application lab profile to cover a non-default CPU request and decimal memory. The shared

manifest builder and existing checkpoint are reused; the default catalogue

manifest remains 1 CPU/256 MB. Current OEG Kubernetes cpuPool requires integer

CPU1–256 and memory1–16384 MB; fractional canonical millicores remain SDK-boundary

coverage rather than an unsupported public scenario. `257Mi` must fail this oracle for a 257 MB request.

## Bootstrap prerequisite

The same role test now reads this example inventory and verifies the generated

path, connection mode, namespace, endpoint host and binding reference without

contacting a cluster. Final current-image OEG acceptance still needs a live lab.

contacting a cluster. Final published-candidate acceptance still needs its exact artifact and image profile.

The application-wide TI recipe above remains unqualified for this release.

Its explicit opt-in gate must stay false until the whole selected route/provider

"""Ordered, cumulative proofs shared by the existing playbook and lab runner."""

PHASES = (

    ("health", "OEG and FM HTTP health", (

        "test_platform.py::test_oeg_and_fm_answer",

    )),

    ("auth", "Authentication and API authorization", (

        "test_platform.py::test_keycloak_issuer_is_the_public_url[oop]",

        "test_platform.py::test_keycloak_issuer_is_the_public_url[federation]",

        "test_auth.py::test_oeg_refuses_calls_without_a_valid_token",

        "test_auth.py::test_oeg_keeps_app_apis_to_app_providers",

    )),

    ("onboarding", "Cloud application catalogue onboarding", (

        "test_apps.py::test_app_provider_registers_reads_and_deletes_an_app",

        "test_apps.py::test_app_provider_lists_its_apps",

    )),

    ("zones", "Configured workload zones exposed by OEG", (

        "test_deploy_to_ti.py::test_cloud_zones_match_configured_workload_targets",

    )),

    ("workload", "OEG request creates a ready Kubernetes workload", (

        "test_deploy_to_ti.py::test_progressive_oeg_request_creates_a_ready_workload",

    )),

    ("resources", "Requested CPU and memory applied to Deployment and Pod", (

        "test_deploy_to_ti.py::test_progressive_workload_applies_requested_resources",

    )),

    ("instance", "OEG returns the deployed application identity", (

        "test_deploy_to_ti.py::test_progressive_oeg_returns_the_application_instance",

    )),

    ("endpoints", "OEG returns the observed application endpoint", (

        "test_deploy_to_ti.py::test_progressive_oeg_returns_the_observed_endpoint",

    )),

    ("reachable", "Returned endpoint reaches the deployed application", (

        "test_deploy_to_ti.py::test_progressive_endpoint_reaches_the_deployed_application",

    )),

    ("terminated", "OEG termination removes the owned workload", (

        "test_deploy_to_ti.py::test_progressive_oeg_termination_removes_the_workload",

    )),

)

def selected_phases(checkpoint):

    names = [key for key, _, _ in PHASES]

    if checkpoint not in names:

        raise ValueError("Unknown platform checkpoint")

    return PHASES[:names.index(checkpoint) + 1]

def selected_nodes(checkpoint):

    return [node for _, _, nodes in selected_phases(checkpoint) for node in nodes]

CHECKPOINTS = {

    key: {node.split("::", 1)[1] for node in selected_nodes(key)}

    for key, _, _ in PHASES

}