Commit c885988e authored by Konstantinos Togias's avatar Konstantinos Togias
Browse files

ci: consolidate actual release feature reports across lab variants

parent 0e1d24c9

.gitlab-ci.yml

+42 −15
Changes for .gitlab-ci.yml: 42 added lines, 15 removed lines.
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ image: python:3.12-slim 
stages:

  - lint

  - test

  - report



variables:

  ANSIBLE_CONFIG: ansible/ansible.cfg
@@ -148,6 +149,7 @@ platform-lab: 
  cache: []

  timeout: 40 minutes

  variables:

    OOP_LAB_REPORT_DIR: .cache/platform-lab/container

    FF_NETWORK_PER_BUILD: "true"

    DOCKER_HOST: tcp://docker:2376

    DOCKER_TLS_CERTDIR: /certs
@@ -168,26 +170,26 @@ platform-lab: 
  before_script:

    - apk add --no-cache python3

  script:

    - sh ci/run_platform_lab.sh "${OOP_LAB_PROFILE:-ci/lab/profile.health.json}" .cache/platform-lab

    - sh ci/run_platform_lab.sh "${OOP_LAB_PROFILE:-ci/lab/profile.health.json}" "$OOP_LAB_REPORT_DIR"

  artifacts:

    when: always

    paths:

      - .cache/platform-lab/result.json

      - .cache/platform-lab/junit.xml

      - .cache/platform-lab/chart.json

      - .cache/platform-lab/candidate.json

      - .cache/platform-lab/feature-matrix.json

      - .cache/platform-lab/feature-matrix.md

      - .cache/platform-lab/wrapper-junit.xml

      - .cache/platform-lab/pods.txt

      - .cache/platform-lab/events.txt

      - .cache/platform-lab/health/result.json

      - .cache/platform-lab/health/junit.xml

      - .cache/platform-lab/health/acceptance/

      - $OOP_LAB_REPORT_DIR/result.json

      - $OOP_LAB_REPORT_DIR/junit.xml

      - $OOP_LAB_REPORT_DIR/chart.json

      - $OOP_LAB_REPORT_DIR/candidate.json

      - $OOP_LAB_REPORT_DIR/feature-matrix.json

      - $OOP_LAB_REPORT_DIR/feature-matrix.md

      - $OOP_LAB_REPORT_DIR/wrapper-junit.xml

      - $OOP_LAB_REPORT_DIR/pods.txt

      - $OOP_LAB_REPORT_DIR/events.txt

      - $OOP_LAB_REPORT_DIR/health/result.json

      - $OOP_LAB_REPORT_DIR/health/junit.xml

      - $OOP_LAB_REPORT_DIR/health/acceptance/

    reports:

      junit:

        - .cache/platform-lab/junit.xml

        - .cache/platform-lab/wrapper-junit.xml

        - $OOP_LAB_REPORT_DIR/junit.xml

        - $OOP_LAB_REPORT_DIR/wrapper-junit.xml

    expire_in: 90 days



# Same lab and lifecycle, using a chart archive instead of a container manifest.
@@ -195,6 +197,7 @@ platform-lab-helm: 
  extends: platform-lab

  variables:

    OOP_LAB_PACKAGE_TYPE: HELM

    OOP_LAB_REPORT_DIR: .cache/platform-lab/helm

  rules:

    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "test/platform-application-checkpoints" || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "test/platform-application-checkpoints"'

      variables:
@@ -208,6 +211,7 @@ platform-lab-network: 
  extends: platform-lab

  variables:

    OOP_LAB_NETWORK_PROVIDER: EURECOM

    OOP_LAB_REPORT_DIR: .cache/platform-lab/network

  rules:

    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "test/platform-application-checkpoints" || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "test/platform-application-checkpoints"'

      variables:
@@ -216,6 +220,29 @@ platform-lab-network: 
    - if: '$RUN_PLATFORM_NETWORK_LAB == "1" && $CI_COMMIT_REF_PROTECTED == "true"'

    - when: never



# Consolidation consumes real job artifacts; it adds no approval or input form.

release-feature-report:

  stage: report

  image: python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1

  cache: []

  when: always

  needs:

    - {job: platform-lab, artifacts: true, optional: true}

    - {job: platform-lab-helm, artifacts: true, optional: true}

    - {job: platform-lab-network, artifacts: true, optional: true}

  rules:

    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "test/platform-application-checkpoints" || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "test/platform-application-checkpoints"'

    - when: never

  script:

    - python ci/merge_feature_reports.py .cache/platform-lab .cache/release-features

  artifacts:

    when: always

    expose_as: Observed release features

    paths:

      - .cache/release-features/feature-matrix.md

      - .cache/release-features/feature-matrix.json

    expire_in: 90 days



# Explicit live mode leaves all default component tests and deployment jobs alone.

sdk-live-acceptance:

  stage: test

ansible/tests/unit/test_feature_consolidation.py

0 → 100644
+51 −0
Changes for ansible/tests/unit/test_feature_consolidation.py: 51 added lines, 0 removed lines.
Original line number Diff line number Diff line 
"""A release summary must preserve actual scopes and reject mixed/missing evidence."""



import json

from pathlib import Path

import sys



import pytest



sys.path.insert(0,str(Path(__file__).resolve().parents[3]/'ci'))

from merge_feature_reports import consolidate





def inputs(path, *, fault=None):

    for variant,checkpoint in [('container','terminated'),('helm','helm-terminated'),('network','network')]:

        if fault=='missing' and variant=='network':continue

        directory=path/variant;directory.mkdir()

        body={'checkpoint':checkpoint,'ansible_source':'a'*40,'pipeline_url':'http://ci/pipeline/1',

              'job_url':'http://ci/job/'+variant,'wrapper_verified':True,'candidate_run_passed':True,

              'images':{'srm':'sha256:immutable'},'chart_revision':'b'*40,

              'artifact_selection':{'source_heads':{'srm':'c'*40},'sdk':{'fingerprint':'same'}},

              'features':[{'feature':name,'behavior':name,'status':

                  'passed' if name=='resources' and variant=='container' or name=='helm' and variant=='helm'

                  or name=='qod' and variant=='network' else 'not_run',

                  'scope':'bounded '+name,'tests':[],'observed_tests':1}

                  for name in ['resources','helm','qod','updates']]}

        if variant=='network':

            body['provider']={'images':{'nef':'sha256:provider'},'cleanup':'passed'}

            if fault=='failed':body['candidate_run_passed']=False

            if fault=='source':body['ansible_source']='old'

            if fault=='pipeline':body['pipeline_url']='another'

            if fault=='candidate':body['artifact_selection']['source_heads']['srm']='changed'

        (directory/'feature-matrix.json').write_text(json.dumps(body))





def test_summary_chooses_the_actual_feature_variant_and_preserves_limits(tmp_path):

    inputs(tmp_path)

    report=consolidate(tmp_path,tmp_path/'out',source='a'*40,pipeline='http://ci/pipeline/1')

    assert report['selected_acceptance_passed'] and report['same_candidate']

    rows={row['feature']:row for row in report['features']}

    assert rows['resources']['status']==rows['helm']['status']==rows['qod']['status']=='passed'

    assert rows['updates']['status']=='not_run'

    assert rows['qod']['scope']=='bounded qod' and rows['qod']['variant']=='network'

    assert report['compatibility'][0]['provider']['images']['nef']=='sha256:provider'





@pytest.mark.parametrize('fault',['missing','failed','source','pipeline','candidate'])

def test_failed_or_mixed_reports_never_qualify_the_candidate(tmp_path,fault):

    inputs(tmp_path,fault=fault)

    report=consolidate(tmp_path,tmp_path/'out',source='a'*40,pipeline='http://ci/pipeline/1')

    assert not report['selected_acceptance_passed'] and report['problems']

    assert (tmp_path/'out/feature-matrix.md').is_file()

ci/lab/README.md

+13 −0
Changes for ci/lab/README.md: 13 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -92,6 +92,19 @@ listed phase. Each invocation creates and removes its own lab. Keep the default 
health checkpoint unchanged; refresh and review candidate pins before qualification

of a later release image.



On the release branch/MR and integration pushes, the small

`release-feature-report` job consumes the actual container, Helm and network

artifacts after execution, including failed runs. Separate report directories

prevent artifact overwrites. It produces one Markdown/JSON feature table exposed

from the MR, retaining each observed phase, source transaction and scope limit.

It checks all three reports identify the same source, pipeline, images, chart

and SDK contents. Missing reports, mixed candidates or failed transactions fail

the summary; earlier proofs remain visible. No manual input or extra approval

step is introduced. Local runs can use `python ci/merge_feature_reports.py

/path/to/variants /path/to/summary` after placing each run under container, helm

or network. The summary qualifies its selected acceptance scopes, not release

approval or unexecuted features.



For the Helm packaging variant, reuse that same profile and controller:



```sh

ci/merge_feature_reports.py

0 → 100644
+96 −0
Changes for ci/merge_feature_reports.py: 96 added lines, 0 removed lines.
Original line number Diff line number Diff line 
"""Consolidate actual lab artifacts without mixing candidates or inferring coverage."""



import argparse

from copy import deepcopy

import json

import os

from pathlib import Path



VARIANTS = {'container':'terminated', 'helm':'helm-terminated', 'network':'network'}

SPECIAL = {'helm':'helm', 'qod':'network', 'location':'network', 'device-ti':'network'}





def consolidate(directory, output, *, source=None, pipeline=None):

    directory, output=Path(directory),Path(output)

    reports, problems={},[]

    for variant,checkpoint in VARIANTS.items():

        path=directory/variant/'feature-matrix.json'

        if not path.is_file():

            problems.append(f'{variant}: missing actual feature report')

            continue

        report=json.loads(path.read_text())

        reports[variant]=report

        if report.get('checkpoint')!=checkpoint:

            problems.append(f'{variant}: unexpected selected scenario')

        if source and report.get('ansible_source')!=source:

            problems.append(f'{variant}: report belongs to another test source')

        if pipeline and report.get('pipeline_url')!=pipeline:

            problems.append(f'{variant}: report belongs to another pipeline')

        if not report.get('wrapper_verified') or not report.get('candidate_run_passed'):

            problems.append(f'{variant}: selected transaction or cleanup failed')

    identities=[]

    for variant,report in reports.items():

        selection=report.get('artifact_selection') or {}

        heads=selection.get('source_heads')

        if not heads:

            problems.append(f'{variant}: missing resolved artifact provenance')

        identities.append((heads,report.get('images'),report.get('chart_revision'),selection.get('sdk')))

    same_candidate=len(reports)==len(VARIANTS) and all(identity==identities[0] for identity in identities)

    if not same_candidate:

        problems.append('Reports do not establish one common artifact candidate')

    keys={row['feature'] for report in reports.values() for row in report['features']}

    rows=[]

    for key in sorted(keys):

        variant=SPECIAL.get(key,'container')

        report=reports.get(variant,{})

        matches=[row for row in report.get('features',[]) if row['feature']==key]

        if len(matches)>1:

            raise ValueError('Duplicate feature identity in an actual report')

        row=deepcopy(matches[0]) if matches else {

            'feature':key,'behavior':key,'status':'not_run','scope':'Selected report unavailable',

            'tests':[],'observed_tests':0}

        row.update(variant=variant,job_url=report.get('job_url'),

                   transaction_passed=report.get('candidate_run_passed',False))

        rows.append(row)

    network=reports.get('network',{})

    result={'schema_version':1,'ansible_source':source,'pipeline_url':pipeline,

        'same_candidate':same_candidate,'selected_acceptance_passed':not problems,

        'problems':problems,'features':rows,

        'variants':{key:{field:report.get(field) for field in (

            'checkpoint','job_url','candidate_run_passed','lab_cleanup','wrapper_cleanup_passed')}

            for key,report in reports.items()},

        'artifact_selection':reports.get('container',{}).get('artifact_selection'),

        'compatibility':[{'provider':network['provider'],

                          'selected_transaction_passed':network.get('candidate_run_passed',False),

                          'features':{key:next((r['status'] for r in rows if r['feature']==key),'not_run')

                                      for key in ('qod','location','device-ti')}}]

                         if network.get('provider') else []}

    output.mkdir(parents=True,exist_ok=True)

    (output/'feature-matrix.json').write_text(json.dumps(result,indent=2)+'\n')

    lines=['# Observed release features','',

        f"Selected acceptance runs passed: **{result['selected_acceptance_passed']}**. "

        f"Same artifact candidate: **{same_candidate}**.",

        'This report records executed scopes; release approval remains separate.','',

        '| Feature | Observed | Source transaction | Verified scope and limits |',

        '|---|---|---|---|']

    for row in rows:

        link=f"[{row['variant']}]({row['job_url']})" if row['job_url'] else row['variant']

        state='passed' if row['transaction_passed'] else 'failed or missing'

        lines.append(f"| {row['behavior']} | {row['status']} | {link}: {state} | {row['scope']} |")

    if problems:

        lines+=['','Candidate limitations:','',*('- '+problem for problem in problems)]

    lines+=['','Exact artifact and provider identities are retained in feature-matrix.json.',

            'Unselected features remain unverified. API compatibility does not imply radio or user-plane effects.','']

    (output/'feature-matrix.md').write_text('\n'.join(lines))

    return result





if __name__=='__main__':

    parser=argparse.ArgumentParser(description=__doc__)

    parser.add_argument('input',type=Path)

    parser.add_argument('output',type=Path)

    args=parser.parse_args()

    result=consolidate(args.input,args.output,source=os.environ.get('CI_COMMIT_SHA'),

                       pipeline=os.environ.get('CI_PIPELINE_URL'))

    print((args.output/'feature-matrix.md').read_text())

    raise SystemExit(0 if result['selected_acceptance_passed'] else 1)