Commit bc9b6a43 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Merge branch 'develop' of ssh://gifrerenom_labs.etsi.org/tfs/controller into...

Merge branch 'develop' of ssh://gifrerenom_labs.etsi.org/tfs/controller into feat/389-cttc-spine-leaf-component
parents 49086224 1ba39bfe
Loading
Loading
Loading
Loading
+5 −1
Changes for src/automation/service/zsm_handlers/PluggablesZSMPlugin.py: 5 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -71,7 +71,11 @@ class PluggablesZSMPlugin(_ZSMHandler):
            LOGGER.info(f'Policy rule state:\n{policy_rule_state}')
        except Exception as ex:
            LOGGER.exception(f'Unable to create policy:\n{str(request.policy)}')
            LOGGER.exception(ex.code())
            # RetryDecorator raises a plain Exception (not grpc.RpcError) once
            # retries are exhausted, so .code() is only safe to call here when
            # the original gRPC error surfaced directly.
            if isinstance(ex, grpc.RpcError):
                LOGGER.exception(ex.code())  # pylint: disable=no-member
            # ToDo: Investigate why PolicyAddService throws exception
            # if ex.code() != grpc.StatusCode.NOT_FOUND: raise  # pylint: disable=no-member
            context_client.close()
+13 −1
Changes for src/opticalcontroller/Dockerfile: 13 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -20,7 +20,7 @@ RUN apt-get --yes --quiet --quiet update && \
    rm -rf /var/lib/apt/lists/*

# Set Python to show logs as they occur
ENV PYTHONUNBUFFERED=0
ENV PYTHONUNBUFFERED=1

# Download the gRPC health probe
RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \
@@ -64,6 +64,18 @@ RUN python3 -m pip install -r requirements.txt
WORKDIR /var/teraflow
COPY src/context/__init__.py context/__init__.py
COPY src/context/client/. context/client/
# Clients the alarm engine talks to: KPI manager, telemetry and analytics
# frontends, and automation.
COPY src/kpi_manager/__init__.py kpi_manager/__init__.py
COPY src/kpi_manager/client/. kpi_manager/client/
COPY src/telemetry/__init__.py telemetry/__init__.py
COPY src/telemetry/frontend/__init__.py telemetry/frontend/__init__.py
COPY src/telemetry/frontend/client/. telemetry/frontend/client/
COPY src/analytics/__init__.py analytics/__init__.py
COPY src/analytics/frontend/__init__.py analytics/frontend/__init__.py
COPY src/analytics/frontend/client/. analytics/frontend/client/
COPY src/automation/__init__.py automation/__init__.py
COPY src/automation/client/. automation/client/
COPY src/opticalcontroller/. opticalcontroller/

# Start the service
+4 −0
Changes for src/opticalcontroller/OpticalController.py: 4 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -583,5 +583,9 @@ class ComputeOpticalConnectivityCandidates(Resource):
from opticalcontroller import p2mp_routes          # noqa: E402
p2mp_routes.register(api, optical)

# Alarm, subscription and recompute resources.
from opticalcontroller import alarm_routes           # noqa: E402
alarm_routes.register(api, optical)

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=10060, debug=True)
+865 −0

File added.

Preview size limit exceeded, changes collapsed.

+328 −0
Changes for src/opticalcontroller/alarm_failover.py: 328 added lines, 0 removed lines.
Original line number Diff line number Diff line
# Copyright 2022-2026 ETSI SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Automatic failover of a monitored optical device to a backup.

A *protection policy* names, for each protected (primary) device, the backup
device its traffic is recomputed onto when the primary raises its first
confirmed alarm, and the resting optical-power band each device sits in when
idle. Everything is keyed by device name, so any number of primary/backup
pairs can be protected at once.

The policy is read from OPTICAL_FAILOVER_POLICY as JSON:

    {"T1.1": {"backup": "T1.2",
              "resting-power-dbm": [-6.0, -3.0],
              "primary-resting-power-dbm": [-9.0, -8.0]}}

With no policy configured nothing fails over and every entry point here is a
no-op, so the alarm API works unchanged on deployments that do not use
protection.

State machine, per protected pair:

    primary monitored
        -- first confirmed alarm on the primary -->  recompute onto backup,
                                                     monitor backup, backup muted
        -- backup shows a genuine in-range reading -->  backup confirmed
        -- backup settles to its resting band     -->  revert to primary
        -- path carrying the backup is deleted    -->  revert to primary
        -- a new path is computed from the primary -->  revert to primary

"Confirmed" is the shared per-device gate in alarm_engine
(_record_breach_state_and_check_confirmed_up): a backup is trusted only once
a genuine non-zero in-range PRE_FEC_BER has been seen together with an
in-range RECEIVED_POWER, so a device cannot confirm itself off its startup
defaults.
"""

import json, logging, os, threading

from opticalcontroller import alarm_engine

LOGGER = logging.getLogger(__name__)

# Resting band used for a device the policy gives no explicit one for.
_DEFAULT_RESTING_POWER_RANGE_DBM = (-9.0, -8.0)

# Guards every read-then-write of the failover state below. Re-entrant
# because a revert can be reached from inside a switch.
_failover_lock = threading.RLock()


def _load_policy() -> dict:
    """{primary: {'backup', 'resting', 'primary_resting'}} from the environment."""
    raw = os.environ.get('OPTICAL_FAILOVER_POLICY', '').strip()
    if not raw:
        return {}
    try:
        parsed = json.loads(raw)
    except ValueError as e:
        LOGGER.warning("OPTICAL_FAILOVER_POLICY is not valid JSON, ignoring it: %s", e)
        return {}
    if not isinstance(parsed, dict):
        LOGGER.warning("OPTICAL_FAILOVER_POLICY is not a JSON object, ignoring it.")
        return {}

    def _range(value, default):
        try:
            low, high = value
            return (float(low), float(high))
        except (TypeError, ValueError):
            return default

    policy = {}
    for primary, entry in parsed.items():
        if not isinstance(entry, dict) or not entry.get('backup'):
            LOGGER.warning("OPTICAL_FAILOVER_POLICY entry for %s has no backup device, skipping it.",
                           primary)
            continue
        policy[str(primary)] = {
            'backup'          : str(entry['backup']),
            'resting'         : _range(entry.get('resting-power-dbm'),
                                       _DEFAULT_RESTING_POWER_RANGE_DBM),
            'primary_resting' : _range(entry.get('primary-resting-power-dbm'),
                                       _DEFAULT_RESTING_POWER_RANGE_DBM),
        }
    return policy


_POLICY = _load_policy()

# primary device -> the device currently monitored on its behalf (the primary
# itself, or its backup once a failover has happened).
_monitored = {primary: primary for primary in _POLICY}


def resting_power_range(device_name : str):
    """The band device_name sits in when idle, whether it is a primary or a backup."""
    for primary, entry in _POLICY.items():
        if device_name == primary:
            return entry['primary_resting']
        if device_name == entry['backup']:
            return entry['resting']
    return _DEFAULT_RESTING_POWER_RANGE_DBM


def is_monitored(device_name : str) -> bool:
    """True if device_name is the device currently monitored for some primary."""
    with _failover_lock:
        return device_name in _monitored.values()


def _primary_monitoring(device_name : str):
    """The primary device_name is currently being monitored on behalf of, if any."""
    for primary, monitored in _monitored.items():
        if monitored == device_name:
            return primary
    return None


def is_muted(device_name : str) -> bool:
    """True while device_name is a backup that has been switched to but has
    not yet proven itself healthy. Its alarms are suppressed until then."""
    with _failover_lock:
        primary = _primary_monitoring(device_name)
        if primary is None or primary == device_name:
            return False        # not a backup, or not failed over
        return not alarm_engine.is_confirmed_up(device_name)


def is_at_resting_baseline(device_name : str) -> bool:
    """True if device_name's last PRE_FEC_BER is exactly 0.0 while its last
    RECEIVED_POWER sits in its own resting band -- i.e. it is showing its
    idle default rather than a genuinely observed healthy sample. This pair
    trivially satisfies the much wider acceptable ranges, so without it a
    device could confirm itself up having never carried a real signal."""
    last_ber   = alarm_engine.last_pre_fec_ber(device_name)
    last_power = alarm_engine.last_received_power(device_name)
    if last_ber != 0.0 or last_power is None:
        return False
    low, high = resting_power_range(device_name)
    return low <= last_power <= high


def update_backup_confirmation_state(device_name : str, kpi_name_upper : str) -> None:
    """Drive the mute -> confirm -> revert cycle for a backup device.

    Called for every RECEIVED_POWER / PRE_FEC_BER sample. Does nothing unless
    device_name is a backup that is currently being monitored after a
    failover. Confirmation itself is the shared gate in alarm_engine, so a
    backup is trusted on exactly the same evidence as any other device; once
    confirmed, settling back to its resting band is read as "this path is
    quiet again" and reverts monitoring to the primary.
    """
    if "RECEIVED_POWER" not in kpi_name_upper and "PRE_FEC_BER" not in kpi_name_upper:
        return

    with _failover_lock:
        primary = _primary_monitoring(device_name)
        if primary is None or primary == device_name:
            return      # not a backup, or no failover in effect

        ber   = alarm_engine.last_pre_fec_ber(device_name)
        power = alarm_engine.last_received_power(device_name)
        if ber is None or power is None:
            return      # need both at least once before deciding anything

        if not alarm_engine.is_confirmed_up(device_name):
            if alarm_engine.record_breach_state_and_check_confirmed_up(device_name):
                LOGGER.info(
                    "Backup %s confirmed up (PRE_FEC_BER=%s, RECEIVED_POWER=%s) -- "
                    "alarm monitoring now trusted for it.", device_name, ber, power,
                )
            else:
                LOGGER.debug(
                    "Backup %s not yet confirmed up (PRE_FEC_BER=%s, RECEIVED_POWER=%s) "
                    "-- still muted.", device_name, ber, power,
                )
        elif ber == 0.0 and is_at_resting_baseline(device_name):
            LOGGER.info(
                "Backup %s settled back to its resting baseline (PRE_FEC_BER=0.0, "
                "RECEIVED_POWER=%s in %s) -- reverting monitoring to %s.",
                device_name, power, resting_power_range(device_name), primary,
            )
            _revert_to_primary(primary)


def switch_to_backup(notification : dict) -> None:
    """Fail the alarming device over to its backup and recompute its path.

    Called once a genuinely new alarm has actually been delivered to a
    subscriber. Performs the real path recompute (the same logic
    /RecomputeSubflow uses) and then starts monitoring the backup on the same
    thresholds. On success it starts the recompute cooldown, so a flood of
    external recompute calls reacting to the same alarm cannot re-churn the
    path.

    Latches per primary: once switched, that primary's later alarms -- now
    resolved against the backup -- do not re-trigger it.
    """
    inner = notification.get("tapi-notification:notification", {})
    failed_src = inner.get("additional-info", {}).get("source-device")
    if not failed_src:
        return

    with _failover_lock:
        primary = _primary_monitoring(failed_src)
        if primary is None or primary != failed_src:
            return      # not a protected primary, or already failed over
        backup = _POLICY[primary]['backup']

        dest, record = next(
            ((d, rec) for d, rec in alarm_engine.active_path_computations().items()
             if failed_src in rec.get("sources", [])),
            (None, None),
        )

        if record is None:
            LOGGER.warning(
                "First confirmed alarm for %s -- switching monitoring to %s, but no "
                "active path currently carries %s, so there is nothing to recompute.",
                failed_src, backup, failed_src,
            )
        else:
            # Best effort: a recompute failure must never stop the monitoring
            # switch below from happening.
            try:
                ok, result = alarm_engine.recompute_subflow(
                    flow_id         = record.get("flow_id"),
                    failed_src      = failed_src,
                    replacement_src = backup,
                    dst             = dest,
                    bitrate         = record.get("bitrate"),
                    bidir           = 1 if record.get("bidirectional", True) else 0,
                    frequency       = record.get("frequency"),
                    band            = record.get("band"),
                    subcarriers     = alarm_engine.subcarriers_for_source(record, failed_src),
                )
            except Exception as e:
                ok, result = False, {"error": str(e)}
                LOGGER.exception(
                    "First confirmed alarm for %s -- the path recompute raised "
                    "unexpectedly; continuing with the monitoring switch anyway.",
                    failed_src,
                )

            if ok:
                alarm_engine.mark_recompute_performed(dest)
                LOGGER.info(
                    "First confirmed alarm for %s -- switched monitoring to %s. "
                    "Recomputed path:\n%s",
                    failed_src, backup, json.dumps(result, indent=2, default=str),
                )
            else:
                LOGGER.warning(
                    "First confirmed alarm for %s -- switched monitoring to %s, but "
                    "the path recompute failed: %s", failed_src, backup, result,
                )

        alarm_engine.start_device_monitoring(backup)
        _monitored[primary] = backup


def _revert_to_primary(primary : str) -> None:
    """Restart monitoring on primary and clear its failover state.

    Re-checks under the lock rather than trusting the caller: every caller
    checks and then calls from outside any lock, so two of them could
    otherwise both restart the primary's monitoring.

    Evicts the primary from the confirmed-up gate first, so a restart is a
    genuine fresh start: it has to satisfy the baseline gate again from
    scratch rather than trading on status earned before it failed over.
    """
    with _failover_lock:
        if _monitored.get(primary, primary) == primary:
            return      # already reverted, or never failed over
        alarm_engine.reset_confirmed_up_gate(primary)
        alarm_engine.start_device_monitoring(primary)
        _monitored[primary] = primary


def maybe_revert_on_source_recompute(sources) -> None:
    """A new path computed from a primary is an explicit signal it is back in
    service: revert monitoring to it. Called after ComputeP2MP or PathRecomp
    successfully registers a path."""
    for primary in list(_POLICY):
        if primary in (sources or []) and _monitored.get(primary, primary) != primary:
            LOGGER.info("Path computed from %s while monitoring %s -- reverting to %s.",
                        primary, _monitored[primary], primary)
            _revert_to_primary(primary)


def restore_after_delete(deleted : dict) -> None:
    """Path-delete counterpart to switch_to_backup: if a deleted path was
    carrying a backup, that recomputed path is gone, so restore monitoring to
    the primary and re-arm the latch."""
    for primary in list(_POLICY):
        monitored = _monitored.get(primary, primary)
        if monitored == primary:
            continue
        if any(monitored in record.get("sources", []) for record in deleted.values()):
            LOGGER.info("Path delete removed the path carrying backup %s -- restoring "
                        "monitoring to %s.", monitored, primary)
            _revert_to_primary(primary)


def role_of(device_name : str):
    """(role, primary) describing device_name's place in the protection
    policy: ("protected", primary) for a primary, ("backup", primary) for a
    backup, or (None, None) for a device the policy does not cover."""
    for primary, entry in _POLICY.items():
        if device_name == primary:
            return ("protected", primary)
        if device_name == entry['backup']:
            return ("backup", primary)
    return (None, None)
Loading