Commit 35489847 authored by Andrea Sgambelluri's avatar Andrea Sgambelluri
Browse files

telemetry: keep per-KPI samples apart on a shared NETCONF/OpenConfig collector

The NETCONF OpenConfig collector is cached per device, so when several
KPIs are subscribed on the same device every GenericCollectorHandler
thread calls GetState() on one instance and the threads consume each
other's samples. The collector now keeps a queue per KPI, and the
backend passes the KPI code to collectors whose GetState accepts one.
Collectors that do not are called exactly as before.

Also registers the gNMI OpenConfig collector for optical ROADMs and
transponders using the OpenConfig driver, which previously had no
collector registered at all.
parent b3cc3550
Loading
Loading
Loading
Loading
+13 −3
Changes for src/telemetry/backend/service/TelemetryBackendService.py: 13 added lines, 3 removed lines.
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import inspect
import json
import time
import logging
@@ -202,9 +203,18 @@ class TelemetryBackendService(GenericGrpcService):
            subscription[0] for subscription in resource_to_subscribe
        ]
                
        for (timestamp, resource_key, value) in device_collector.GetState(
            duration=duration, blocking=True, terminate=stop_event
        ):
        # Some collectors (e.g. NetconfOpenConfigCollector) are cached and shared
        # by every KPI subscribed on the same device, so several
        # GenericCollectorHandler threads call GetState() on one instance at
        # once. Passing this KPI's kpi_code lets such a collector keep each
        # KPI's samples in its own queue instead of the threads consuming each
        # other's values. Collectors that do not accept kpi_code are called
        # exactly as before.
        get_state_kwargs = {"duration": duration, "blocking": True, "terminate": stop_event}
        if resource_to_subscribe and "kpi_code" in inspect.signature(device_collector.GetState).parameters:
            get_state_kwargs["kpi_code"] = resource_to_subscribe[0][1].get("kpi")

        for (timestamp, resource_key, value) in device_collector.GetState(**get_state_kwargs):
            if stop_event.is_set():
                break
            LOGGER.info(f"KPI ID: {kpi_id} - resource={resource_key} value={value}")
+13 −0
Changes for src/telemetry/backend/service/collectors/__init__.py: 13 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -57,6 +57,19 @@ if LOAD_ALL_DEVICE_DRIVERS:
        ])
    )

if LOAD_ALL_DEVICE_DRIVERS:
    from .gnmi_oc.GnmiOpenConfigCollector import GNMIOpenConfigCollector # pylint: disable=wrong-import-position
    COLLECTORS.append(
        (GNMIOpenConfigCollector, [
            {
                FilterFieldEnum.DEVICE_TYPE: [
                    DeviceTypeEnum.OPTICAL_ROADM,
                    DeviceTypeEnum.OPTICAL_TRANSPONDER
                ],
                FilterFieldEnum.DRIVER     : DeviceDriverEnum.DEVICEDRIVER_OC,
            }
        ]))

if LOAD_ALL_DEVICE_DRIVERS:
    from .netconf_oc.NetconfOCCollector import NetconfOpenConfigCollector # pylint: disable=wrong-import-position
    COLLECTORS.append(
+94 −6
Changes for src/telemetry/backend/service/collectors/netconf_oc/NetconfOCCollector.py: 94 added lines, 6 removed lines.
Original line number Diff line number Diff line
@@ -47,6 +47,31 @@ from .KPI import to_metric_leaf

LOGGER = logging.getLogger(__name__)

# Two metrics polled on the same device (e.g. pre-fec-ber and input-power,
# both on channel-5) have been observed to occasionally return each other's
# value when their NETCONF GETs land close together in wall-clock time --
# each collector's own scheduler already serializes ITS OWN jobs
# (max_workers=1), but a second metric on the same device can be handled by
# a *different* NetconfOpenConfigCollector instance (its own scheduler, its
# own session), so nothing serializes the two devices' polls against each
# other. Staggering each new subscription's first run by
# _DEVICE_POLL_STAGGER_S relative to the others already scheduled for the
# same (address, port) -- and keeping that same stagger on every later
# interval, since they share the same interval_s -- keeps their GETs from
# ever landing at the same moment, regardless of which layer the real race
# is in.
_DEVICE_POLL_STAGGER_S = 3.0
_device_poll_slot_lock = threading.Lock()
_device_poll_slot_count: dict = {}  # (address, port) -> number of subscriptions already scheduled


def _next_poll_offset_s(address: str, port: int) -> float:
    key = (address, port)
    with _device_poll_slot_lock:
        slot = _device_poll_slot_count.get(key, 0)
        _device_poll_slot_count[key] = slot + 1
    return slot * _DEVICE_POLL_STAGGER_S


class NetconfOpenConfigCollector(_Collector):
    """
@@ -60,9 +85,29 @@ class NetconfOpenConfigCollector(_Collector):
        self._hostkey_verify : bool              = settings.get('hostkey_verify', False)
        self._session        : Optional[Manager] = None
        self._session_lock   : threading.Lock    = threading.Lock()
        self._out_samples    : queue.Queue       = queue.Queue()
        # One DriverInstanceCache-cached collector instance is shared by every KPI
        # subscribed on this same device (see DriverInstanceCache: cached by
        # device_uuid). Two KPIs on the same device (e.g. pre-fec-ber and
        # input-power, both on channel-5) therefore each run their own
        # GenericCollectorHandler thread against THIS SAME instance, each with
        # its own GetState() generator. A single shared queue would let either
        # thread dequeue a sample produced for the OTHER KPI's poll job -- that
        # was the actual cause of PRE_FEC_BER alarms showing RECEIVED_POWER
        # values (and vice versa), independent of poll timing. Keeping one
        # queue per kpi_code, and having GetState drain only its own kpi_code's
        # queue, makes the two metrics' sample streams fully independent.
        self._out_samples_by_kpi      : dict               = {}  # kpi_code -> queue.Queue
        self._out_samples_lock        : threading.Lock     = threading.Lock()
        self._scheduler      = self._build_scheduler()

    def _queue_for_kpi(self, kpi_code) -> queue.Queue:
        with self._out_samples_lock:
            q = self._out_samples_by_kpi.get(kpi_code)
            if q is None:
                q = queue.Queue()
                self._out_samples_by_kpi[kpi_code] = q
            return q

    def _build_scheduler(self) -> BackgroundScheduler:
        scheduler = BackgroundScheduler(daemon=True)
        scheduler.configure(
@@ -175,7 +220,7 @@ class NetconfOpenConfigCollector(_Collector):
                reply = self._session.get(filter=filter_xml)
            value = PathBuilder.parse(reply.data_ele, channel_name, metric_leaf)
            if value is not None:
                self._out_samples.put((time.time(), channel_name, value))
                self._queue_for_kpi(kpi_code).put((time.time(), channel_name, value))
                LOGGER.debug(
                    "_poll_metric channel=%s metric=%s value=%s", channel_name, metric_leaf, value
                )
@@ -203,19 +248,31 @@ class NetconfOpenConfigCollector(_Collector):
                self._ensure_scheduler_started()
                channel_name = endpoint_dict['endpoint']
                kpi_code     = endpoint_dict['kpi']
                end_date     = datetime.now(pytz.utc) + timedelta(seconds=float(duration_s))
                # duration_s == 0 means "collect indefinitely" (see
                # telemetry_frontend.proto's duration_s doc comment) -- leave
                # end_date as None (APScheduler's "no end") rather than
                # computing one that would land in the past/present.
                end_date     = (
                    datetime.now(pytz.utc) + timedelta(seconds=float(duration_s))
                    if duration_s > 0 else None
                )
                offset_s     = _next_poll_offset_s(self._address, self._port)
                first_run    = datetime.now(pytz.utc) + timedelta(seconds=offset_s)
                self._scheduler.add_job(
                    self._poll_metric,
                    trigger         = 'interval',
                    seconds         = float(interval_s),
                    next_run_time   = first_run,
                    args            = [channel_name, kpi_code],
                    id              = sub_id,
                    replace_existing= True,
                    end_date        = end_date,
                )
                LOGGER.info(
                    "SubscribeState: job %s added for channel=%s kpi=%s interval=%.1fs duration=%.1fs",
                    "SubscribeState: job %s added for channel=%s kpi=%s interval=%.1fs duration=%.1fs "
                    "(staggered by %.1fs on %s:%s to avoid overlapping another metric's poll)",
                    sub_id, channel_name, kpi_code, interval_s, duration_s,
                    offset_s, self._address, self._port,
                )
                results.append(True)
            except Exception as exc:
@@ -249,14 +306,21 @@ class NetconfOpenConfigCollector(_Collector):
        duration  : int,
        blocking  : bool                      = True,
        terminate : Optional[threading.Event] = None,
        kpi_code  : Optional[Any]              = None,
    ) -> Iterator[Tuple[float, str, Any]]:
        """Drain the sample queue, yielding (timestamp, channel_name, value) tuples.
        """Drain this KPI's sample queue, yielding (timestamp, channel_name, value) tuples.

        Args:
            duration:  Maximum seconds to run (same semantics as EmulatedCollector).
            blocking:  If True, block waiting for new samples (generator mode).
                       If False, return once the queue is empty.
            terminate: Optional threading.Event; stop immediately when set.
            kpi_code:  Which KPI's sample stream to drain. This collector instance
                       is shared (via DriverInstanceCache) by every KPI subscribed
                       on this device, so each concurrent caller must pass its own
                       kpi_code to avoid consuming another KPI's samples. If None
                       (caller doesn't know about per-KPI queues), falls back to
                       the single queue in use when only one KPI is subscribed.
        """
        start_time = datetime.now(pytz.utc)
        while True:
@@ -264,13 +328,37 @@ class NetconfOpenConfigCollector(_Collector):
                LOGGER.info("GetState: termination signal received")
                return

            # duration <= 0 means "collect indefinitely" (see
            # telemetry_frontend.proto's duration_s doc comment) -- skip the
            # expiry check entirely rather than expiring on the first loop
            # (elapsed >= 0 would otherwise always be true immediately).
            if duration > 0:
                elapsed = (datetime.now(pytz.utc) - start_time).total_seconds()
                if elapsed >= duration:
                    LOGGER.info("GetState: duration %.1fs expired", duration)
                    return

            if kpi_code is not None:
                out_queue = self._queue_for_kpi(kpi_code)
            else:
                with self._out_samples_lock:
                    queues = list(self._out_samples_by_kpi.values())
                if len(queues) == 1:
                    out_queue = queues[0]
                elif len(queues) == 0:
                    time.sleep(0.1 if not blocking else min(1.0, duration))
                    continue
                else:
                    LOGGER.warning(
                        "GetState: called without kpi_code while %d KPI queues are active on "
                        "%s:%s -- cannot disambiguate, dropping this poll",
                        len(queues), self._address, self._port,
                    )
                    time.sleep(0.1 if not blocking else 1.0)
                    continue

            try:
                sample = self._out_samples.get(block=blocking, timeout=1.0 if blocking else 0.1)
                sample = out_queue.get(block=blocking, timeout=1.0 if blocking else 0.1)
                LOGGER.debug("GetState: yielding sample %s", sample)
                yield sample
            except queue.Empty: