(CNIT) Telemetry: per-KPI samples on a NETCONF/OpenConfig collector

Closes #441 (closed)

This is a fix for the KPI collectors. It keeps one queue per KPI, and the backend passes the KPI value to collectors whose GetState() accepts one. This ensures that all KPIs are well tracked to their collectors.

Edited by Lluis Gifre Renom

Merge request reports

Loading
+135 −6
Changes for src/telemetry/backend/service/collectors/netconf_oc/NetconfOCCollector.py: 135 added lines, 6 removed lines.
Original line number Diff line number Diff line
@@ -47,6 +47,42 @@ from .KPI import to_metric_leaf

LOGGER = logging.getLogger(__name__)

# Two metrics polled on the same device (e.g. pre-fec-ber and input-power,
# both on channel-5) have been observed to occasionally return each other's
# value when their NETCONF GETs land close together in wall-clock time --
# each collector's own scheduler already serializes ITS OWN jobs
# (max_workers=1), but a second metric on the same device can be handled by
# a *different* NetconfOpenConfigCollector instance (its own scheduler, its
# own session), so nothing serializes the two devices' polls against each
# other. Staggering each new subscription's first run by
# _DEVICE_POLL_STAGGER_S relative to the others already scheduled for the
# same (address, port) -- and keeping that same stagger on every later
# interval, since they share the same interval_s -- keeps their GETs from
# ever landing at the same moment, regardless of which layer the real race
# is in.
_DEVICE_POLL_STAGGER_S = 3.0
_device_poll_slot_lock = threading.Lock()
_device_poll_slot_count: dict = {}  # (address, port) -> number of subscriptions already scheduled


def _next_poll_offset_s(address: str, port: int) -> float:
    key = (address, port)
    with _device_poll_slot_lock:
        slot = _device_poll_slot_count.get(key, 0)
        _device_poll_slot_count[key] = slot + 1
    return slot * _DEVICE_POLL_STAGGER_S


def _release_poll_offsets(address: str, port: int, count: int = 1) -> None:
    """Hand back `count` stagger slots taken by _next_poll_offset_s()."""
    key = (address, port)
    with _device_poll_slot_lock:
        remaining = _device_poll_slot_count.get(key, 0) - count
        if remaining > 0:
            _device_poll_slot_count[key] = remaining
        else:
            _device_poll_slot_count.pop(key, None)


class NetconfOpenConfigCollector(_Collector):
    """
@@ -60,9 +96,43 @@ class NetconfOpenConfigCollector(_Collector):
        self._hostkey_verify : bool              = settings.get('hostkey_verify', False)
        self._session        : Optional[Manager] = None
        self._session_lock   : threading.Lock    = threading.Lock()
        self._out_samples    : queue.Queue       = queue.Queue()
        # One DriverInstanceCache-cached collector instance is shared by every KPI
        # subscribed on this same device (see DriverInstanceCache: cached by
        # device_uuid). Two KPIs on the same device (e.g. pre-fec-ber and
        # input-power, both on channel-5) therefore each run their own
        # GenericCollectorHandler thread against THIS SAME instance, each with
        # its own GetState() generator. A single shared queue would let either
        # thread dequeue a sample produced for the OTHER KPI's poll job -- that
        # was the actual cause of PRE_FEC_BER alarms showing RECEIVED_POWER
        # values (and vice versa), independent of poll timing. Keeping one
        # queue per kpi_code, and having GetState drain only its own kpi_code's
        # queue, makes the two metrics' sample streams fully independent.
        self._out_samples_by_kpi      : dict               = {}  # kpi_code -> queue.Queue
        self._out_samples_lock        : threading.Lock     = threading.Lock()
        # sub_id -> the stagger slot it holds, so UnsubscribeState and
        # Disconnect can hand the slot back even once APScheduler has
        # dropped the job on its own.
        self._sub_slots               : dict               = {}
        self._sub_slots_lock          : threading.Lock     = threading.Lock()
        self._scheduler      = self._build_scheduler()

    def _release_slots_for(self, resource_key: str) -> None:
        """Return the stagger slots held by the subscriptions matching resource_key."""
        with self._sub_slots_lock:
            sub_ids = [s for s in self._sub_slots if resource_key in s]
            for sub_id in sub_ids:
                self._sub_slots.pop(sub_id, None)
        if sub_ids:
            _release_poll_offsets(self._address, self._port, len(sub_ids))

    def _queue_for_kpi(self, kpi_code) -> queue.Queue:
        with self._out_samples_lock:
            q = self._out_samples_by_kpi.get(kpi_code)
            if q is None:
                q = queue.Queue()
                self._out_samples_by_kpi[kpi_code] = q
            return q

    def _build_scheduler(self) -> BackgroundScheduler:
        scheduler = BackgroundScheduler(daemon=True)
        scheduler.configure(
@@ -154,6 +224,11 @@ class NetconfOpenConfigCollector(_Collector):
                )
            finally:
                self._session = None
        with self._sub_slots_lock:
            held = len(self._sub_slots)
            self._sub_slots.clear()
        if held:
            _release_poll_offsets(self._address, self._port, held)
        self._scheduler = self._build_scheduler()
        return True

@@ -175,7 +250,7 @@ class NetconfOpenConfigCollector(_Collector):
                reply = self._session.get(filter=filter_xml)
            value = PathBuilder.parse(reply.data_ele, channel_name, metric_leaf)
            if value is not None:
                self._out_samples.put((time.time(), channel_name, value))
                self._queue_for_kpi(kpi_code).put((time.time(), channel_name, value))
                LOGGER.debug(
                    "_poll_metric channel=%s metric=%s value=%s", channel_name, metric_leaf, value
                )
@@ -203,19 +278,33 @@ class NetconfOpenConfigCollector(_Collector):
                self._ensure_scheduler_started()
                channel_name = endpoint_dict['endpoint']
                kpi_code     = endpoint_dict['kpi']
                end_date     = datetime.now(pytz.utc) + timedelta(seconds=float(duration_s))
                # duration_s == 0 means "collect indefinitely" (see
                # telemetry_frontend.proto's duration_s doc comment) -- leave
                # end_date as None (APScheduler's "no end") rather than
                # computing one that would land in the past/present.
                end_date     = (
                    datetime.now(pytz.utc) + timedelta(seconds=float(duration_s))
                    if duration_s > 0 else None
                )
                offset_s     = _next_poll_offset_s(self._address, self._port)
                with self._sub_slots_lock:
                    self._sub_slots[sub_id] = offset_s
                first_run    = datetime.now(pytz.utc) + timedelta(seconds=offset_s)
                self._scheduler.add_job(
                    self._poll_metric,
                    trigger         = 'interval',
                    seconds         = float(interval_s),
                    next_run_time   = first_run,
                    args            = [channel_name, kpi_code],
                    id              = sub_id,
                    replace_existing= True,
                    end_date        = end_date,
                )
                LOGGER.info(
                    "SubscribeState: job %s added for channel=%s kpi=%s interval=%.1fs duration=%.1fs",
                    "SubscribeState: job %s added for channel=%s kpi=%s interval=%.1fs duration=%.1fs "
                    "(staggered by %.1fs on %s:%s to avoid overlapping another metric's poll)",
                    sub_id, channel_name, kpi_code, interval_s, duration_s,
                    offset_s, self._address, self._port,
                )
                results.append(True)
            except Exception as exc:
@@ -227,6 +316,15 @@ class NetconfOpenConfigCollector(_Collector):
        """Remove all scheduler jobs whose ID contains resource_key."""
        try:
            job_ids = [job.id for job in self._scheduler.get_jobs() if resource_key in job.id]
            # Give the stagger slots back, or the offset handed to the next
            # subscription on this device grows for the lifetime of the
            # process and its first poll is delayed by that much. This is
            # driven off our own record of the subscriptions rather than off
            # job_ids: a job whose duration_s has elapsed is dropped by
            # APScheduler itself, so by the time TerminateCollector gets here
            # there is nothing left to remove and the slot would never come
            # back.
            self._release_slots_for(resource_key)
            if not job_ids:
                LOGGER.warning(
                    "UnsubscribeState: no active jobs found for key '%s'", resource_key
@@ -249,14 +347,21 @@ class NetconfOpenConfigCollector(_Collector):
        duration  : int,
        blocking  : bool                      = True,
        terminate : Optional[threading.Event] = None,
        kpi_code  : Optional[Any]              = None,
    ) -> Iterator[Tuple[float, str, Any]]:
        """Drain the sample queue, yielding (timestamp, channel_name, value) tuples.
        """Drain this KPI's sample queue, yielding (timestamp, channel_name, value) tuples.

        Args:
            duration:  Maximum seconds to run (same semantics as EmulatedCollector).
            blocking:  If True, block waiting for new samples (generator mode).
                       If False, return once the queue is empty.
            terminate: Optional threading.Event; stop immediately when set.
            kpi_code:  Which KPI's sample stream to drain. This collector instance
                       is shared (via DriverInstanceCache) by every KPI subscribed
                       on this device, so each concurrent caller must pass its own
                       kpi_code to avoid consuming another KPI's samples. If None
                       (caller doesn't know about per-KPI queues), falls back to
                       the single queue in use when only one KPI is subscribed.
        """
        start_time = datetime.now(pytz.utc)
        while True:
@@ -264,13 +369,37 @@ class NetconfOpenConfigCollector(_Collector):
                LOGGER.info("GetState: termination signal received")
                return

            # duration <= 0 means "collect indefinitely" (see
            # telemetry_frontend.proto's duration_s doc comment) -- skip the
            # expiry check entirely rather than expiring on the first loop
            # (elapsed >= 0 would otherwise always be true immediately).
            if duration > 0:
                elapsed = (datetime.now(pytz.utc) - start_time).total_seconds()
                if elapsed >= duration:
                    LOGGER.info("GetState: duration %.1fs expired", duration)
                    return

            if kpi_code is not None:
                out_queue = self._queue_for_kpi(kpi_code)
            else:
                with self._out_samples_lock:
                    queues = list(self._out_samples_by_kpi.values())
                if len(queues) == 1:
                    out_queue = queues[0]
                elif len(queues) == 0:
                    time.sleep(0.1 if not blocking else min(1.0, duration))
                    continue
                else:
                    LOGGER.warning(
                        "GetState: called without kpi_code while %d KPI queues are active on "
                        "%s:%s -- cannot disambiguate, dropping this poll",
                        len(queues), self._address, self._port,
                    )
                    time.sleep(0.1 if not blocking else 1.0)
                    continue

            try:
                sample = self._out_samples.get(block=blocking, timeout=1.0 if blocking else 0.1)
                sample = out_queue.get(block=blocking, timeout=1.0 if blocking else 0.1)
                LOGGER.debug("GetState: yielding sample %s", sample)
                yield sample
            except queue.Empty:
+13 −3
Changes for src/telemetry/backend/service/TelemetryBackendService.py: 13 added lines, 3 removed lines.
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import inspect
import json
import time
import logging
@@ -202,9 +203,18 @@ class TelemetryBackendService(GenericGrpcService):
            subscription[0] for subscription in resource_to_subscribe
        ]
                
        for (timestamp, resource_key, value) in device_collector.GetState(
            duration=duration, blocking=True, terminate=stop_event
        ):
        # Some collectors (e.g. NetconfOpenConfigCollector) are cached and shared
        # by every KPI subscribed on the same device, so several
        # GenericCollectorHandler threads call GetState() on one instance at
        # once. Passing this KPI's kpi_code lets such a collector keep each
        # KPI's samples in its own queue instead of the threads consuming each
        # other's values. Collectors that do not accept kpi_code are called
        # exactly as before.
        get_state_kwargs = {"duration": duration, "blocking": True, "terminate": stop_event}
        if resource_to_subscribe and "kpi_code" in inspect.signature(device_collector.GetState).parameters:
            get_state_kwargs["kpi_code"] = resource_to_subscribe[0][1].get("kpi")

        for (timestamp, resource_key, value) in device_collector.GetState(**get_state_kwargs):
            if stop_event.is_set():
                break
            LOGGER.info(f"KPI ID: {kpi_id} - resource={resource_key} value={value}")
Loading