Commit 9318eab6 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Merge branch 'feat/441-telemetry-netconf-oc-queues' into 'develop'

(CNIT) Telemetry: per-KPI samples on a NETCONF/OpenConfig collector

See merge request !507
parents bec04682 d644d17d
Loading
Loading
Loading
Loading
+13 −3
Changes for src/telemetry/backend/service/TelemetryBackendService.py: 13 added lines, 3 removed lines.
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import inspect
import json
import time
import logging
@@ -202,9 +203,18 @@ class TelemetryBackendService(GenericGrpcService):
            subscription[0] for subscription in resource_to_subscribe
        ]
                
        for (timestamp, resource_key, value) in device_collector.GetState(
            duration=duration, blocking=True, terminate=stop_event
        ):
        # Some collectors (e.g. NetconfOpenConfigCollector) are cached and shared
        # by every KPI subscribed on the same device, so several
        # GenericCollectorHandler threads call GetState() on one instance at
        # once. Passing this KPI's kpi_code lets such a collector keep each
        # KPI's samples in its own queue instead of the threads consuming each
        # other's values. Collectors that do not accept kpi_code are called
        # exactly as before.
        get_state_kwargs = {"duration": duration, "blocking": True, "terminate": stop_event}
        if resource_to_subscribe and "kpi_code" in inspect.signature(device_collector.GetState).parameters:
            get_state_kwargs["kpi_code"] = resource_to_subscribe[0][1].get("kpi")

        for (timestamp, resource_key, value) in device_collector.GetState(**get_state_kwargs):
            if stop_event.is_set():
                break
            LOGGER.info(f"KPI ID: {kpi_id} - resource={resource_key} value={value}")
+135 −6
Changes for src/telemetry/backend/service/collectors/netconf_oc/NetconfOCCollector.py: 135 added lines, 6 removed lines.
Original line number Diff line number Diff line
@@ -47,6 +47,42 @@ from .KPI import to_metric_leaf

LOGGER = logging.getLogger(__name__)

# Two metrics polled on the same device (e.g. pre-fec-ber and input-power,
# both on channel-5) have been observed to occasionally return each other's
# value when their NETCONF GETs land close together in wall-clock time --
# each collector's own scheduler already serializes ITS OWN jobs
# (max_workers=1), but a second metric on the same device can be handled by
# a *different* NetconfOpenConfigCollector instance (its own scheduler, its
# own session), so nothing serializes the two devices' polls against each
# other. Staggering each new subscription's first run by
# _DEVICE_POLL_STAGGER_S relative to the others already scheduled for the
# same (address, port) -- and keeping that same stagger on every later
# interval, since they share the same interval_s -- keeps their GETs from
# ever landing at the same moment, regardless of which layer the real race
# is in.
_DEVICE_POLL_STAGGER_S = 3.0
_device_poll_slot_lock = threading.Lock()
_device_poll_slot_count: dict = {}  # (address, port) -> number of subscriptions already scheduled


def _next_poll_offset_s(address: str, port: int) -> float:
    key = (address, port)
    with _device_poll_slot_lock:
        slot = _device_poll_slot_count.get(key, 0)
        _device_poll_slot_count[key] = slot + 1
    return slot * _DEVICE_POLL_STAGGER_S


def _release_poll_offsets(address: str, port: int, count: int = 1) -> None:
    """Hand back `count` stagger slots taken by _next_poll_offset_s()."""
    key = (address, port)
    with _device_poll_slot_lock:
        remaining = _device_poll_slot_count.get(key, 0) - count
        if remaining > 0:
            _device_poll_slot_count[key] = remaining
        else:
            _device_poll_slot_count.pop(key, None)


class NetconfOpenConfigCollector(_Collector):
    """
@@ -60,9 +96,43 @@ class NetconfOpenConfigCollector(_Collector):
        self._hostkey_verify : bool              = settings.get('hostkey_verify', False)
        self._session        : Optional[Manager] = None
        self._session_lock   : threading.Lock    = threading.Lock()
        self._out_samples    : queue.Queue       = queue.Queue()
        # One DriverInstanceCache-cached collector instance is shared by every KPI
        # subscribed on this same device (see DriverInstanceCache: cached by
        # device_uuid). Two KPIs on the same device (e.g. pre-fec-ber and
        # input-power, both on channel-5) therefore each run their own
        # GenericCollectorHandler thread against THIS SAME instance, each with
        # its own GetState() generator. A single shared queue would let either
        # thread dequeue a sample produced for the OTHER KPI's poll job -- that
        # was the actual cause of PRE_FEC_BER alarms showing RECEIVED_POWER
        # values (and vice versa), independent of poll timing. Keeping one
        # queue per kpi_code, and having GetState drain only its own kpi_code's
        # queue, makes the two metrics' sample streams fully independent.
        self._out_samples_by_kpi      : dict               = {}  # kpi_code -> queue.Queue
        self._out_samples_lock        : threading.Lock     = threading.Lock()
        # sub_id -> the stagger slot it holds, so UnsubscribeState and
        # Disconnect can hand the slot back even once APScheduler has
        # dropped the job on its own.
        self._sub_slots               : dict               = {}
        self._sub_slots_lock          : threading.Lock     = threading.Lock()
        self._scheduler      = self._build_scheduler()

    def _release_slots_for(self, resource_key: str) -> None:
        """Return the stagger slots held by the subscriptions matching resource_key."""
        with self._sub_slots_lock:
            sub_ids = [s for s in self._sub_slots if resource_key in s]
            for sub_id in sub_ids:
                self._sub_slots.pop(sub_id, None)
        if sub_ids:
            _release_poll_offsets(self._address, self._port, len(sub_ids))

    def _queue_for_kpi(self, kpi_code) -> queue.Queue:
        with self._out_samples_lock:
            q = self._out_samples_by_kpi.get(kpi_code)
            if q is None:
                q = queue.Queue()
                self._out_samples_by_kpi[kpi_code] = q
            return q

    def _build_scheduler(self) -> BackgroundScheduler:
        scheduler = BackgroundScheduler(daemon=True)
        scheduler.configure(
@@ -154,6 +224,11 @@ class NetconfOpenConfigCollector(_Collector):
                )
            finally:
                self._session = None
        with self._sub_slots_lock:
            held = len(self._sub_slots)
            self._sub_slots.clear()
        if held:
            _release_poll_offsets(self._address, self._port, held)
        self._scheduler = self._build_scheduler()
        return True

@@ -175,7 +250,7 @@ class NetconfOpenConfigCollector(_Collector):
                reply = self._session.get(filter=filter_xml)
            value = PathBuilder.parse(reply.data_ele, channel_name, metric_leaf)
            if value is not None:
                self._out_samples.put((time.time(), channel_name, value))
                self._queue_for_kpi(kpi_code).put((time.time(), channel_name, value))
                LOGGER.debug(
                    "_poll_metric channel=%s metric=%s value=%s", channel_name, metric_leaf, value
                )
@@ -203,19 +278,33 @@ class NetconfOpenConfigCollector(_Collector):
                self._ensure_scheduler_started()
                channel_name = endpoint_dict['endpoint']
                kpi_code     = endpoint_dict['kpi']
                end_date     = datetime.now(pytz.utc) + timedelta(seconds=float(duration_s))
                # duration_s == 0 means "collect indefinitely" (see
                # telemetry_frontend.proto's duration_s doc comment) -- leave
                # end_date as None (APScheduler's "no end") rather than
                # computing one that would land in the past/present.
                end_date     = (
                    datetime.now(pytz.utc) + timedelta(seconds=float(duration_s))
                    if duration_s > 0 else None
                )
                offset_s     = _next_poll_offset_s(self._address, self._port)
                with self._sub_slots_lock:
                    self._sub_slots[sub_id] = offset_s
                first_run    = datetime.now(pytz.utc) + timedelta(seconds=offset_s)
                self._scheduler.add_job(
                    self._poll_metric,
                    trigger         = 'interval',
                    seconds         = float(interval_s),
                    next_run_time   = first_run,
                    args            = [channel_name, kpi_code],
                    id              = sub_id,
                    replace_existing= True,
                    end_date        = end_date,
                )
                LOGGER.info(
                    "SubscribeState: job %s added for channel=%s kpi=%s interval=%.1fs duration=%.1fs",
                    "SubscribeState: job %s added for channel=%s kpi=%s interval=%.1fs duration=%.1fs "
                    "(staggered by %.1fs on %s:%s to avoid overlapping another metric's poll)",
                    sub_id, channel_name, kpi_code, interval_s, duration_s,
                    offset_s, self._address, self._port,
                )
                results.append(True)
            except Exception as exc:
@@ -227,6 +316,15 @@ class NetconfOpenConfigCollector(_Collector):
        """Remove all scheduler jobs whose ID contains resource_key."""
        try:
            job_ids = [job.id for job in self._scheduler.get_jobs() if resource_key in job.id]
            # Give the stagger slots back, or the offset handed to the next
            # subscription on this device grows for the lifetime of the
            # process and its first poll is delayed by that much. This is
            # driven off our own record of the subscriptions rather than off
            # job_ids: a job whose duration_s has elapsed is dropped by
            # APScheduler itself, so by the time TerminateCollector gets here
            # there is nothing left to remove and the slot would never come
            # back.
            self._release_slots_for(resource_key)
            if not job_ids:
                LOGGER.warning(
                    "UnsubscribeState: no active jobs found for key '%s'", resource_key
@@ -249,14 +347,21 @@ class NetconfOpenConfigCollector(_Collector):
        duration  : int,
        blocking  : bool                      = True,
        terminate : Optional[threading.Event] = None,
        kpi_code  : Optional[Any]              = None,
    ) -> Iterator[Tuple[float, str, Any]]:
        """Drain the sample queue, yielding (timestamp, channel_name, value) tuples.
        """Drain this KPI's sample queue, yielding (timestamp, channel_name, value) tuples.

        Args:
            duration:  Maximum seconds to run (same semantics as EmulatedCollector).
            blocking:  If True, block waiting for new samples (generator mode).
                       If False, return once the queue is empty.
            terminate: Optional threading.Event; stop immediately when set.
            kpi_code:  Which KPI's sample stream to drain. This collector instance
                       is shared (via DriverInstanceCache) by every KPI subscribed
                       on this device, so each concurrent caller must pass its own
                       kpi_code to avoid consuming another KPI's samples. If None
                       (caller doesn't know about per-KPI queues), falls back to
                       the single queue in use when only one KPI is subscribed.
        """
        start_time = datetime.now(pytz.utc)
        while True:
@@ -264,13 +369,37 @@ class NetconfOpenConfigCollector(_Collector):
                LOGGER.info("GetState: termination signal received")
                return

            # duration <= 0 means "collect indefinitely" (see
            # telemetry_frontend.proto's duration_s doc comment) -- skip the
            # expiry check entirely rather than expiring on the first loop
            # (elapsed >= 0 would otherwise always be true immediately).
            if duration > 0:
                elapsed = (datetime.now(pytz.utc) - start_time).total_seconds()
                if elapsed >= duration:
                    LOGGER.info("GetState: duration %.1fs expired", duration)
                    return

            if kpi_code is not None:
                out_queue = self._queue_for_kpi(kpi_code)
            else:
                with self._out_samples_lock:
                    queues = list(self._out_samples_by_kpi.values())
                if len(queues) == 1:
                    out_queue = queues[0]
                elif len(queues) == 0:
                    time.sleep(0.1 if not blocking else min(1.0, duration))
                    continue
                else:
                    LOGGER.warning(
                        "GetState: called without kpi_code while %d KPI queues are active on "
                        "%s:%s -- cannot disambiguate, dropping this poll",
                        len(queues), self._address, self._port,
                    )
                    time.sleep(0.1 if not blocking else 1.0)
                    continue

            try:
                sample = self._out_samples.get(block=blocking, timeout=1.0 if blocking else 0.1)
                sample = out_queue.get(block=blocking, timeout=1.0 if blocking else 0.1)
                LOGGER.debug("GetState: yielding sample %s", sample)
                yield sample
            except queue.Empty: