Commit 573ac8c4 authored by Andrea Sgambelluri's avatar Andrea Sgambelluri
Browse files

telemetry: return a stagger slot when its job expires, not only on unsubscribe

UnsubscribeState released slots from the jobs it removed, but a job whose
duration_s has elapsed is dropped by APScheduler itself, so the call that
TerminateCollector makes afterwards finds nothing to remove, returns early
and the slot is never handed back. Duration-limited collectors are the
common case, so the offset still grew for the life of the process.

Track the slot taken by each subscription id, and release it from that
record in UnsubscribeState and in Disconnect, whichever happens first.
parent 13f56428
Loading
Loading
Loading
Loading
+30 −4
Changes for src/telemetry/backend/service/collectors/netconf_oc/NetconfOCCollector.py: 30 added lines, 4 removed lines.
Original line number Diff line number Diff line
@@ -109,8 +109,22 @@ class NetconfOpenConfigCollector(_Collector):
        # queue, makes the two metrics' sample streams fully independent.
        self._out_samples_by_kpi      : dict               = {}  # kpi_code -> queue.Queue
        self._out_samples_lock        : threading.Lock     = threading.Lock()
        # sub_id -> the stagger slot it holds, so UnsubscribeState and
        # Disconnect can hand the slot back even once APScheduler has
        # dropped the job on its own.
        self._sub_slots               : dict               = {}
        self._sub_slots_lock          : threading.Lock     = threading.Lock()
        self._scheduler      = self._build_scheduler()

    def _release_slots_for(self, resource_key: str) -> None:
        """Return the stagger slots held by the subscriptions matching resource_key."""
        with self._sub_slots_lock:
            sub_ids = [s for s in self._sub_slots if resource_key in s]
            for sub_id in sub_ids:
                self._sub_slots.pop(sub_id, None)
        if sub_ids:
            _release_poll_offsets(self._address, self._port, len(sub_ids))

    def _queue_for_kpi(self, kpi_code) -> queue.Queue:
        with self._out_samples_lock:
            q = self._out_samples_by_kpi.get(kpi_code)
@@ -210,6 +224,11 @@ class NetconfOpenConfigCollector(_Collector):
                )
            finally:
                self._session = None
        with self._sub_slots_lock:
            held = len(self._sub_slots)
            self._sub_slots.clear()
        if held:
            _release_poll_offsets(self._address, self._port, held)
        self._scheduler = self._build_scheduler()
        return True

@@ -268,6 +287,8 @@ class NetconfOpenConfigCollector(_Collector):
                    if duration_s > 0 else None
                )
                offset_s     = _next_poll_offset_s(self._address, self._port)
                with self._sub_slots_lock:
                    self._sub_slots[sub_id] = offset_s
                first_run    = datetime.now(pytz.utc) + timedelta(seconds=offset_s)
                self._scheduler.add_job(
                    self._poll_metric,
@@ -295,6 +316,15 @@ class NetconfOpenConfigCollector(_Collector):
        """Remove all scheduler jobs whose ID contains resource_key."""
        try:
            job_ids = [job.id for job in self._scheduler.get_jobs() if resource_key in job.id]
            # Give the stagger slots back, or the offset handed to the next
            # subscription on this device grows for the lifetime of the
            # process and its first poll is delayed by that much. This is
            # driven off our own record of the subscriptions rather than off
            # job_ids: a job whose duration_s has elapsed is dropped by
            # APScheduler itself, so by the time TerminateCollector gets here
            # there is nothing left to remove and the slot would never come
            # back.
            self._release_slots_for(resource_key)
            if not job_ids:
                LOGGER.warning(
                    "UnsubscribeState: no active jobs found for key '%s'", resource_key
@@ -302,10 +332,6 @@ class NetconfOpenConfigCollector(_Collector):
                return False
            for job_id in job_ids:
                self._scheduler.remove_job(job_id)
            # Give the stagger slots back, or the offset handed to the next
            # subscription on this device grows for the lifetime of the
            # process and its first poll is delayed by that much.
            _release_poll_offsets(self._address, self._port, len(job_ids))
            LOGGER.info("UnsubscribeState: removed jobs %s for key '%s'", job_ids, resource_key)
            return True
        except Exception: