Commit 52767398 authored by Andrea Sgambelluri's avatar Andrea Sgambelluri
Browse files

policy: forward every optical KPI sample to the optical controller

AlarmListener forwards all samples, breach or not, so the optical
controller evaluates the configured ranges itself rather than trusting
whichever threshold Analytics happens to be running with. The
destination is configured by tfs.optical-controller.alarm-notification-url.

Keeps develop's GENERATE_FAILURE_NOTIFICATION cases; the action does not
call the forwarder a second time, which would double-post each triggered
alarm.

Also fixes a crash in PluggablesZSMPlugin: RetryDecorator raises a plain
Exception once retries are exhausted, so calling .code() on it raised
AttributeError while logging the original failure.
parent 603681de
Loading
Loading
Loading
Loading
+5 −1
Changes for src/automation/service/zsm_handlers/PluggablesZSMPlugin.py: 5 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -71,7 +71,11 @@ class PluggablesZSMPlugin(_ZSMHandler):
            LOGGER.info(f'Policy rule state:\n{policy_rule_state}')
        except Exception as ex:
            LOGGER.exception(f'Unable to create policy:\n{str(request.policy)}')
            LOGGER.exception(ex.code())
            # RetryDecorator raises a plain Exception (not grpc.RpcError) once
            # retries are exhausted, so .code() is only safe to call here when
            # the original gRPC error surfaced directly.
            if isinstance(ex, grpc.RpcError):
                LOGGER.exception(ex.code())  # pylint: disable=no-member
            # ToDo: Investigate why PolicyAddService throws exception
            # if ex.code() != grpc.StatusCode.NOT_FOUND: raise  # pylint: disable=no-member
            context_client.close()
+0 −1
Changes for src/opticalcontroller/alarm_routes.py: 0 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -967,4 +967,3 @@ def register(api, optical):

    optical.route('/restconf/operations/tapi-notification:get-subscriptions', methods=['GET'])(TapiGetSubscriptions)
    optical.response(200, 'Subscription list')(TapiGetSubscriptions)
+125 −0
Changes for src/policy/src/main/java/org/etsi/tfs/policy/policy/CommonPolicyServiceImpl.java: 125 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -20,13 +20,24 @@ import static org.etsi.tfs.policy.common.ApplicationProperties.ACTIVE_POLICYRULE
import static org.etsi.tfs.policy.common.ApplicationProperties.ENFORCED_POLICYRULE_STATE;
import static org.etsi.tfs.policy.common.ApplicationProperties.INVALID_MESSAGE;

import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpRequest.BodyPublishers;
import java.net.http.HttpResponse.BodyHandlers;
import java.time.Duration;
// import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
// import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.etsi.tfs.policy.context.ContextService;
import org.etsi.tfs.policy.context.model.ConfigActionEnum;
import org.etsi.tfs.policy.context.model.ConfigRule;
@@ -37,6 +48,7 @@ import org.etsi.tfs.policy.context.model.ConstraintCustom;
import org.etsi.tfs.policy.context.model.ConstraintTypeCustom;
import org.etsi.tfs.policy.context.model.ServiceConfig;
import org.etsi.tfs.policy.device.DeviceService;
import org.etsi.tfs.policy.policy.model.AlarmTopicDTO;
import org.etsi.tfs.policy.policy.model.PolicyRule;
import org.etsi.tfs.policy.policy.model.PolicyRuleAction;
import org.etsi.tfs.policy.policy.model.PolicyRuleActionConfig;
@@ -58,6 +70,31 @@ public class CommonPolicyServiceImpl {
    @Inject private ServiceService serviceService;
    @Inject private DeviceService deviceService;

    @ConfigProperty(name = "tfs.optical-controller.alarm-notification-url")
    String opticalControllerAlarmNotificationUrl;

    private static final ObjectMapper ALARM_NOTIFICATION_JSON = new ObjectMapper();

    // Deliberately NOT a shared, reused-forever HttpClient: a single
    // long-lived client keeps HTTP/1.1 connections open in its own internal
    // pool, and if OpticalController's backing pod is ever replaced (a
    // redeploy, a restart) while one of those connections is still open, it
    // can stay pinned to the old, now-dead pod for the rest of this JVM's
    // life -- every send on it still completes and reports a real HTTP 200
    // from this client's own point of view, but it never actually reaches
    // the current pod, so this logs "forwarded to OpticalController"
    // indefinitely while OpticalController silently receives nothing at
    // all (observed in practice). Building a fresh client per call means a
    // fresh connection -- and fresh Service routing -- every single time,
    // which fully closes that failure mode. The extra per-call connection
    // setup is trivial next to the alarm-delivery reliability it buys, at
    // this call's actual frequency (once per KPI sample, roughly every
    // 10s). Short connect timeout: this is an in-cluster call and should
    // fail fast rather than hang if OpticalController is down.
    private static HttpClient newAlarmNotificationHttpClient() {
        return HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(3)).build();
    }

    // private static final int POLICY_EVALUATION_TIMEOUT = 5;
    // private static final int ACCEPTABLE_NUMBER_OF_ALARMS = 3;
    // private static final int MONITORING_WINDOW_IN_SECONDS = 5;
@@ -179,6 +216,12 @@ public class CommonPolicyServiceImpl {
                }
            case POLICY_RULE_ACTION_GENERATE_FAILURE_NOTIFICATION:
                {
                    // Not calling notifyOpticalController here: AlarmListener.receiveAlarm
                    // already unconditionally calls forwardAllAlarmsToOpticalController for
                    // every message (triggered or not), which reaches OpticalController via
                    // this same notifyOpticalController helper. Calling it again here for a
                    // triggered alarm double-posted the identical alarm to OpticalController
                    // (two different UUIDs, same content, milliseconds apart).
                    LOGGER.infof(
                            "Policy for KPI %s with action: Generate Failure Notification", policyRuleKpiId);
                    // TODO: ADD YOUR ACTION CALL HERE: publish notification to Kafka, expose via NBI
@@ -195,6 +238,88 @@ public class CommonPolicyServiceImpl {
        if (applied) addActivePolicyRuleAction(policyRuleKpiId, policyRuleAction);
    }

    /**
     * Unconditionally forwards every alarm sample -- breach or not -- to OpticalController, so it can
     * independently decide when a real trigger is warranted (it needs to observe healthy samples too,
     * not just breaches, e.g. to establish that a device has ever been seen healthy). Deliberately
     * separate from applyActionServiceBasedOnKpiId's policy-action state machine above: that one
     * still only fires its configured action (recompute path, add constraint, ...) once per genuine
     * breach transition, unchanged. This just relays the raw sample.
     */
    public void forwardAllAlarmsToOpticalController(AlarmTopicDTO alarmTopicDto) {
        PolicyRuleService policyRuleService = activePolicyRuleServices.get(alarmTopicDto.getKpiId());
        if (policyRuleService == null) {
            return;
        }
        notifyOpticalController(policyRuleService, alarmTopicDto);
    }

    /**
     * Best-effort, fire-and-forget notification to OpticalController's alarm endpoint, which forwards
     * the alarm to any registered TAPI subscribers (e.g. the e2e-orchestrator webhook). Sent
     * asynchronously with a bounded timeout so a slow/unreachable OpticalController can never block
     * or break alarm processing here; any failure is caught and logged, never thrown.
     */
    private void notifyOpticalController(
            PolicyRuleService policyRuleService, AlarmTopicDTO alarmTopicDto) {
        String serviceId = policyRuleService.getServiceId().getId();
        try {
            Map<String, Object> body = new LinkedHashMap<>();
            body.put("service_id", serviceId);
            body.put("kpi_name", alarmTopicDto.getKpiId());
            body.put("measured_value", alarmTopicDto.getValue());
            body.put("threshold_low_breached", alarmTopicDto.getValueThresholdLow());
            body.put("threshold_high_breached", alarmTopicDto.getValueThresholdHigh());
            String json = ALARM_NOTIFICATION_JSON.writeValueAsString(body);

            HttpRequest request =
                    HttpRequest.newBuilder()
                            .uri(URI.create(opticalControllerAlarmNotificationUrl))
                            .timeout(Duration.ofSeconds(5))
                            .header("Content-Type", "application/json")
                            .POST(BodyPublishers.ofString(json))
                            .build();

            // Held in a local variable and explicitly referenced from both
            // callbacks below (see the two Objects.requireNonNull(client, ...)
            // calls) so this client stays strongly reachable for as long as
            // the async request is in flight. A per-call HttpClient with
            // nothing else referencing it would otherwise be eligible for
            // garbage collection the moment sendAsync() returns (it returns
            // immediately; the actual request runs on the client's own
            // internal executor) -- if that happened mid-request, the
            // request could be silently abandoned instead of completing.
            HttpClient client = newAlarmNotificationHttpClient();
            client
                    .sendAsync(request, BodyHandlers.ofString())
                    .thenAccept(
                            response -> {
                                Objects.requireNonNull(client, "keep client reachable until response arrives");
                                if (response.statusCode() >= 200 && response.statusCode() < 300) {
                                    LOGGER.infof(
                                            "Alarm notification for service %s forwarded to OpticalController (HTTP %d)",
                                            serviceId, response.statusCode());
                                } else {
                                    LOGGER.warnf(
                                            "OpticalController rejected alarm notification for service %s: HTTP %d - %s",
                                            serviceId, response.statusCode(), response.body());
                                }
                            })
                    .exceptionally(
                            ex -> {
                                Objects.requireNonNull(client, "keep client reachable until failure is reported");
                                LOGGER.warnf(
                                        "Failed to forward alarm notification for service %s to OpticalController: %s",
                                        serviceId, ex.getMessage());
                                return null;
                            });
        } catch (Exception ex) {
            LOGGER.warnf(
                    "Could not build/send alarm notification for service %s to OpticalController: %s",
                    serviceId, ex.getMessage());
        }
    }

    public void applyActionService(String policyRuleKpiId) {
        PolicyRuleService policyRuleService = activePolicyRuleServices.get(policyRuleKpiId);

+6 −0
Changes for src/policy/src/main/java/org/etsi/tfs/policy/policy/kafka/AlarmListener.java: 6 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -41,6 +41,12 @@ public class AlarmListener {
    @Blocking
    public void receiveAlarm(AlarmTopicDTO alarmTopicDto) {
        logger.infof("Received message from Analytics service:\n %s", alarmTopicDto.toString());

        // Always forward to OpticalController, breach or not -- it decides for
        // itself when a real trigger is warranted, and needs to see healthy
        // samples too (not just what's forwarded below) to do that correctly.
        commonPolicyServiceImpl.forwardAllAlarmsToOpticalController(alarmTopicDto);

        if (alarmTopicDto.isValid() && alarmTopicDto.isTriggered()) {
            logger.info("************************** Alarm **************************");
            logger.infof("Alarm for kpiId: %s", alarmTopicDto.getKpiId());
+9 −0
Changes for src/policy/src/main/resources/application.yml: 9 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -64,6 +64,7 @@ quarkus:
        monitoring-service-host: "monitoringservice"
        service-service-host: "serviceservice"
        kafka-broker-host: "kafka-service.kafka.svc.cluster.local"
        optical-controller-service-host: "opticalcontrollerservice"
    resources:
      requests:
        cpu: 50m
@@ -92,3 +93,11 @@ mp:
kafka:
  bootstrap:
    servers: ${quarkus.kubernetes.env.vars.kafka-broker-host}:9092

tfs:
  optical-controller:
    # Called from CommonPolicyServiceImpl when POLICY_RULE_ACTION_GENERATE_FAILURE_NOTIFICATION
    # fires, so OpticalController can forward the alarm to its TAPI subscribers
    # (e.g. e2e-orchestrator). Override via env var
    # TFS_OPTICAL_CONTROLLER_ALARM_NOTIFICATION_URL if ever needed.
    alarm-notification-url: "http://${quarkus.kubernetes.env.vars.optical-controller-service-host}:10060/OpticalTFS/restconf/operations/tapi-notification:post-alarm-notification"