Changes to the DBSCANServing service updating its general structure.

src/dbscanserving/client/DbscanServingClient.py

@@ -14,8 +14,8 @@
 
 import grpc, logging
 from common.tools.client.RetryDecorator import retry, delay_exponential
-from dbscanserving.proto.dbscanserving_pb2 import DetectionRequest, DetectionResponse
-from dbscanserving.proto.dbscanserving_pb2_grpc import DetectorStub
+from common.proto.dbscanserving_pb2 import DetectionRequest, DetectionResponse
+from common.proto.dbscanserving_pb2_grpc import DetectorStub
 
 LOGGER = logging.getLogger(__name__)
 MAX_RETRIES = 15

src/dbscanserving/proto/__init__.py

-# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-

src/dbscanserving/proto/dbscanserving_pb2.py

-# -*- coding: utf-8 -*-
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# source: dbscanserving.proto
-"""Generated protocol buffer code."""
-from google.protobuf.internal import enum_type_wrapper
-from google.protobuf import descriptor as _descriptor
-from google.protobuf import message as _message
-from google.protobuf import reflection as _reflection
-from google.protobuf import symbol_database as _symbol_database
-# @@protoc_insertion_point(imports)
-
-_sym_db = _symbol_database.Default()
-
-
-
-
-DESCRIPTOR = _descriptor.FileDescriptor(
-  name='dbscanserving.proto',
-  package='dbscanserving',
-  syntax='proto3',
-  serialized_options=None,
-  create_key=_descriptor._internal_create_key,
-  serialized_pb=b'\n\x13\x64\x62scanserving.proto\x12\rdbscanserving\"\x1a\n\x06Sample\x12\x10\n\x08\x66\x65\x61tures\x18\x01 \x03(\x02\"\xc2\x01\n\x10\x44\x65tectionRequest\x12\x0b\n\x03\x65ps\x18\x01 \x01(\x02\x12\x13\n\x0bmin_samples\x18\x02 \x01(\x05\x12%\n\x06metric\x18\x03 \x01(\x0e\x32\x15.dbscanserving.Metric\x12\x13\n\x0bnum_samples\x18\x04 \x01(\x05\x12\x14\n\x0cnum_features\x18\x05 \x01(\x05\x12&\n\x07samples\x18\x06 \x03(\x0b\x32\x15.dbscanserving.Sample\x12\x12\n\nidentifier\x18\x07 \x01(\x05\",\n\x11\x44\x65tectionResponse\x12\x17\n\x0f\x63luster_indices\x18\x01 \x03(\x05*\x17\n\x06Metric\x12\r\n\tEUCLIDEAN\x10\x00\x32W\n\x08\x44\x65tector\x12K\n\x06\x44\x65tect\x12\x1f.dbscanserving.DetectionRequest\x1a .dbscanserving.DetectionResponseb\x06proto3'
-)
-
-_METRIC = _descriptor.EnumDescriptor(
-  name='Metric',
-  full_name='dbscanserving.Metric',
-  filename=None,
-  file=DESCRIPTOR,
-  create_key=_descriptor._internal_create_key,
-  values=[
-    _descriptor.EnumValueDescriptor(
-      name='EUCLIDEAN', index=0, number=0,
-      serialized_options=None,
-      type=None,
-      create_key=_descriptor._internal_create_key),
-  ],
-  containing_type=None,
-  serialized_options=None,
-  serialized_start=309,
-  serialized_end=332,
-)
-_sym_db.RegisterEnumDescriptor(_METRIC)
-
-Metric = enum_type_wrapper.EnumTypeWrapper(_METRIC)
-EUCLIDEAN = 0
-
-
-
-_SAMPLE = _descriptor.Descriptor(
-  name='Sample',
-  full_name='dbscanserving.Sample',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  create_key=_descriptor._internal_create_key,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='features', full_name='dbscanserving.Sample.features', index=0,
-      number=1, type=2, cpp_type=6, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  serialized_options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=38,
-  serialized_end=64,
-)
-
-
-_DETECTIONREQUEST = _descriptor.Descriptor(
-  name='DetectionRequest',
-  full_name='dbscanserving.DetectionRequest',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  create_key=_descriptor._internal_create_key,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='eps', full_name='dbscanserving.DetectionRequest.eps', index=0,
-      number=1, type=2, cpp_type=6, label=1,
-      has_default_value=False, default_value=float(0),
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-    _descriptor.FieldDescriptor(
-      name='min_samples', full_name='dbscanserving.DetectionRequest.min_samples', index=1,
-      number=2, type=5, cpp_type=1, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-    _descriptor.FieldDescriptor(
-      name='metric', full_name='dbscanserving.DetectionRequest.metric', index=2,
-      number=3, type=14, cpp_type=8, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-    _descriptor.FieldDescriptor(
-      name='num_samples', full_name='dbscanserving.DetectionRequest.num_samples', index=3,
-      number=4, type=5, cpp_type=1, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-    _descriptor.FieldDescriptor(
-      name='num_features', full_name='dbscanserving.DetectionRequest.num_features', index=4,
-      number=5, type=5, cpp_type=1, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-    _descriptor.FieldDescriptor(
-      name='samples', full_name='dbscanserving.DetectionRequest.samples', index=5,
-      number=6, type=11, cpp_type=10, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-    _descriptor.FieldDescriptor(
-      name='identifier', full_name='dbscanserving.DetectionRequest.identifier', index=6,
-      number=7, type=5, cpp_type=1, label=1,
-      has_default_value=False, default_value=0,
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  serialized_options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=67,
-  serialized_end=261,
-)
-
-
-_DETECTIONRESPONSE = _descriptor.Descriptor(
-  name='DetectionResponse',
-  full_name='dbscanserving.DetectionResponse',
-  filename=None,
-  file=DESCRIPTOR,
-  containing_type=None,
-  create_key=_descriptor._internal_create_key,
-  fields=[
-    _descriptor.FieldDescriptor(
-      name='cluster_indices', full_name='dbscanserving.DetectionResponse.cluster_indices', index=0,
-      number=1, type=5, cpp_type=1, label=3,
-      has_default_value=False, default_value=[],
-      message_type=None, enum_type=None, containing_type=None,
-      is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
-  ],
-  extensions=[
-  ],
-  nested_types=[],
-  enum_types=[
-  ],
-  serialized_options=None,
-  is_extendable=False,
-  syntax='proto3',
-  extension_ranges=[],
-  oneofs=[
-  ],
-  serialized_start=263,
-  serialized_end=307,
-)
-
-_DETECTIONREQUEST.fields_by_name['metric'].enum_type = _METRIC
-_DETECTIONREQUEST.fields_by_name['samples'].message_type = _SAMPLE
-DESCRIPTOR.message_types_by_name['Sample'] = _SAMPLE
-DESCRIPTOR.message_types_by_name['DetectionRequest'] = _DETECTIONREQUEST
-DESCRIPTOR.message_types_by_name['DetectionResponse'] = _DETECTIONRESPONSE
-DESCRIPTOR.enum_types_by_name['Metric'] = _METRIC
-_sym_db.RegisterFileDescriptor(DESCRIPTOR)
-
-Sample = _reflection.GeneratedProtocolMessageType('Sample', (_message.Message,), {
-  'DESCRIPTOR' : _SAMPLE,
-  '__module__' : 'dbscanserving_pb2'
-  # @@protoc_insertion_point(class_scope:dbscanserving.Sample)
-  })
-_sym_db.RegisterMessage(Sample)
-
-DetectionRequest = _reflection.GeneratedProtocolMessageType('DetectionRequest', (_message.Message,), {
-  'DESCRIPTOR' : _DETECTIONREQUEST,
-  '__module__' : 'dbscanserving_pb2'
-  # @@protoc_insertion_point(class_scope:dbscanserving.DetectionRequest)
-  })
-_sym_db.RegisterMessage(DetectionRequest)
-
-DetectionResponse = _reflection.GeneratedProtocolMessageType('DetectionResponse', (_message.Message,), {
-  'DESCRIPTOR' : _DETECTIONRESPONSE,
-  '__module__' : 'dbscanserving_pb2'
-  # @@protoc_insertion_point(class_scope:dbscanserving.DetectionResponse)
-  })
-_sym_db.RegisterMessage(DetectionResponse)
-
-
-
-_DETECTOR = _descriptor.ServiceDescriptor(
-  name='Detector',
-  full_name='dbscanserving.Detector',
-  file=DESCRIPTOR,
-  index=0,
-  serialized_options=None,
-  create_key=_descriptor._internal_create_key,
-  serialized_start=334,
-  serialized_end=421,
-  methods=[
-  _descriptor.MethodDescriptor(
-    name='Detect',
-    full_name='dbscanserving.Detector.Detect',
-    index=0,
-    containing_service=None,
-    input_type=_DETECTIONREQUEST,
-    output_type=_DETECTIONRESPONSE,
-    serialized_options=None,
-    create_key=_descriptor._internal_create_key,
-  ),
-])
-_sym_db.RegisterServiceDescriptor(_DETECTOR)
-
-DESCRIPTOR.services_by_name['Detector'] = _DETECTOR
-
-# @@protoc_insertion_point(module_scope)

src/dbscanserving/proto/dbscanserving_pb2_grpc.py

-# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
-"""Client and server classes corresponding to protobuf-defined services."""
-import grpc
-
-from . import dbscanserving_pb2 as dbscanserving__pb2
-
-
-class DetectorStub(object):
-    """Missing associated documentation comment in .proto file."""
-
-    def __init__(self, channel):
-        """Constructor.
-
-        Args:
-            channel: A grpc.Channel.
-        """
-        self.Detect = channel.unary_unary(
-                '/dbscanserving.Detector/Detect',
-                request_serializer=dbscanserving__pb2.DetectionRequest.SerializeToString,
-                response_deserializer=dbscanserving__pb2.DetectionResponse.FromString,
-                )
-
-
-class DetectorServicer(object):
-    """Missing associated documentation comment in .proto file."""
-
-    def Detect(self, request, context):
-        """Missing associated documentation comment in .proto file."""
-        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-        context.set_details('Method not implemented!')
-        raise NotImplementedError('Method not implemented!')
-
-
-def add_DetectorServicer_to_server(servicer, server):
-    rpc_method_handlers = {
-            'Detect': grpc.unary_unary_rpc_method_handler(
-                    servicer.Detect,
-                    request_deserializer=dbscanserving__pb2.DetectionRequest.FromString,
-                    response_serializer=dbscanserving__pb2.DetectionResponse.SerializeToString,
-            ),
-    }
-    generic_handler = grpc.method_handlers_generic_handler(
-            'dbscanserving.Detector', rpc_method_handlers)
-    server.add_generic_rpc_handlers((generic_handler,))
-
-
- # This class is part of an EXPERIMENTAL API.
-class Detector(object):
-    """Missing associated documentation comment in .proto file."""
-
-    @staticmethod
-    def Detect(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
-        return grpc.experimental.unary_unary(request, target, '/dbscanserving.Detector/Detect',
-            dbscanserving__pb2.DetectionRequest.SerializeToString,
-            dbscanserving__pb2.DetectionResponse.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

src/dbscanserving/service/DbscanService.py

@@ -18,7 +18,7 @@ from concurrent import futures
 from grpc_health.v1.health import HealthServicer, OVERALL_HEALTH
 from grpc_health.v1.health_pb2 import HealthCheckResponse
 from grpc_health.v1.health_pb2_grpc import add_HealthServicer_to_server
-from dbscanserving.proto.dbscanserving_pb2_grpc import add_DetectorServicer_to_server
+from common.proto.dbscanserving_pb2_grpc import add_DetectorServicer_to_server
 from dbscanserving.service.DbscanServiceServicerImpl import DbscanServiceServicerImpl
 from dbscanserving.Config import GRPC_SERVICE_PORT, GRPC_MAX_WORKERS, GRPC_GRACE_PERIOD
 

src/dbscanserving/service/DbscanServiceServicerImpl.py

@@ -15,8 +15,8 @@
 import os, grpc, logging
 from sklearn.cluster import DBSCAN
 from common.rpc_method_wrapper.Decorator import create_metrics, safe_and_metered_rpc_method
-from dbscanserving.proto.dbscanserving_pb2 import DetectionRequest, DetectionResponse
-from dbscanserving.proto.dbscanserving_pb2_grpc import DetectorServicer
+from common.proto.dbscanserving_pb2 import DetectionRequest, DetectionResponse
+from common.proto.dbscanserving_pb2_grpc import DetectorServicer
 
 LOGGER = logging.getLogger(__name__)
 

src/dbscanserving/service/__main__.py

@@ -14,7 +14,7 @@
 
 import os, logging, signal, sys, time, threading, multiprocessing
 from prometheus_client import start_http_server
-from common.Settings import get_setting
+from common.Settings import get_log_level, get_metrics_port, get_setting
 from dbscanserving.Config import (
     GRPC_SERVICE_PORT, GRPC_MAX_WORKERS, GRPC_GRACE_PERIOD, LOG_LEVEL, METRICS_PORT)
 from dbscanserving.service.DbscanService import DbscanService
@@ -29,14 +29,14 @@ def signal_handler(signal, frame): # pylint: disable=redefined-outer-name
 def main():
     global LOGGER # pylint: disable=global-statement
 
+    log_level = get_log_level()
+    logging.basicConfig(level=log_level)
+    LOGGER = logging.getLogger(__name__)
+
     service_port = get_setting('DBSCANSERVICE_SERVICE_PORT_GRPC', default=GRPC_SERVICE_PORT)
     max_workers  = get_setting('MAX_WORKERS',                     default=GRPC_MAX_WORKERS )
     grace_period = get_setting('GRACE_PERIOD',                    default=GRPC_GRACE_PERIOD)
     log_level    = get_setting('LOG_LEVEL',                       default=LOG_LEVEL        )
-    metrics_port = get_setting('METRICS_PORT',                    default=METRICS_PORT     )
-
-    logging.basicConfig(level=log_level)
-    LOGGER = logging.getLogger(__name__)
 
     signal.signal(signal.SIGINT,  signal_handler)
     signal.signal(signal.SIGTERM, signal_handler)
@@ -44,6 +44,7 @@ def main():
     LOGGER.info('Starting...')
 
     # Start metrics server
+    metrics_port = get_metrics_port()
     start_http_server(metrics_port)
 
     # Starting CentralizedCybersecurity service

src/dbscanserving/tests/test_unitary.py

@@ -16,7 +16,7 @@ import random, logging, pytest, numpy
 from dbscanserving.Config import GRPC_SERVICE_PORT, GRPC_MAX_WORKERS, GRPC_GRACE_PERIOD
 from dbscanserving.client.DbscanServingClient import DbscanServingClient
 from dbscanserving.service.DbscanService import DbscanService
-from dbscanserving.proto.dbscanserving_pb2 import DetectionRequest, DetectionResponse, Sample
+from common.proto.dbscanserving_pb2 import DetectionRequest, DetectionResponse, Sample
 
 port = 10000 + GRPC_SERVICE_PORT # avoid privileged ports