From 7772c39ee2667bbbc014e002282d39d26d2719cf Mon Sep 17 00:00:00 2001
From: Carlos Natalino <carlos.natalino@chalmers.se>
Date: Thu, 28 Jul 2022 16:43:00 +0200
Subject: [PATCH] Changes to the DBSCANServing service updating its general
structure.
---
.../client/DbscanServingClient.py | 4 +-
src/dbscanserving/proto/__init__.py | 14 -
src/dbscanserving/proto/dbscanserving_pb2.py | 244 ------------------
.../proto/dbscanserving_pb2_grpc.py | 66 -----
src/dbscanserving/service/DbscanService.py | 2 +-
.../service/DbscanServiceServicerImpl.py | 4 +-
src/dbscanserving/service/__main__.py | 11 +-
src/dbscanserving/tests/test_unitary.py | 2 +-
8 files changed, 12 insertions(+), 335 deletions(-)
delete mode 100644 src/dbscanserving/proto/__init__.py
delete mode 100644 src/dbscanserving/proto/dbscanserving_pb2.py
delete mode 100644 src/dbscanserving/proto/dbscanserving_pb2_grpc.py
diff --git a/src/dbscanserving/client/DbscanServingClient.py b/src/dbscanserving/client/DbscanServingClient.py
index 286b3318e..4e77cfb60 100644
--- a/src/dbscanserving/client/DbscanServingClient.py
+++ b/src/dbscanserving/client/DbscanServingClient.py
@@ -14,8 +14,8 @@
import grpc, logging
from common.tools.client.RetryDecorator import retry, delay_exponential
-from dbscanserving.proto.dbscanserving_pb2 import DetectionRequest, DetectionResponse
-from dbscanserving.proto.dbscanserving_pb2_grpc import DetectorStub
+from common.proto.dbscanserving_pb2 import DetectionRequest, DetectionResponse
+from common.proto.dbscanserving_pb2_grpc import DetectorStub
LOGGER = logging.getLogger(__name__)
MAX_RETRIES = 15
diff --git a/src/dbscanserving/proto/__init__.py b/src/dbscanserving/proto/__init__.py
deleted file mode 100644
index 70a332512..000000000
--- a/src/dbscanserving/proto/__init__.py
+++ /dev/null
@@ -1,14 +0,0 @@
-# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
diff --git a/src/dbscanserving/proto/dbscanserving_pb2.py b/src/dbscanserving/proto/dbscanserving_pb2.py
deleted file mode 100644
index f2d6c37c7..000000000
--- a/src/dbscanserving/proto/dbscanserving_pb2.py
+++ /dev/null
@@ -1,244 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by the protocol buffer compiler. DO NOT EDIT!
-# source: dbscanserving.proto
-"""Generated protocol buffer code."""
-from google.protobuf.internal import enum_type_wrapper
-from google.protobuf import descriptor as _descriptor
-from google.protobuf import message as _message
-from google.protobuf import reflection as _reflection
-from google.protobuf import symbol_database as _symbol_database
-# @@protoc_insertion_point(imports)
-
-_sym_db = _symbol_database.Default()
-
-
-
-
-DESCRIPTOR = _descriptor.FileDescriptor(
- name='dbscanserving.proto',
- package='dbscanserving',
- syntax='proto3',
- serialized_options=None,
- create_key=_descriptor._internal_create_key,
- serialized_pb=b'\n\x13\x64\x62scanserving.proto\x12\rdbscanserving\"\x1a\n\x06Sample\x12\x10\n\x08\x66\x65\x61tures\x18\x01 \x03(\x02\"\xc2\x01\n\x10\x44\x65tectionRequest\x12\x0b\n\x03\x65ps\x18\x01 \x01(\x02\x12\x13\n\x0bmin_samples\x18\x02 \x01(\x05\x12%\n\x06metric\x18\x03 \x01(\x0e\x32\x15.dbscanserving.Metric\x12\x13\n\x0bnum_samples\x18\x04 \x01(\x05\x12\x14\n\x0cnum_features\x18\x05 \x01(\x05\x12&\n\x07samples\x18\x06 \x03(\x0b\x32\x15.dbscanserving.Sample\x12\x12\n\nidentifier\x18\x07 \x01(\x05\",\n\x11\x44\x65tectionResponse\x12\x17\n\x0f\x63luster_indices\x18\x01 \x03(\x05*\x17\n\x06Metric\x12\r\n\tEUCLIDEAN\x10\x00\x32W\n\x08\x44\x65tector\x12K\n\x06\x44\x65tect\x12\x1f.dbscanserving.DetectionRequest\x1a .dbscanserving.DetectionResponseb\x06proto3'
-)
-
-_METRIC = _descriptor.EnumDescriptor(
- name='Metric',
- full_name='dbscanserving.Metric',
- filename=None,
- file=DESCRIPTOR,
- create_key=_descriptor._internal_create_key,
- values=[
- _descriptor.EnumValueDescriptor(
- name='EUCLIDEAN', index=0, number=0,
- serialized_options=None,
- type=None,
- create_key=_descriptor._internal_create_key),
- ],
- containing_type=None,
- serialized_options=None,
- serialized_start=309,
- serialized_end=332,
-)
-_sym_db.RegisterEnumDescriptor(_METRIC)
-
-Metric = enum_type_wrapper.EnumTypeWrapper(_METRIC)
-EUCLIDEAN = 0
-
-
-
-_SAMPLE = _descriptor.Descriptor(
- name='Sample',
- full_name='dbscanserving.Sample',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- create_key=_descriptor._internal_create_key,
- fields=[
- _descriptor.FieldDescriptor(
- name='features', full_name='dbscanserving.Sample.features', index=0,
- number=1, type=2, cpp_type=6, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- serialized_options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=38,
- serialized_end=64,
-)
-
-
-_DETECTIONREQUEST = _descriptor.Descriptor(
- name='DetectionRequest',
- full_name='dbscanserving.DetectionRequest',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- create_key=_descriptor._internal_create_key,
- fields=[
- _descriptor.FieldDescriptor(
- name='eps', full_name='dbscanserving.DetectionRequest.eps', index=0,
- number=1, type=2, cpp_type=6, label=1,
- has_default_value=False, default_value=float(0),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
- _descriptor.FieldDescriptor(
- name='min_samples', full_name='dbscanserving.DetectionRequest.min_samples', index=1,
- number=2, type=5, cpp_type=1, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
- _descriptor.FieldDescriptor(
- name='metric', full_name='dbscanserving.DetectionRequest.metric', index=2,
- number=3, type=14, cpp_type=8, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
- _descriptor.FieldDescriptor(
- name='num_samples', full_name='dbscanserving.DetectionRequest.num_samples', index=3,
- number=4, type=5, cpp_type=1, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
- _descriptor.FieldDescriptor(
- name='num_features', full_name='dbscanserving.DetectionRequest.num_features', index=4,
- number=5, type=5, cpp_type=1, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
- _descriptor.FieldDescriptor(
- name='samples', full_name='dbscanserving.DetectionRequest.samples', index=5,
- number=6, type=11, cpp_type=10, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
- _descriptor.FieldDescriptor(
- name='identifier', full_name='dbscanserving.DetectionRequest.identifier', index=6,
- number=7, type=5, cpp_type=1, label=1,
- has_default_value=False, default_value=0,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- serialized_options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=67,
- serialized_end=261,
-)
-
-
-_DETECTIONRESPONSE = _descriptor.Descriptor(
- name='DetectionResponse',
- full_name='dbscanserving.DetectionResponse',
- filename=None,
- file=DESCRIPTOR,
- containing_type=None,
- create_key=_descriptor._internal_create_key,
- fields=[
- _descriptor.FieldDescriptor(
- name='cluster_indices', full_name='dbscanserving.DetectionResponse.cluster_indices', index=0,
- number=1, type=5, cpp_type=1, label=3,
- has_default_value=False, default_value=[],
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
- ],
- extensions=[
- ],
- nested_types=[],
- enum_types=[
- ],
- serialized_options=None,
- is_extendable=False,
- syntax='proto3',
- extension_ranges=[],
- oneofs=[
- ],
- serialized_start=263,
- serialized_end=307,
-)
-
-_DETECTIONREQUEST.fields_by_name['metric'].enum_type = _METRIC
-_DETECTIONREQUEST.fields_by_name['samples'].message_type = _SAMPLE
-DESCRIPTOR.message_types_by_name['Sample'] = _SAMPLE
-DESCRIPTOR.message_types_by_name['DetectionRequest'] = _DETECTIONREQUEST
-DESCRIPTOR.message_types_by_name['DetectionResponse'] = _DETECTIONRESPONSE
-DESCRIPTOR.enum_types_by_name['Metric'] = _METRIC
-_sym_db.RegisterFileDescriptor(DESCRIPTOR)
-
-Sample = _reflection.GeneratedProtocolMessageType('Sample', (_message.Message,), {
- 'DESCRIPTOR' : _SAMPLE,
- '__module__' : 'dbscanserving_pb2'
- # @@protoc_insertion_point(class_scope:dbscanserving.Sample)
- })
-_sym_db.RegisterMessage(Sample)
-
-DetectionRequest = _reflection.GeneratedProtocolMessageType('DetectionRequest', (_message.Message,), {
- 'DESCRIPTOR' : _DETECTIONREQUEST,
- '__module__' : 'dbscanserving_pb2'
- # @@protoc_insertion_point(class_scope:dbscanserving.DetectionRequest)
- })
-_sym_db.RegisterMessage(DetectionRequest)
-
-DetectionResponse = _reflection.GeneratedProtocolMessageType('DetectionResponse', (_message.Message,), {
- 'DESCRIPTOR' : _DETECTIONRESPONSE,
- '__module__' : 'dbscanserving_pb2'
- # @@protoc_insertion_point(class_scope:dbscanserving.DetectionResponse)
- })
-_sym_db.RegisterMessage(DetectionResponse)
-
-
-
-_DETECTOR = _descriptor.ServiceDescriptor(
- name='Detector',
- full_name='dbscanserving.Detector',
- file=DESCRIPTOR,
- index=0,
- serialized_options=None,
- create_key=_descriptor._internal_create_key,
- serialized_start=334,
- serialized_end=421,
- methods=[
- _descriptor.MethodDescriptor(
- name='Detect',
- full_name='dbscanserving.Detector.Detect',
- index=0,
- containing_service=None,
- input_type=_DETECTIONREQUEST,
- output_type=_DETECTIONRESPONSE,
- serialized_options=None,
- create_key=_descriptor._internal_create_key,
- ),
-])
-_sym_db.RegisterServiceDescriptor(_DETECTOR)
-
-DESCRIPTOR.services_by_name['Detector'] = _DETECTOR
-
-# @@protoc_insertion_point(module_scope)
diff --git a/src/dbscanserving/proto/dbscanserving_pb2_grpc.py b/src/dbscanserving/proto/dbscanserving_pb2_grpc.py
deleted file mode 100644
index 895ced148..000000000
--- a/src/dbscanserving/proto/dbscanserving_pb2_grpc.py
+++ /dev/null
@@ -1,66 +0,0 @@
-# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
-"""Client and server classes corresponding to protobuf-defined services."""
-import grpc
-
-from . import dbscanserving_pb2 as dbscanserving__pb2
-
-
-class DetectorStub(object):
- """Missing associated documentation comment in .proto file."""
-
- def __init__(self, channel):
- """Constructor.
-
- Args:
- channel: A grpc.Channel.
- """
- self.Detect = channel.unary_unary(
- '/dbscanserving.Detector/Detect',
- request_serializer=dbscanserving__pb2.DetectionRequest.SerializeToString,
- response_deserializer=dbscanserving__pb2.DetectionResponse.FromString,
- )
-
-
-class DetectorServicer(object):
- """Missing associated documentation comment in .proto file."""
-
- def Detect(self, request, context):
- """Missing associated documentation comment in .proto file."""
- context.set_code(grpc.StatusCode.UNIMPLEMENTED)
- context.set_details('Method not implemented!')
- raise NotImplementedError('Method not implemented!')
-
-
-def add_DetectorServicer_to_server(servicer, server):
- rpc_method_handlers = {
- 'Detect': grpc.unary_unary_rpc_method_handler(
- servicer.Detect,
- request_deserializer=dbscanserving__pb2.DetectionRequest.FromString,
- response_serializer=dbscanserving__pb2.DetectionResponse.SerializeToString,
- ),
- }
- generic_handler = grpc.method_handlers_generic_handler(
- 'dbscanserving.Detector', rpc_method_handlers)
- server.add_generic_rpc_handlers((generic_handler,))
-
-
- # This class is part of an EXPERIMENTAL API.
-class Detector(object):
- """Missing associated documentation comment in .proto file."""
-
- @staticmethod
- def Detect(request,
- target,
- options=(),
- channel_credentials=None,
- call_credentials=None,
- insecure=False,
- compression=None,
- wait_for_ready=None,
- timeout=None,
- metadata=None):
- return grpc.experimental.unary_unary(request, target, '/dbscanserving.Detector/Detect',
- dbscanserving__pb2.DetectionRequest.SerializeToString,
- dbscanserving__pb2.DetectionResponse.FromString,
- options, channel_credentials,
- insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
diff --git a/src/dbscanserving/service/DbscanService.py b/src/dbscanserving/service/DbscanService.py
index 97654fa30..b5d3edbe5 100644
--- a/src/dbscanserving/service/DbscanService.py
+++ b/src/dbscanserving/service/DbscanService.py
@@ -18,7 +18,7 @@ from concurrent import futures
from grpc_health.v1.health import HealthServicer, OVERALL_HEALTH
from grpc_health.v1.health_pb2 import HealthCheckResponse
from grpc_health.v1.health_pb2_grpc import add_HealthServicer_to_server
-from dbscanserving.proto.dbscanserving_pb2_grpc import add_DetectorServicer_to_server
+from common.proto.dbscanserving_pb2_grpc import add_DetectorServicer_to_server
from dbscanserving.service.DbscanServiceServicerImpl import DbscanServiceServicerImpl
from dbscanserving.Config import GRPC_SERVICE_PORT, GRPC_MAX_WORKERS, GRPC_GRACE_PERIOD
diff --git a/src/dbscanserving/service/DbscanServiceServicerImpl.py b/src/dbscanserving/service/DbscanServiceServicerImpl.py
index 5560eec1e..258f3a619 100644
--- a/src/dbscanserving/service/DbscanServiceServicerImpl.py
+++ b/src/dbscanserving/service/DbscanServiceServicerImpl.py
@@ -15,8 +15,8 @@
import os, grpc, logging
from sklearn.cluster import DBSCAN
from common.rpc_method_wrapper.Decorator import create_metrics, safe_and_metered_rpc_method
-from dbscanserving.proto.dbscanserving_pb2 import DetectionRequest, DetectionResponse
-from dbscanserving.proto.dbscanserving_pb2_grpc import DetectorServicer
+from common.proto.dbscanserving_pb2 import DetectionRequest, DetectionResponse
+from common.proto.dbscanserving_pb2_grpc import DetectorServicer
LOGGER = logging.getLogger(__name__)
diff --git a/src/dbscanserving/service/__main__.py b/src/dbscanserving/service/__main__.py
index cad074a40..d8d1af664 100644
--- a/src/dbscanserving/service/__main__.py
+++ b/src/dbscanserving/service/__main__.py
@@ -14,7 +14,7 @@
import os, logging, signal, sys, time, threading, multiprocessing
from prometheus_client import start_http_server
-from common.Settings import get_setting
+from common.Settings import get_log_level, get_metrics_port, get_setting
from dbscanserving.Config import (
GRPC_SERVICE_PORT, GRPC_MAX_WORKERS, GRPC_GRACE_PERIOD, LOG_LEVEL, METRICS_PORT)
from dbscanserving.service.DbscanService import DbscanService
@@ -29,14 +29,14 @@ def signal_handler(signal, frame): # pylint: disable=redefined-outer-name
def main():
global LOGGER # pylint: disable=global-statement
+ log_level = get_log_level()
+ logging.basicConfig(level=log_level)
+ LOGGER = logging.getLogger(__name__)
+
service_port = get_setting('DBSCANSERVICE_SERVICE_PORT_GRPC', default=GRPC_SERVICE_PORT)
max_workers = get_setting('MAX_WORKERS', default=GRPC_MAX_WORKERS )
grace_period = get_setting('GRACE_PERIOD', default=GRPC_GRACE_PERIOD)
log_level = get_setting('LOG_LEVEL', default=LOG_LEVEL )
- metrics_port = get_setting('METRICS_PORT', default=METRICS_PORT )
-
- logging.basicConfig(level=log_level)
- LOGGER = logging.getLogger(__name__)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
@@ -44,6 +44,7 @@ def main():
LOGGER.info('Starting...')
# Start metrics server
+ metrics_port = get_metrics_port()
start_http_server(metrics_port)
# Starting CentralizedCybersecurity service
diff --git a/src/dbscanserving/tests/test_unitary.py b/src/dbscanserving/tests/test_unitary.py
index 8198c9e21..4156734e6 100644
--- a/src/dbscanserving/tests/test_unitary.py
+++ b/src/dbscanserving/tests/test_unitary.py
@@ -16,7 +16,7 @@ import random, logging, pytest, numpy
from dbscanserving.Config import GRPC_SERVICE_PORT, GRPC_MAX_WORKERS, GRPC_GRACE_PERIOD
from dbscanserving.client.DbscanServingClient import DbscanServingClient
from dbscanserving.service.DbscanService import DbscanService
-from dbscanserving.proto.dbscanserving_pb2 import DetectionRequest, DetectionResponse, Sample
+from common.proto.dbscanserving_pb2 import DetectionRequest, DetectionResponse, Sample
port = 10000 + GRPC_SERVICE_PORT # avoid privileged ports
--
GitLab