From e9d8f4a7d05e8ccd8afb2d3498453e10aed98762 Mon Sep 17 00:00:00 2001 From: Jorge Moratinos Salcines Date: Fri, 28 Jun 2024 10:37:10 +0200 Subject: [PATCH 1/9] Add LOG LEVEL to services --- .../api_invoker_management/app.py | 10 +- .../api_provider_management/app.py | 10 +- .../capif_acl/app.py | 10 +- .../TS29222_CAPIF_Auditing_API/logs/app.py | 10 +- .../service_apis/app.py | 9 +- .../capif_events/app.py | 10 +- .../api_invocation_logs/app.py | 9 +- .../published_apis/app.py | 9 +- .../capif_security/__main__.py | 149 ------------------ .../capif_security/app.py | 9 +- services/docker-compose-capif.yml | 11 ++ services/docker-compose-register.yml | 1 + services/helper/helper_service/app.py | 7 +- services/mock_server/mock_server.py | 10 +- services/register/register_service/app.py | 6 +- services/run.sh | 12 +- 16 files changed, 97 insertions(+), 185 deletions(-) delete mode 100644 services/TS29222_CAPIF_Security_API/capif_security/__main__.py diff --git a/services/TS29222_CAPIF_API_Invoker_Management_API/api_invoker_management/app.py b/services/TS29222_CAPIF_API_Invoker_Management_API/api_invoker_management/app.py index b459410..9d55bc5 100644 --- a/services/TS29222_CAPIF_API_Invoker_Management_API/api_invoker_management/app.py +++ b/services/TS29222_CAPIF_API_Invoker_Management_API/api_invoker_management/app.py @@ -22,6 +22,10 @@ from opentelemetry.instrumentation.redis import RedisInstrumentor NAME = "Invoker-Service" +# Setting log level +log_level = os.getenv('LOG_LEVEL', 'INFO').upper() +numeric_level = getattr(logging, log_level, logging.INFO) + def configure_monitoring(app, config): resource = Resource(attributes={"service.name": NAME}) @@ -86,10 +90,10 @@ def configure_logging(app): loggers = [app.logger, ] handlers = [] console_handler = logging.StreamHandler() - console_handler.setLevel(logging.DEBUG) + console_handler.setLevel(numeric_level) console_handler.setFormatter(verbose_formatter()) file_handler = RotatingFileHandler(filename="invoker_logs.log", maxBytes=1024 * 1024 * 100, backupCount=20) - file_handler.setLevel(logging.DEBUG) + file_handler.setLevel(numeric_level) file_handler.setFormatter(verbose_formatter()) handlers.append(console_handler) handlers.append(file_handler) @@ -99,7 +103,7 @@ def configure_logging(app): for handler in handlers: l.addHandler(handler) l.propagate = False - l.setLevel(logging.DEBUG) + l.setLevel(numeric_level) diff --git a/services/TS29222_CAPIF_API_Provider_Management_API/api_provider_management/app.py b/services/TS29222_CAPIF_API_Provider_Management_API/api_provider_management/app.py index 94ca3ed..2896830 100644 --- a/services/TS29222_CAPIF_API_Provider_Management_API/api_provider_management/app.py +++ b/services/TS29222_CAPIF_API_Provider_Management_API/api_provider_management/app.py @@ -23,6 +23,10 @@ from opentelemetry.instrumentation.redis import RedisInstrumentor NAME = "Provider-Service" +# Setting log level +log_level = os.getenv('LOG_LEVEL', 'INFO').upper() +numeric_level = getattr(logging, log_level, logging.INFO) + def configure_monitoring(app, config): resource = Resource(attributes={"service.name": NAME}) @@ -85,10 +89,10 @@ def configure_logging(app): loggers = [app.logger, ] handlers = [] console_handler = logging.StreamHandler() - console_handler.setLevel(logging.DEBUG) + console_handler.setLevel(numeric_level) console_handler.setFormatter(verbose_formatter()) file_handler = RotatingFileHandler(filename="provider_logs.log", maxBytes=1024 * 1024 * 100, backupCount=20) - file_handler.setLevel(logging.DEBUG) + file_handler.setLevel(numeric_level) file_handler.setFormatter(verbose_formatter()) handlers.append(console_handler) handlers.append(file_handler) @@ -97,7 +101,7 @@ def configure_logging(app): for handler in handlers: l.addHandler(handler) l.propagate = False - l.setLevel(logging.DEBUG) + l.setLevel(numeric_level) def verbose_formatter(): return logging.Formatter( diff --git a/services/TS29222_CAPIF_Access_Control_Policy_API/capif_acl/app.py b/services/TS29222_CAPIF_Access_Control_Policy_API/capif_acl/app.py index a473355..59a0f90 100644 --- a/services/TS29222_CAPIF_Access_Control_Policy_API/capif_acl/app.py +++ b/services/TS29222_CAPIF_Access_Control_Policy_API/capif_acl/app.py @@ -26,6 +26,10 @@ from opentelemetry.instrumentation.redis import RedisInstrumentor NAME = "Acl-Service" +# Setting log level +log_level = os.getenv('LOG_LEVEL', 'INFO').upper() +numeric_level = getattr(logging, log_level, logging.INFO) + def configure_monitoring(app, config): resource = Resource(attributes={"service.name": NAME}) @@ -89,10 +93,10 @@ def configure_logging(app): loggers = [app.logger, ] handlers = [] console_handler = logging.StreamHandler() - console_handler.setLevel(logging.DEBUG) + console_handler.setLevel(numeric_level) console_handler.setFormatter(verbose_formatter()) file_handler = RotatingFileHandler(filename="acl_logs.log", maxBytes=1024 * 1024 * 100, backupCount=20) - file_handler.setLevel(logging.DEBUG) + file_handler.setLevel(numeric_level) file_handler.setFormatter(verbose_formatter()) handlers.append(console_handler) handlers.append(file_handler) @@ -101,7 +105,7 @@ def configure_logging(app): for handler in handlers: l.addHandler(handler) l.propagate = False - l.setLevel(logging.DEBUG) + l.setLevel(numeric_level) def verbose_formatter(): return logging.Formatter( diff --git a/services/TS29222_CAPIF_Auditing_API/logs/app.py b/services/TS29222_CAPIF_Auditing_API/logs/app.py index a0bd01c..f68880c 100644 --- a/services/TS29222_CAPIF_Auditing_API/logs/app.py +++ b/services/TS29222_CAPIF_Auditing_API/logs/app.py @@ -18,6 +18,10 @@ from opentelemetry.sdk.trace.export import BatchSpanProcessor NAME = "Logs-Service" +# Setting log level +log_level = os.getenv('LOG_LEVEL', 'INFO').upper() +numeric_level = getattr(logging, log_level, logging.INFO) + def configure_monitoring(app, config): resource = Resource(attributes={"service.name": NAME}) @@ -79,10 +83,10 @@ def configure_logging(app): loggers = [app.logger, ] handlers = [] console_handler = logging.StreamHandler() - console_handler.setLevel(logging.DEBUG) + console_handler.setLevel(numeric_level) console_handler.setFormatter(verbose_formatter()) file_handler = RotatingFileHandler(filename="service_logs.log", maxBytes=1024 * 1024 * 100, backupCount=20) - file_handler.setLevel(logging.DEBUG) + file_handler.setLevel(numeric_level) file_handler.setFormatter(verbose_formatter()) handlers.append(console_handler) handlers.append(file_handler) @@ -91,7 +95,7 @@ def configure_logging(app): for handler in handlers: l.addHandler(handler) l.propagate = False - l.setLevel(logging.DEBUG) + l.setLevel(numeric_level) def verbose_formatter(): return logging.Formatter( diff --git a/services/TS29222_CAPIF_Discover_Service_API/service_apis/app.py b/services/TS29222_CAPIF_Discover_Service_API/service_apis/app.py index 2e37d0e..beec708 100644 --- a/services/TS29222_CAPIF_Discover_Service_API/service_apis/app.py +++ b/services/TS29222_CAPIF_Discover_Service_API/service_apis/app.py @@ -22,6 +22,9 @@ from opentelemetry.instrumentation.redis import RedisInstrumentor NAME = "Discover-Service" +# Setting log level +log_level = os.getenv('LOG_LEVEL', 'INFO').upper() +numeric_level = getattr(logging, log_level, logging.INFO) def configure_monitoring(app, config): @@ -86,10 +89,10 @@ def configure_logging(app): loggers = [app.logger, ] handlers = [] console_handler = logging.StreamHandler() - console_handler.setLevel(logging.DEBUG) + console_handler.setLevel(numeric_level) console_handler.setFormatter(verbose_formatter()) file_handler = RotatingFileHandler(filename="discover_logs.log", maxBytes=1024 * 1024 * 100, backupCount=20) - file_handler.setLevel(logging.DEBUG) + file_handler.setLevel(numeric_level) file_handler.setFormatter(verbose_formatter()) handlers.append(console_handler) handlers.append(file_handler) @@ -98,7 +101,7 @@ def configure_logging(app): for handler in handlers: l.addHandler(handler) l.propagate = False - l.setLevel(logging.DEBUG) + l.setLevel(numeric_level) def verbose_formatter(): return logging.Formatter( diff --git a/services/TS29222_CAPIF_Events_API/capif_events/app.py b/services/TS29222_CAPIF_Events_API/capif_events/app.py index eb43ca1..d1d3de3 100644 --- a/services/TS29222_CAPIF_Events_API/capif_events/app.py +++ b/services/TS29222_CAPIF_Events_API/capif_events/app.py @@ -34,6 +34,10 @@ from opentelemetry.instrumentation.redis import RedisInstrumentor NAME = "Events-Service" +# Setting log level +log_level = os.getenv('LOG_LEVEL', 'INFO').upper() +numeric_level = getattr(logging, log_level, logging.INFO) + def configure_monitoring(app, config): resource = Resource(attributes={"service.name": NAME}) @@ -97,10 +101,10 @@ def configure_logging(app): loggers = [app.logger, ] handlers = [] console_handler = logging.StreamHandler() - console_handler.setLevel(logging.DEBUG) + console_handler.setLevel(numeric_level) console_handler.setFormatter(verbose_formatter()) file_handler = RotatingFileHandler(filename="events_logs.log", maxBytes=1024 * 1024 * 100, backupCount=20) - file_handler.setLevel(logging.DEBUG) + file_handler.setLevel(numeric_level) file_handler.setFormatter(verbose_formatter()) handlers.append(console_handler) handlers.append(file_handler) @@ -109,7 +113,7 @@ def configure_logging(app): for handler in handlers: l.addHandler(handler) l.propagate = False - l.setLevel(logging.DEBUG) + l.setLevel(numeric_level) diff --git a/services/TS29222_CAPIF_Logging_API_Invocation_API/api_invocation_logs/app.py b/services/TS29222_CAPIF_Logging_API_Invocation_API/api_invocation_logs/app.py index 5eb97b3..bf76d0f 100644 --- a/services/TS29222_CAPIF_Logging_API_Invocation_API/api_invocation_logs/app.py +++ b/services/TS29222_CAPIF_Logging_API_Invocation_API/api_invocation_logs/app.py @@ -18,6 +18,9 @@ from opentelemetry.sdk.trace.export import BatchSpanProcessor NAME = "Logging-Service" +# Setting log level +log_level = os.getenv('LOG_LEVEL', 'INFO').upper() +numeric_level = getattr(logging, log_level, logging.INFO) def configure_monitoring(app, config): @@ -80,10 +83,10 @@ def configure_logging(app): loggers = [app.logger, ] handlers = [] console_handler = logging.StreamHandler() - console_handler.setLevel(logging.DEBUG) + console_handler.setLevel(numeric_level) console_handler.setFormatter(verbose_formatter()) file_handler = RotatingFileHandler(filename="logging_logs.log", maxBytes=1024 * 1024 * 100, backupCount=20) - file_handler.setLevel(logging.DEBUG) + file_handler.setLevel(numeric_level) file_handler.setFormatter(verbose_formatter()) handlers.append(console_handler) handlers.append(file_handler) @@ -93,7 +96,7 @@ def configure_logging(app): for handler in handlers: l.addHandler(handler) l.propagate = False - l.setLevel(logging.DEBUG) + l.setLevel(numeric_level) def verbose_formatter(): diff --git a/services/TS29222_CAPIF_Publish_Service_API/published_apis/app.py b/services/TS29222_CAPIF_Publish_Service_API/published_apis/app.py index 3c0c473..dedd2b5 100644 --- a/services/TS29222_CAPIF_Publish_Service_API/published_apis/app.py +++ b/services/TS29222_CAPIF_Publish_Service_API/published_apis/app.py @@ -25,6 +25,9 @@ from opentelemetry.instrumentation.redis import RedisInstrumentor NAME = "Publish-Service" +# Setting log level +log_level = os.getenv('LOG_LEVEL', 'INFO').upper() +numeric_level = getattr(logging, log_level, logging.INFO) def configure_monitoring(app, config): @@ -89,10 +92,10 @@ def configure_logging(app): loggers = [app.logger, ] handlers = [] console_handler = logging.StreamHandler() - console_handler.setLevel(logging.DEBUG) + console_handler.setLevel(numeric_level) console_handler.setFormatter(verbose_formatter()) file_handler = RotatingFileHandler(filename="publish_logs.log", maxBytes=1024 * 1024 * 100, backupCount=20) - file_handler.setLevel(logging.DEBUG) + file_handler.setLevel(numeric_level) file_handler.setFormatter(verbose_formatter()) handlers.append(console_handler) handlers.append(file_handler) @@ -102,7 +105,7 @@ def configure_logging(app): for handler in handlers: l.addHandler(handler) l.propagate = False - l.setLevel(logging.DEBUG) + l.setLevel(numeric_level) def verbose_formatter(): diff --git a/services/TS29222_CAPIF_Security_API/capif_security/__main__.py b/services/TS29222_CAPIF_Security_API/capif_security/__main__.py deleted file mode 100644 index 26eec19..0000000 --- a/services/TS29222_CAPIF_Security_API/capif_security/__main__.py +++ /dev/null @@ -1,149 +0,0 @@ -#!/usr/bin/env python3 - -import connexion -import logging -from capif_security import encoder -from flask_jwt_extended import JWTManager -from .config import Config -from .core.consumer_messager import Subscriber -from threading import Thread -from flask_executor import Executor -from logging.handlers import RotatingFileHandler -import sys -import os -from fluent import sender -from flask_executor import Executor -from opentelemetry.instrumentation.flask import FlaskInstrumentor -from opentelemetry import trace -from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter -from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator -from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchSpanProcessor -from opentelemetry.instrumentation.redis import RedisInstrumentor - - - -NAME = "Security-Service" - -def configure_monitoring(app, config): - - resource = Resource(attributes={"service.name": NAME}) - - fluent_bit_host = config['monitoring']['fluent_bit_host'] - fluent_bit_port = config['monitoring']['fluent_bit_port'] - fluent_bit_sender = sender.FluentSender('Security-Service', host=fluent_bit_host, port=fluent_bit_port) - propagator = TraceContextTextMapPropagator() - - tracer_provider = TracerProvider(resource=resource) - trace.set_tracer_provider(tracer_provider) - exporter = OTLPSpanExporter(endpoint=f"http://{config['monitoring']['opentelemetry_url']}:{config['monitoring']['opentelemetry_port']}", insecure=True) - span_processor = BatchSpanProcessor( - exporter, - max_queue_size=config['monitoring']['opentelemetry_max_queue_size'], - schedule_delay_millis=config['monitoring']['opentelemetry_schedule_delay_millis'], - max_export_batch_size=config['monitoring']['opentelemetry_max_export_batch_size'], - export_timeout_millis=config['monitoring']['opentelemetry_export_timeout_millis'], - ) - - trace.get_tracer_provider().add_span_processor(span_processor) - - FlaskInstrumentor().instrument_app(app) - - RedisInstrumentor().instrument() - - class FluentBitHandler(logging.Handler): - - def __init__(self): - logging.Handler.__init__(self) - - def emit(self, record): - log_entry = self.format(record) - log_data = { - 'message': log_entry, - 'level': record.levelname, - 'timestamp': record.created, - 'logger': record.name, - 'function': record.funcName, - 'line': record.lineno, - 'container_name': os.environ.get('CONTAINER_NAME', ''), - } - - # # Obtener el trace ID actual - current_context = trace.get_current_span().get_span_context() - - trace_id = current_context.trace_id - traceparent_id = current_context.span_id - log_data['traceID'] = hex(trace_id)[2:] - if traceparent_id != None: - log_data['traceparent'] = hex(traceparent_id)[2:] - fluent_bit_sender.emit('Security-Service', log_data) - - loggers = [app.logger, ] - for l in loggers: - l.addHandler(FluentBitHandler()) - - -def configure_logging(app): - del app.logger.handlers[:] - loggers = [app.logger, ] - handlers = [] - console_handler = logging.StreamHandler() - console_handler.setLevel(logging.DEBUG) - console_handler.setFormatter(verbose_formatter()) - file_handler = RotatingFileHandler(filename="security_logs.log", maxBytes=1024 * 1024 * 100, backupCount=20) - file_handler.setLevel(logging.DEBUG) - file_handler.setFormatter(verbose_formatter()) - handlers.append(console_handler) - handlers.append(file_handler) - - for l in loggers: - for handler in handlers: - l.addHandler(handler) - l.propagate = False - l.setLevel(logging.DEBUG) - - -def verbose_formatter(): - return logging.Formatter( - '{"timestamp": "%(asctime)s", "level": "%(levelname)s", "logger": "%(name)s", "function": "%(funcName)s", "line": %(lineno)d, "message": %(message)s}', - datefmt='%d/%m/%Y %H:%M:%S' - ) - -def main(): - - with open("/usr/src/app/capif_security/server.key", "rb") as key_file: - key_data = key_file.read() - - app = connexion.App(__name__, specification_dir='./openapi/') - app.app.json_encoder = encoder.JSONEncoder - - - app.app.config['JWT_ALGORITHM'] = 'RS256' - app.app.config['JWT_PRIVATE_KEY'] = key_data - app.add_api('openapi.yaml', - arguments={'title': 'CAPIF_Security_API'}, - pythonic_params=True) - - JWTManager(app.app) - subscriber = Subscriber() - - config = Config() - configure_logging(app.app) - - monitoring_value = os.environ.get("MONITORING", "").lower() - if monitoring_value == "true": - configure_monitoring(app.app, config.get_config()) - - executor = Executor(app.app) - - @app.app.before_first_request - def up_listener(): - executor.submit(subscriber.listen) - - - app.run(port=8080, debug=True) - -if __name__ == '__main__': - main() - diff --git a/services/TS29222_CAPIF_Security_API/capif_security/app.py b/services/TS29222_CAPIF_Security_API/capif_security/app.py index 0f06b7a..4345700 100644 --- a/services/TS29222_CAPIF_Security_API/capif_security/app.py +++ b/services/TS29222_CAPIF_Security_API/capif_security/app.py @@ -21,6 +21,9 @@ from opentelemetry.instrumentation.redis import RedisInstrumentor NAME = "Security-Service" +# Setting log level +log_level = os.getenv('LOG_LEVEL', 'INFO').upper() +numeric_level = getattr(logging, log_level, logging.INFO) def configure_monitoring(app, config): @@ -85,10 +88,10 @@ def configure_logging(app): loggers = [app.logger, ] handlers = [] console_handler = logging.StreamHandler() - console_handler.setLevel(logging.DEBUG) + console_handler.setLevel(numeric_level) console_handler.setFormatter(verbose_formatter()) file_handler = RotatingFileHandler(filename="security_logs.log", maxBytes=1024 * 1024 * 100, backupCount=20) - file_handler.setLevel(logging.DEBUG) + file_handler.setLevel(numeric_level) file_handler.setFormatter(verbose_formatter()) handlers.append(console_handler) handlers.append(file_handler) @@ -97,7 +100,7 @@ def configure_logging(app): for handler in handlers: l.addHandler(handler) l.propagate = False - l.setLevel(logging.DEBUG) + l.setLevel(numeric_level) def verbose_formatter(): diff --git a/services/docker-compose-capif.yml b/services/docker-compose-capif.yml index 779df21..44ed625 100644 --- a/services/docker-compose-capif.yml +++ b/services/docker-compose-capif.yml @@ -31,6 +31,7 @@ services: - VAULT_HOSTNAME=vault - VAULT_ACCESS_TOKEN=dev-only-token - VAULT_PORT=8200 + - LOG_LEVEL=${LOG_LEVEL} depends_on: - nginx @@ -47,6 +48,7 @@ services: environment: - CONTAINER_NAME=access-control-policy - MONITORING=${MONITORING} + - LOG_LEVEL=${LOG_LEVEL} restart: unless-stopped image: public.ecr.aws/o2v4a8t6/opencapif/access-control-policy:3.1.3 depends_on: @@ -71,6 +73,7 @@ services: - VAULT_HOSTNAME=vault - VAULT_ACCESS_TOKEN=dev-only-token - VAULT_PORT=8200 + - LOG_LEVEL=${LOG_LEVEL} restart: unless-stopped image: public.ecr.aws/o2v4a8t6/opencapif/api-invoker-management-api:3.1.3 depends_on: @@ -95,6 +98,7 @@ services: - VAULT_HOSTNAME=vault - VAULT_ACCESS_TOKEN=dev-only-token - VAULT_PORT=8200 + - LOG_LEVEL=${LOG_LEVEL} depends_on: - redis - nginx @@ -115,6 +119,7 @@ services: environment: - CONTAINER_NAME=api-auditing - MONITORING=${MONITORING} + - LOG_LEVEL=${LOG_LEVEL} depends_on: - mongo @@ -134,6 +139,7 @@ services: environment: - CONTAINER_NAME=services-apis - MONITORING=${MONITORING} + - LOG_LEVEL=${LOG_LEVEL} depends_on: - mongo @@ -147,6 +153,7 @@ services: environment: - CONTAINER_NAME=api-events - MONITORING=${MONITORING} + - LOG_LEVEL=${LOG_LEVEL} extra_hosts: - host.docker.internal:host-gateway - fluent-bit:host-gateway @@ -172,6 +179,7 @@ services: - CAPIF_HOSTNAME=${CAPIF_HOSTNAME} - CONTAINER_NAME=api-invocation-logs - MONITORING=${MONITORING} + - LOG_LEVEL=${LOG_LEVEL} depends_on: - mongo @@ -190,6 +198,7 @@ services: environment: - CONTAINER_NAME=api-publish-apis - MONITORING=${MONITORING} + - LOG_LEVEL=${LOG_LEVEL} depends_on: - redis - mongo @@ -215,6 +224,7 @@ services: - VAULT_HOSTNAME=vault - VAULT_ACCESS_TOKEN=dev-only-token - VAULT_PORT=8200 + - LOG_LEVEL=${LOG_LEVEL} extra_hosts: - host.docker.internal:host-gateway - fluent-bit:host-gateway @@ -262,6 +272,7 @@ services: - VAULT_HOSTNAME=vault - VAULT_ACCESS_TOKEN=dev-only-token - VAULT_PORT=8200 + - LOG_LEVEL=${LOG_LEVEL} hostname: ${CAPIF_HOSTNAME} volumes: - ./nginx/certs:/etc/nginx/certs diff --git a/services/docker-compose-register.yml b/services/docker-compose-register.yml index b0e5571..3505057 100644 --- a/services/docker-compose-register.yml +++ b/services/docker-compose-register.yml @@ -11,6 +11,7 @@ services: - VAULT_HOSTNAME=vault - VAULT_ACCESS_TOKEN=dev-only-token - VAULT_PORT=8200 + - LOG_LEVEL=${LOG_LEVEL} extra_hosts: - host.docker.internal:host-gateway - vault:host-gateway diff --git a/services/helper/helper_service/app.py b/services/helper/helper_service/app.py index c0af3df..e7a65ec 100644 --- a/services/helper/helper_service/app.py +++ b/services/helper/helper_service/app.py @@ -5,10 +5,15 @@ from OpenSSL.crypto import PKey, TYPE_RSA, X509Req, dump_certificate_request, FI from config import Config import json import requests +import os app = Flask(__name__) config = Config().get_config() +# Setting log level +log_level = os.getenv('LOG_LEVEL', 'INFO').upper() +numeric_level = getattr(logging, log_level, logging.INFO) + # Create a superadmin CSR and keys key = PKey() key.generate_key(TYPE_RSA, 2048) @@ -61,5 +66,5 @@ cert_file.write(bytes(ca_root, 'utf-8')) cert_file.close() app.register_blueprint(helper_routes) -app.logger.setLevel(logging.DEBUG) +app.logger.setLevel(numeric_level) diff --git a/services/mock_server/mock_server.py b/services/mock_server/mock_server.py index e45fae0..41ba650 100644 --- a/services/mock_server/mock_server.py +++ b/services/mock_server/mock_server.py @@ -5,6 +5,10 @@ import os app = Flask(__name__) +# Setting log level +log_level = os.getenv('LOG_LEVEL', 'INFO').upper() +numeric_level = getattr(logging, log_level, logging.INFO) + # Lista para almacenar las solicitudes recibidas requests_received = [] @@ -20,10 +24,10 @@ def configure_logging(app): loggers = [app.logger, ] handlers = [] console_handler = logging.StreamHandler() - console_handler.setLevel(logging.DEBUG) + console_handler.setLevel(numeric_level) console_handler.setFormatter(verbose_formatter()) file_handler = RotatingFileHandler(filename="mock_server.log", maxBytes=1024 * 1024 * 100, backupCount=20) - file_handler.setLevel(logging.DEBUG) + file_handler.setLevel(numeric_level) file_handler.setFormatter(verbose_formatter()) handlers.append(console_handler) handlers.append(file_handler) @@ -33,7 +37,7 @@ def configure_logging(app): for handler in handlers: l.addHandler(handler) l.propagate = False - l.setLevel(logging.DEBUG) + l.setLevel(numeric_level) @app.route('/testing', methods=['POST', 'GET']) def index(): diff --git a/services/register/register_service/app.py b/services/register/register_service/app.py index 1e38eda..093aa9f 100644 --- a/services/register/register_service/app.py +++ b/services/register/register_service/app.py @@ -16,6 +16,10 @@ jwt_manager = JWTManager(app) config = Config().get_config() +# Setting log level +log_level = os.getenv('LOG_LEVEL', 'INFO').upper() +numeric_level = getattr(logging, log_level, logging.INFO) + # Create a superadmin CSR and keys key = PKey() key.generate_key(TYPE_RSA, 2048) @@ -85,6 +89,6 @@ app.config['JWT_ALGORITHM'] = 'RS256' app.config['JWT_PRIVATE_KEY'] = key_data app.config['REGISTRE_SECRET_KEY'] = config["register"]["register_uuid"] -app.logger.setLevel(logging.DEBUG) +app.logger.setLevel(numeric_level) app.register_blueprint(register_routes) \ No newline at end of file diff --git a/services/run.sh b/services/run.sh index 3053019..91cbb5c 100755 --- a/services/run.sh +++ b/services/run.sh @@ -5,6 +5,7 @@ help() { echo " -c : Setup different hostname for capif" echo " -s : Run Mock server" echo " -m : Clean monitoring service" + echo " -l : Set Log Level (default DEBUG)" echo " -h : show this help" exit 1 } @@ -12,6 +13,7 @@ help() { HOSTNAME=capifcore MONITORING_STATE=false DEPLOY=all +LOG_LEVEL=DEBUG # Needed to avoid write permissions on bind volumes with prometheus and grafana DUID=$(id -u) @@ -33,7 +35,7 @@ else fi # Read params -while getopts ":c:msh" opt; do +while getopts ":c:l:msh" opt; do case $opt in c) HOSTNAME="$OPTARG" @@ -46,7 +48,9 @@ while getopts ":c:msh" opt; do ;; h) help - ;; + ;; + l) + LOG_LEVEL="$OPTARG" \?) echo "Not valid option: -$OPTARG" >&2 help @@ -86,7 +90,7 @@ else exit $status fi -CAPIF_HOSTNAME=$HOSTNAME MONITORING=$MONITORING_STATE docker compose -f "docker-compose-capif.yml" up --detach --build +CAPIF_HOSTNAME=$HOSTNAME MONITORING=$MONITORING_STATE LOG_LEVEL=$LOG_LEVEL docker compose -f "docker-compose-capif.yml" up --detach --build status=$? if [ $status -eq 0 ]; then @@ -98,7 +102,7 @@ fi CAPIF_PRIV_KEY_BASE_64=$(echo "$(cat nginx/certs/server.key)") -CAPIF_PRIV_KEY=$CAPIF_PRIV_KEY_BASE_64 docker compose -f "docker-compose-register.yml" up --detach --build +CAPIF_PRIV_KEY=$CAPIF_PRIV_KEY_BASE_64 LOG_LEVEL=$LOG_LEVEL docker compose -f "docker-compose-register.yml" up --detach --build status=$? if [ $status -eq 0 ]; then -- GitLab From b3c7c8f4a90a4d52e2c7082c82179cef179cf825 Mon Sep 17 00:00:00 2001 From: Jorge Moratinos Salcines Date: Fri, 28 Jun 2024 10:38:15 +0200 Subject: [PATCH 2/9] fix run.sh --- services/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/services/run.sh b/services/run.sh index 91cbb5c..682d22e 100755 --- a/services/run.sh +++ b/services/run.sh @@ -51,6 +51,7 @@ while getopts ":c:l:msh" opt; do ;; l) LOG_LEVEL="$OPTARG" + ;; \?) echo "Not valid option: -$OPTARG" >&2 help -- GitLab From 4665ea16345276e621e7c2215a7022409cdc5feb Mon Sep 17 00:00:00 2001 From: Jorge Moratinos Salcines Date: Fri, 28 Jun 2024 10:41:16 +0200 Subject: [PATCH 3/9] add os import to register --- services/register/register_service/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/register/register_service/app.py b/services/register/register_service/app.py index 093aa9f..53d4b05 100644 --- a/services/register/register_service/app.py +++ b/services/register/register_service/app.py @@ -8,6 +8,7 @@ import json from config import Config from db.db import MongoDatabse import logging +import os app = Flask(__name__) -- GitLab From 9f00a1517d4e506d3c2896b19df004f6ed03251c Mon Sep 17 00:00:00 2001 From: Jorge Moratinos Salcines Date: Fri, 28 Jun 2024 11:40:22 +0200 Subject: [PATCH 4/9] Change loggin driver at docker compose register --- services/docker-compose-register.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/docker-compose-register.yml b/services/docker-compose-register.yml index 3505057..53ed101 100644 --- a/services/docker-compose-register.yml +++ b/services/docker-compose-register.yml @@ -22,6 +22,8 @@ services: mongo_register: image: mongo:6.0.2 + logging: + driver: 'none' restart: unless-stopped environment: MONGO_INITDB_ROOT_USERNAME: root -- GitLab From e895c328a1651838ec08ff9970e988afde7678dd Mon Sep 17 00:00:00 2001 From: Jorge Moratinos Salcines Date: Fri, 28 Jun 2024 11:48:43 +0200 Subject: [PATCH 5/9] Improve run.sh help with enum values for log level --- services/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/run.sh b/services/run.sh index 682d22e..cf500a9 100755 --- a/services/run.sh +++ b/services/run.sh @@ -5,7 +5,7 @@ help() { echo " -c : Setup different hostname for capif" echo " -s : Run Mock server" echo " -m : Clean monitoring service" - echo " -l : Set Log Level (default DEBUG)" + echo " -l : Set Log Level (default DEBUG). Select one of: [CRITICAL, FATAL, ERROR, WARNING, WARN, INFO, DEBUG, NOTSET]" echo " -h : show this help" exit 1 } -- GitLab From 8a666892fc0f672a3d6bac78aa4152bcbb62d150 Mon Sep 17 00:00:00 2001 From: Jorge Moratinos Salcines Date: Fri, 28 Jun 2024 13:45:04 +0200 Subject: [PATCH 6/9] Improve tests with teardown by test --- services/clean_capif_docker_services.sh | 2 +- services/nginx/nginx.conf | 2 +- .../capif_api_access_control_policy.robot | 1 + .../CAPIF Api Auditing Service/capif_auditing_api.robot | 1 + .../CAPIF Api Discover Service/capif_api_service_discover.robot | 2 +- tests/features/CAPIF Api Events/capif_events_api.robot | 1 + .../capif_api_invoker_managenet.robot | 1 + .../features/CAPIF Api Logging Service/capif_logging_api.robot | 1 + .../capif_api_provider_management.robot | 1 + .../CAPIF Api Publish Service/capif_api_publish_service.robot | 1 + tests/features/CAPIF Security Api/capif_security_api.robot | 1 + 11 files changed, 11 insertions(+), 3 deletions(-) diff --git a/services/clean_capif_docker_services.sh b/services/clean_capif_docker_services.sh index 617ffed..bd6ef11 100755 --- a/services/clean_capif_docker_services.sh +++ b/services/clean_capif_docker_services.sh @@ -68,7 +68,7 @@ echo "${FILES[@]}" for FILE in "${FILES[@]}"; do echo "Executing 'docker compose down' for file $FILE" - CAPIF_PRIV_KEY=$CAPIF_PRIV_KEY_BASE_64 DUID=$DUID DGID=$DGID MONITORING=$MONITORING_STATE docker compose -f "$FILE" down --rmi all + CAPIF_PRIV_KEY=$CAPIF_PRIV_KEY_BASE_64 DUID=$DUID DGID=$DGID MONITORING=$MONITORING_STATE LOG_LEVEL=$LOG_LEVEL docker compose -f "$FILE" down --rmi all status=$? if [ $status -eq 0 ]; then echo "*** Removed Service from $FILE ***" diff --git a/services/nginx/nginx.conf b/services/nginx/nginx.conf index ec70d57..82d4dc5 100644 --- a/services/nginx/nginx.conf +++ b/services/nginx/nginx.conf @@ -1,6 +1,6 @@ worker_processes auto; -error_log /var/log/nginx/error.log debug; +error_log /var/log/nginx/error.log error; pid /tmp/nginx.pid; events { diff --git a/tests/features/CAPIF Api Access Control Policy/capif_api_access_control_policy.robot b/tests/features/CAPIF Api Access Control Policy/capif_api_access_control_policy.robot index ec0f082..cd49784 100644 --- a/tests/features/CAPIF Api Access Control Policy/capif_api_access_control_policy.robot +++ b/tests/features/CAPIF Api Access Control Policy/capif_api_access_control_policy.robot @@ -7,6 +7,7 @@ Resource ../../resources/common.resource Suite Teardown Reset Testing Environment Test Setup Reset Testing Environment +Test Teardown Reset Testing Environment *** Variables *** diff --git a/tests/features/CAPIF Api Auditing Service/capif_auditing_api.robot b/tests/features/CAPIF Api Auditing Service/capif_auditing_api.robot index 0676168..d850a57 100644 --- a/tests/features/CAPIF Api Auditing Service/capif_auditing_api.robot +++ b/tests/features/CAPIF Api Auditing Service/capif_auditing_api.robot @@ -7,6 +7,7 @@ Resource ../../resources/common.resource Suite Teardown Reset Testing Environment Test Setup Reset Testing Environment +Test Teardown Reset Testing Environment *** Variables *** diff --git a/tests/features/CAPIF Api Discover Service/capif_api_service_discover.robot b/tests/features/CAPIF Api Discover Service/capif_api_service_discover.robot index ff82ec3..e780526 100644 --- a/tests/features/CAPIF Api Discover Service/capif_api_service_discover.robot +++ b/tests/features/CAPIF Api Discover Service/capif_api_service_discover.robot @@ -6,7 +6,7 @@ Library /opt/robot-tests/tests/libraries/bodyRequests.py Suite Teardown Reset Testing Environment Test Setup Reset Testing Environment -# Test Setup Initialize Test And Register role=invoker +Test Teardown Reset Testing Environment *** Variables *** diff --git a/tests/features/CAPIF Api Events/capif_events_api.robot b/tests/features/CAPIF Api Events/capif_events_api.robot index d5e02d2..fdec6a9 100644 --- a/tests/features/CAPIF Api Events/capif_events_api.robot +++ b/tests/features/CAPIF Api Events/capif_events_api.robot @@ -8,6 +8,7 @@ Resource ../../resources/common.resource Suite Teardown Reset Testing Environment Test Setup Reset Testing Environment +Test Teardown Reset Testing Environment *** Variables *** diff --git a/tests/features/CAPIF Api Invoker Management/capif_api_invoker_managenet.robot b/tests/features/CAPIF Api Invoker Management/capif_api_invoker_managenet.robot index 27006b1..6ab8ba5 100644 --- a/tests/features/CAPIF Api Invoker Management/capif_api_invoker_managenet.robot +++ b/tests/features/CAPIF Api Invoker Management/capif_api_invoker_managenet.robot @@ -8,6 +8,7 @@ Library Collections Suite Teardown Reset Testing Environment Test Setup Reset Testing Environment +Test Teardown Reset Testing Environment *** Variables *** diff --git a/tests/features/CAPIF Api Logging Service/capif_logging_api.robot b/tests/features/CAPIF Api Logging Service/capif_logging_api.robot index a7a0253..4db54b2 100644 --- a/tests/features/CAPIF Api Logging Service/capif_logging_api.robot +++ b/tests/features/CAPIF Api Logging Service/capif_logging_api.robot @@ -7,6 +7,7 @@ Resource ../../resources/common.resource Suite Teardown Reset Testing Environment Test Setup Reset Testing Environment +Test Teardown Reset Testing Environment *** Variables *** diff --git a/tests/features/CAPIF Api Provider Management/capif_api_provider_management.robot b/tests/features/CAPIF Api Provider Management/capif_api_provider_management.robot index a6f9674..31aa55e 100644 --- a/tests/features/CAPIF Api Provider Management/capif_api_provider_management.robot +++ b/tests/features/CAPIF Api Provider Management/capif_api_provider_management.robot @@ -7,6 +7,7 @@ Library Collections Suite Teardown Reset Testing Environment Test Setup Reset Testing Environment +Test Teardown Reset Testing Environment *** Variables *** diff --git a/tests/features/CAPIF Api Publish Service/capif_api_publish_service.robot b/tests/features/CAPIF Api Publish Service/capif_api_publish_service.robot index 56484d2..f3556a4 100644 --- a/tests/features/CAPIF Api Publish Service/capif_api_publish_service.robot +++ b/tests/features/CAPIF Api Publish Service/capif_api_publish_service.robot @@ -6,6 +6,7 @@ Library /opt/robot-tests/tests/libraries/bodyRequests.py Suite Teardown Reset Testing Environment Test Setup Reset Testing Environment +Test Teardown Reset Testing Environment *** Variables *** diff --git a/tests/features/CAPIF Security Api/capif_security_api.robot b/tests/features/CAPIF Security Api/capif_security_api.robot index 6342aa9..f644839 100644 --- a/tests/features/CAPIF Security Api/capif_security_api.robot +++ b/tests/features/CAPIF Security Api/capif_security_api.robot @@ -7,6 +7,7 @@ Resource ../../resources/common.resource Suite Teardown Reset Testing Environment Test Setup Reset Testing Environment +Test Teardown Reset Testing Environment *** Variables *** -- GitLab From dc1d09408be707987f3fa9b97b4f34e7ab3acc87 Mon Sep 17 00:00:00 2001 From: Pelayo Torres Date: Fri, 28 Jun 2024 17:12:24 +0200 Subject: [PATCH 7/9] fix nginx log level critical --- services/nginx/nginx.conf | 2 +- services/nginx/nginx_prepare.sh | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/services/nginx/nginx.conf b/services/nginx/nginx.conf index 82d4dc5..f51e177 100644 --- a/services/nginx/nginx.conf +++ b/services/nginx/nginx.conf @@ -1,6 +1,6 @@ worker_processes auto; -error_log /var/log/nginx/error.log error; +error_log /var/log/nginx/error.log ${LOG_LEVEL}; pid /tmp/nginx.pid; events { diff --git a/services/nginx/nginx_prepare.sh b/services/nginx/nginx_prepare.sh index 081b2c9..52c11ab 100644 --- a/services/nginx/nginx_prepare.sh +++ b/services/nginx/nginx_prepare.sh @@ -34,5 +34,20 @@ curl -k -retry 30 \ --header "X-Vault-Token: $VAULT_TOKEN" \ --request GET "$VAULT_ADDR/v1/secret/data/server_cert/private" 2>/dev/null | jq -r '.data.data.key' -j > $CERTS_FOLDER/server.key +LOG_LEVEL=$(echo "${LOG_LEVEL}" | tr '[:upper:]' '[:lower:]')  ✔  base  +case "$LOG_LEVEL" in + critical) + LOG_LEVEL="crit" + ;; + fatal) + LOG_LEVEL="error" + ;; + notset) + LOG_LEVEL="info" + ;; +esac + +envsubst '$LOG_LEVEL' < /etc/nginx/nginx.conf > /etc/nginx/nginx.conf.tmp +mv /etc/nginx/nginx.conf.tmp /etc/nginx/nginx.conf nginx \ No newline at end of file -- GitLab From d52562fc332f3408fa70bd18eaa0e7e93189773c Mon Sep 17 00:00:00 2001 From: Jorge Moratinos Salcines Date: Mon, 1 Jul 2024 08:37:17 +0200 Subject: [PATCH 8/9] Added Interrupt Exception --- services/run_capif_tests.sh | 3 ++- tests/libraries/interrupt_listener.py | 24 ++++++++++++++++++++++++ tests/resources/common.resource | 1 + 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 tests/libraries/interrupt_listener.py diff --git a/services/run_capif_tests.sh b/services/run_capif_tests.sh index 7762fcd..f0a3f07 100755 --- a/services/run_capif_tests.sh +++ b/services/run_capif_tests.sh @@ -1,11 +1,12 @@ #!/bin/bash +timestamp=$(date +"%Y%m%d_%H%M%S") DOCKER_ROBOT_IMAGE=dockerhub.hi.inet/5ghacking/5gnow-robot-test-image DOCKER_ROBOT_IMAGE_VERSION=4.0 cd .. REPOSITORY_BASE_FOLDER=${PWD} TEST_FOLDER=$REPOSITORY_BASE_FOLDER/tests -RESULT_FOLDER=$REPOSITORY_BASE_FOLDER/results +RESULT_FOLDER=$REPOSITORY_BASE_FOLDER/results/$timestamp ROBOT_DOCKER_FILE_FOLDER=$REPOSITORY_BASE_FOLDER/tools/robot # nginx Hostname and http port (80 by default) to reach for tests diff --git a/tests/libraries/interrupt_listener.py b/tests/libraries/interrupt_listener.py new file mode 100644 index 0000000..ec2f5d4 --- /dev/null +++ b/tests/libraries/interrupt_listener.py @@ -0,0 +1,24 @@ +import signal +from robot.libraries.BuiltIn import BuiltIn + +class InterruptListener: + ROBOT_LISTENER_API_VERSION = 3 + + def __init__(self): + signal.signal(signal.SIGINT, self._handle_interrupt) + self.builtin = BuiltIn() + + def _handle_interrupt(self, signum, frame): + print("Execution interrupted! Running cleanup keyword...") + try: + self.builtin.run_keyword('Reset Testing Environment') + except Exception as e: + print(f"Error during cleanup: {e}") + finally: + exit(0) + + def start_suite(self, name, attrs): + print(f"Starting suite: {name}") + + def end_suite(self, name, attrs): + print(f"Ending suite: {name}") \ No newline at end of file diff --git a/tests/resources/common.resource b/tests/resources/common.resource index b2f19ae..fcd9a97 100644 --- a/tests/resources/common.resource +++ b/tests/resources/common.resource @@ -1,5 +1,6 @@ *** Settings *** Library /opt/robot-tests/tests/libraries/helpers.py +Library /opt/robot-tests/tests/libraries/interrupt_listener.py Library Process Library Collections Variables /opt/robot-tests/tests/libraries/environment.py -- GitLab From 4b303f23e88feb73f2f97b19201f2abe4df12534 Mon Sep 17 00:00:00 2001 From: Pelayo Torres Date: Mon, 1 Jul 2024 08:50:53 +0200 Subject: [PATCH 9/9] fix base --- services/nginx/nginx_prepare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/nginx/nginx_prepare.sh b/services/nginx/nginx_prepare.sh index 52c11ab..75fc9fd 100644 --- a/services/nginx/nginx_prepare.sh +++ b/services/nginx/nginx_prepare.sh @@ -34,7 +34,7 @@ curl -k -retry 30 \ --header "X-Vault-Token: $VAULT_TOKEN" \ --request GET "$VAULT_ADDR/v1/secret/data/server_cert/private" 2>/dev/null | jq -r '.data.data.key' -j > $CERTS_FOLDER/server.key -LOG_LEVEL=$(echo "${LOG_LEVEL}" | tr '[:upper:]' '[:lower:]')  ✔  base  +LOG_LEVEL=$(echo "${LOG_LEVEL}" | tr '[:upper:]' '[:lower:]') case "$LOG_LEVEL" in critical) -- GitLab