Commit 757fdfce authored by Jorge Moratinos's avatar Jorge Moratinos
Browse files

Celery improvement

parent 46eedd88
Loading
Loading
Loading
Loading
Loading
+6
Changes for services/celery/.dockerignore: 6 added lines, 0 removed lines.
Original line number Diff line number Diff line
.git
__pycache__
*.pyc
.venv
celerybeat-schedule*
*.log
 No newline at end of file
+13 −3
Changes for services/celery/Dockerfile: 13 added lines, 3 removed lines.
Original line number Diff line number Diff line
FROM labs.etsi.org:5050/ocf/capif/python:3.13-slim

# User withouth privileges
RUN useradd --create-home --uid 1000 celery \
 && mkdir -p /celery \
 && chown celery:celery /celery

WORKDIR /celery

# Dependencies as root
COPY requirements.txt /celery/
RUN pip install --no-cache-dir -r requirements.txt
RUN pip install --no-cache-dir -r requirements.txt \
 && pip check

COPY . /celery
# Code and script with propely user and permissions
COPY --chown=celery:celery . /celery
COPY --chown=celery:celery --chmod=755 start_celery.sh /celery/

RUN chmod +x /celery/start_celery.sh
# Next commands executed as celery user
USER celery

CMD ["/celery/start_celery.sh"]
+1 −1
Changes for services/celery/start_celery.sh: 1 added line, 1 removed line.
Original line number Diff line number Diff line
@@ -4,7 +4,7 @@ if [ "$CELERY_MODE" = "worker" ]; then
    echo "Starting Celery Worker..."
    celery -A tasks worker
elif [ "$CELERY_MODE" = "beat" ]; then
    echo "Iniciando Celery Beat..."
    echo "Starting Celery Beat..."
    celery -A tasks beat
else
    echo "ERROR: The environment variable CELERY_MODE is not set correctly (worker|beat)"
+27 −12
Changes for services/celery/tasks.py: 27 added lines, 12 removed lines.
Original line number Diff line number Diff line
@@ -9,6 +9,7 @@ import aiohttp
import pymongo
from bson.codec_options import CodecOptions
from celery import Celery
from celery.signals import worker_process_init
from config import Config

# Initialize Celery
@@ -28,6 +29,7 @@ celery.conf.beat_schedule = {

celery.conf.timezone = "UTC"
celery.conf.update(worker_hijack_root_logger=False)
celery.conf.broker_connection_retry_on_startup = True


# Setting log level
@@ -47,23 +49,28 @@ def configure_logging():

    formatter = verbose_formatter()

    # Root logger configuration
    root_logger = logging.getLogger()
    root_logger.setLevel(numeric_level)
    root_logger.handlers = []

    # Stream handler
    console_handler = logging.StreamHandler()
    console_handler.setLevel(numeric_level)
    console_handler.setFormatter(formatter)

    root_logger.addHandler(console_handler)

    # File handler only if configured
    log_file = os.getenv("CELERY_LOG_FILE")
    if log_file:
        file_handler = RotatingFileHandler(
        filename="celery_logs.log",
            filename=log_file,
            maxBytes=1024 * 1024 * 100,
            backupCount=20
        )
        file_handler.setLevel(numeric_level)
        file_handler.setFormatter(formatter)

    # Root logger configuration
    root_logger = logging.getLogger()
    root_logger.setLevel(numeric_level)
    root_logger.handlers = []
    root_logger.addHandler(console_handler)
        root_logger.addHandler(file_handler)

    # Optional: configure specific logger
@@ -74,13 +81,11 @@ def configure_logging():

logger = configure_logging()

# MongoDB Connection
# Read configuration
config = Config().get_config()

mongo_uri = f"mongodb://{config['mongo']['user']}:{config['mongo']['password']}@" \
                      f"{config['mongo']['host']}:{config['mongo']['port']}"
client = pymongo.MongoClient(mongo_uri)
notifications_col = client[config['mongo']['db']][config['mongo']['notifications_col']].with_options(codec_options=CodecOptions(tz_aware=True))
mongo_client = None
notifications_col = None

def serialize_clean_camel_case(obj):
    res = obj.to_dict()
@@ -174,3 +179,13 @@ def my_periodic_task():
        except Exception as e:
            logger.info(f"Error sending notification: {e}")


@worker_process_init.connect
def init_mongo(**kwargs):
    global mongo_client
    global notifications_col
    # MongoDB Connection
    mongo_uri = f"mongodb://{config['mongo']['user']}:{config['mongo']['password']}@" \
                        f"{config['mongo']['host']}:{config['mongo']['port']}"
    mongo_client = pymongo.MongoClient(mongo_uri)
    notifications_col = mongo_client[config['mongo']['db']][config['mongo']['notifications_col']].with_options(codec_options=CodecOptions(tz_aware=True))
+2
Changes for services/docker-compose-capif.yml: 2 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -222,6 +222,7 @@ services:
      - REDIS_HOST=redis
      - REDIS_PORT=6379
      - LOG_LEVEL=${LOG_LEVEL}
      - CELERY_LOG_FILE=worker.log
    depends_on:
      - redis
      - mongo
@@ -236,6 +237,7 @@ services:
      - REDIS_HOST=redis
      - REDIS_PORT=6379
      - LOG_LEVEL=${LOG_LEVEL}
      - CELERY_LOG_FILE=beat.log
    depends_on:
      - redis
      - mongo