Commit 5d3979fa authored by Waleed Akbar's avatar Waleed Akbar
Browse files

ManagementDB.py is created to manage DB operation of 'KPImanager' and 'TelemetryManager'.

parent cd73d36e

scripts/run_tests_locally-telemetry-frontend.sh

+1 −1
Original line number Diff line number Diff line
@@ -24,5 +24,5 @@ cd $PROJECTDIR/src 
# python3 kpi_manager/tests/test_unitary.py



RCFILE=$PROJECTDIR/coverage/.coveragerc

python3 -m pytest --log-level=INFO --verbose \

python3 -m pytest --log-level=INFO --log-cli-level=INFO --verbose \

    telemetry/frontend/tests/test_frontend.py
 
 No newline at end of file

scripts/run_tests_locally-telemetry-mgtDB.sh

0 → 100755
+26 −0
Original line number Diff line number Diff line 
#!/bin/bash

# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.





PROJECTDIR=`pwd`



cd $PROJECTDIR/src

# RCFILE=$PROJECTDIR/coverage/.coveragerc

# coverage run --rcfile=$RCFILE --append -m pytest --log-level=INFO --verbose \

#     kpi_manager/tests/test_unitary.py



RCFILE=$PROJECTDIR/coverage/.coveragerc

python3 -m pytest --log-cli-level=INFO --verbose \

    telemetry/database/tests/managementDBtests.py
 
 No newline at end of file

src/telemetry/database/TelemetryDBmanager.py

+46 −21
Original line number Diff line number Diff line
@@ -13,7 +13,8 @@ 
# limitations under the License.



import logging, time

from sqlalchemy import inspect

import sqlalchemy

from sqlalchemy import inspect, MetaData, Table

from sqlalchemy.orm import sessionmaker

from telemetry.database.TelemetryModel import Collector as CollectorModel

from telemetry.database.TelemetryModel import Kpi as KpiModel
@@ -22,13 +23,10 @@ from telemetry.database.TelemetryEngine import TelemetryEngine 
from common.proto.kpi_manager_pb2 import KpiDescriptor, KpiId

from common.proto.telemetry_frontend_pb2 import Collector, CollectorId

from sqlalchemy.exc import SQLAlchemyError



from telemetry.database.TelemetryModel import Base



LOGGER = logging.getLogger(__name__)

DB_NAME = "TelemetryFrontend"



# Create a base class for declarative models

Base = declarative_base()

DB_NAME = "telemetryfrontend"



class TelemetryDBmanager:

    def __init__(self):
@@ -41,18 +39,19 @@ class TelemetryDBmanager: 


    def create_database(self):

        try:

            with self.db_engine.connect() as connection:

                connection.execute(f"CREATE DATABASE {self.db_name};")

            LOGGER.info('TelemetryDBmanager initalized DB Name: {self.db_name}')

            # with self.db_engine.connect() as connection:

            #     connection.execute(f"CREATE DATABASE {self.db_name};")

            TelemetryEngine.create_database(self.db_engine)

            LOGGER.info('TelemetryDBmanager initalized DB Name: {:}'.format(self.db_name))

            return True

        except: # pylint: disable=bare-except # pragma: no cover

            LOGGER.exception('Failed to check/create the database: {:s}'.format(str(self.db_engine.url)))

        except Exception as e: # pylint: disable=bare-except # pragma: no cover

            LOGGER.exception('Failed to check/create the database: {:s}'.format(str(e)))

            return False



    def create_tables(self):

        try:

            Base.metadata.create_all(self.db_engine)     # type: ignore

            LOGGER.info("Tables created in the DB Name: {:}".format(self.db_name))

            LOGGER.info("Tables created in database ({:}) the as per Models".format(self.db_name))

        except Exception as e:

            LOGGER.info("Tables cannot be created in the TelemetryFrontend database. {:s}".format(str(e)))
@@ -61,10 +60,30 @@ class TelemetryDBmanager: 
            with self.db_engine.connect() as connection:

                result = connection.execute("SHOW TABLES;")

                tables = result.fetchall()

                LOGGER.info("Tables verified: {:}".format(tables))

                LOGGER.info("Tables in DB: {:}".format(tables))

        except Exception as e:

            LOGGER.info("Unable to fetch Table names. {:s}".format(str(e)))



    def drop_table(self, table_to_drop: str):

        try:

            inspector = inspect(self.db_engine)

            existing_tables = inspector.get_table_names()

            if table_to_drop in existing_tables:

                table = Table(table_to_drop, MetaData(), autoload_with=self.db_engine)

                table.drop(self.db_engine)

                LOGGER.info("Tables delete in the DB Name: {:}".format(self.db_name))

            else:

                LOGGER.warning("No table {:} in database {:} ".format(table_to_drop, DB_NAME))

        except Exception as e:

            LOGGER.info("Tables cannot be deleted in the {:} database. {:s}".format(DB_NAME, str(e)))



    def list_databases(self):

        query = "SHOW DATABASES"

        with self.db_engine.connect() as connection:

            result = connection.execute(query)

            databases = [row[0] for row in result]

        LOGGER.info("List of available DBs: {:}".format(databases))

        

# ------------------ INSERT METHODs --------------------------------------



    def inser_kpi(self, request: KpiDescriptor):
@@ -84,7 +103,7 @@ class TelemetryDBmanager: 
            # Add the instance to the session

            session.add(kpi_to_insert)

            session.commit()

            LOGGER.info("Row inserted into kpi table: {:}".format(kpi_to_insert))

            LOGGER.info("Row inserted into kpi table: {:}".format(kpi_to_insert.kpi_id))

        except Exception as e:

            session.rollback()

            LOGGER.info("Failed to insert new kpi. {:s}".format(str(e)))
@@ -108,7 +127,7 @@ class TelemetryDBmanager: 
            

            session.add(collector_to_insert)

            session.commit()

            LOGGER.info("Row inserted into collector table: {:}".format(collector_to_insert))

            LOGGER.info("Row inserted into collector table: {:}".format(collector_to_insert.collector_id))

        except Exception as e:

            session.rollback()

            LOGGER.info("Failed to insert new collector. {:s}".format(str(e)))
@@ -163,7 +182,10 @@ class TelemetryDBmanager: 
            for column, value in filters.items():

                query = query.filter(getattr(KpiModel, column) == value)

            result = query.all()

            LOGGER.info("Fetched filtered rows from KPI table with filters ---------- : {:s}".format(str(result)))

            if len(result) != 0:

                LOGGER.info("Fetched filtered rows from KPI table with filters : {:s}".format(str(result)))

            else:

                LOGGER.warning("No matching row found : {:s}".format(str(result)))

            return result

        except SQLAlchemyError as e:

            LOGGER.error("Error fetching filtered rows from KPI table with filters {:}: {:}".format(filters, e))
@@ -178,7 +200,10 @@ class TelemetryDBmanager: 
            for column, value in filters.items():

                query = query.filter(getattr(CollectorModel, column) == value)

            result = query.all()

            LOGGER.info("Fetched filtered rows from KPI table with filters ---------- : {:s}".format(str(result)))

            if len(result) != 0:

                LOGGER.info("Fetched filtered rows from KPI table with filters : {:s}".format(str(result)))

            else:

                LOGGER.warning("No matching row found : {:s}".format(str(result)))            

            return result

        except SQLAlchemyError as e:

            LOGGER.error("Error fetching filtered rows from KPI table with filters {:}: {:}".format(filters, e))
@@ -213,11 +238,11 @@ class TelemetryDBmanager: 
            if collector:

                session.delete(collector)

                session.commit()

                LOGGER.info("Deleted KPI with kpi_id: %s", collector_id_to_delete)

                LOGGER.info("Deleted collector with collector_id: %s", collector_id_to_delete)

            else:

                LOGGER.warning("KPI with kpi_id %s not found", collector_id_to_delete)

                LOGGER.warning("collector with collector_id %s not found", collector_id_to_delete)

        except SQLAlchemyError as e:

            session.rollback()

            LOGGER.error("Error deleting KPI with kpi_id %s: %s", collector_id_to_delete, e)

            LOGGER.error("Error deleting collector with collector_id %s: %s", collector_id_to_delete, e)

        finally:

            session.close()
 
 No newline at end of file

src/telemetry/database/TelemetryEngine.py

+3 −3
Original line number Diff line number Diff line
@@ -30,7 +30,7 @@ class TelemetryEngine: 
    def get_engine() -> sqlalchemy.engine.Engine:

        CRDB_NAMESPACE = "crdb"

        CRDB_SQL_PORT  = "26257"

        CRDB_DATABASE  = "TelemetryFrontend"

        CRDB_DATABASE  = "telemetryfrontend"

        CRDB_USERNAME  = "tfs"

        CRDB_PASSWORD  = "tfs123"

        CRDB_SSLMODE   = "require"
@@ -41,8 +41,8 @@ class TelemetryEngine: 
        try:

            # engine = sqlalchemy.create_engine(

            #     crdb_uri, connect_args={'application_name': APP_NAME}, echo=ECHO, future=True)

            engine = sqlalchemy.create_engine(crdb_uri)

            LOGGER.info(' --- TelemetryDBmanager initalized with DB URL: {:}'.format(crdb_uri))

            engine = sqlalchemy.create_engine(crdb_uri, echo=False)

            LOGGER.info(' TelemetryDBmanager initalized with DB URL: {:}'.format(crdb_uri))

        except: # pylint: disable=bare-except # pragma: no cover

            LOGGER.exception('Failed to connect to database: {:s}'.format(str(crdb_uri)))

            return None # type: ignore

src/telemetry/database/managementDB.py

0 → 100644
+72 −0
Original line number Diff line number Diff line 
# Copyright 2022-2023 ETSI TeraFlowSDN - TFS OSG (https://tfs.etsi.org/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



import logging, time

from sqlalchemy.orm import sessionmaker

from sqlalchemy.ext.declarative import declarative_base

from telemetry.database.TelemetryEngine import TelemetryEngine





LOGGER = logging.getLogger(__name__)

TELEMETRY_DB_NAME = "telemetryfrontend"



# Create a base class for declarative models

Base = declarative_base()



class managementDB:

    def __init__(self):

        self.db_engine = TelemetryEngine.get_engine()

        if self.db_engine is None:

            LOGGER.error('Unable to get SQLAlchemy DB Engine...')

            return False

        self.db_name = TELEMETRY_DB_NAME

        self.Session = sessionmaker(bind=self.db_engine)



    def create_database(self):

        try:

            with self.db_engine.connect() as connection:

                connection.execute(f"CREATE DATABASE {self.db_name};")

            LOGGER.info('managementDB initalizes database. Name: {self.db_name}')

            return True

        except: 

            LOGGER.exception('Failed to check/create the database: {:s}'.format(str(self.db_engine.url)))

            return False

    

    def create_tables(self):

        try:

            Base.metadata.create_all(self.db_engine)     # type: ignore

            LOGGER.info("Tables created in the DB Name: {:}".format(self.db_name))

        except Exception as e:

            LOGGER.info("Tables cannot be created in the TelemetryFrontend database. {:s}".format(str(e)))



    def verify_tables(self):

        try:

            with self.db_engine.connect() as connection:

                result = connection.execute("SHOW TABLES;")

                tables = result.fetchall()      # type: ignore

                LOGGER.info("Tables verified: {:}".format(tables))

        except Exception as e:

            LOGGER.info("Unable to fetch Table names. {:s}".format(str(e)))



    def add_row_to_db(self, row):

        session = self.Session()

        try:

            session.add(row)

            session.commit()

            LOGGER.info(f"Row inserted into {row.__class__.__name__} table. {row.__class__.__name__} Id: : {row.collector_id}")

        except Exception as e:

            session.rollback()

            LOGGER.error(f"Failed to insert new row into {row.__class__.__name__} table. {str(e)}")

        finally:

            session.close()
 
 No newline at end of file