Commit e8477b02 authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

NBI component: 

- Added Kafka secret in manifest file and updated readiness/liveness probes
- Corrected logs in health_probes connector
- Updated requirements.in / Dockerfile / README.md
- Added creation of required Kafka topics
- Improved overall logging framework
- Added VNT Manager Recommendation endpoint
- Added Topology Updates endpoint
parent 5f7a012b

manifests/nbiservice.yaml

+7 −4
Original line number Diff line number Diff line
@@ -44,20 +44,23 @@ spec: 
              value: "production"  # change to "development" if developing

            - name: IETF_NETWORK_RENDERER

              value: "LIBYANG"

          envFrom:

            - secretRef:

                name: kfk-kpi-data

          readinessProbe:

            httpGet:

              path: /healthz

              port: 8080

            initialDelaySeconds: 5

            initialDelaySeconds: 30   # NBI's gunicorn takes 30~40 seconds to bootstrap

            periodSeconds: 10

            failureThreshold: 3

            failureThreshold: 6

          livenessProbe:

            httpGet:

              path: /healthz

              port: 8080

            initialDelaySeconds: 5

            initialDelaySeconds: 30   # NBI's gunicorn takes 30~40 seconds to bootstrap

            periodSeconds: 10

            failureThreshold: 3

            failureThreshold: 6

          resources:

            requests:

              cpu: 150m

src/nbi/Dockerfile

+1 −2
Original line number Diff line number Diff line
@@ -89,5 +89,4 @@ RUN mkdir -p /var/teraflow/tests/tools 
COPY src/tests/tools/mock_osm/. tests/tools/mock_osm/



# Start the service

ENTRYPOINT ["gunicorn", "-w", "4", "--worker-class", "eventlet", "-b", "0.0.0.0:8080", "nbi.service.app:app"]

#ENTRYPOINT ["gunicorn", "-w", "4", "--worker-class", "geventwebsocket.gunicorn.workers.GeventWebSocketWorker", "-b", "0.0.0.0:8080", "nbi.service.app:app"]
 
ENTRYPOINT ["gunicorn", "--workers", "4", "--worker-class", "eventlet", "--bind", "0.0.0.0:8080", "nbi.service.app:app"]

src/nbi/README.md

+9 −0
Original line number Diff line number Diff line
@@ -2,6 +2,15 @@ 


The NBI component uses libyang to validate and process messages. Follow instructions below to install it.





## IMPORTANT

**TL;DR**: Use kafka-python for consuming from kafka in the NBI component.



Why:



`confluent-kafka` is written in C, thus, it bypasses eventlet monkey_patches that convert normal threads into green_threads.

That implies methods such as consumer.poll() become blocking in eventlet scenario used by gunicorn web server.



## Install libyang

- Ref: https://github.com/CESNET/libyang

- Ref: https://github.com/CESNET/libyang-python/

src/nbi/requirements.in

+4 −1
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



confluent-kafka==2.3.*  # only for creating topics and compatibility

deepdiff==6.7.*

deepmerge==1.1.*

eventlet==0.39.0
@@ -19,16 +20,18 @@ Flask==2.1.3 
Flask-HTTPAuth==4.5.0

Flask-RESTful==0.3.9

flask-socketio==5.5.1

jsonschema==4.4.0

#gevent==24.11.1

#gevent-websocket==0.10.1

#greenlet==3.1.1

gunicorn==23.0.0

jsonschema==4.4.0

kafka-python==2.0.6     # for publishing and consuming messages in an eventlet-compatible way

libyang==2.8.4

netaddr==0.9.0

pyang==2.6.0

git+https://github.com/robshakir/pyangbind.git

pydantic==2.6.3

python-socketio==5.12.1

requests==2.27.1

werkzeug==2.3.7

#websockets==12.0

src/nbi/service/app.py

+30 −13
Original line number Diff line number Diff line
@@ -20,6 +20,7 @@ eventlet.monkey_patch() 


#pylint: disable=wrong-import-position

import logging

from common.tools.kafka.Variables import KafkaTopic

from common.Constants import ServiceNameEnum

from common.Settings import (

    ENVVAR_SUFIX_SERVICE_HOST, ENVVAR_SUFIX_SERVICE_PORT_GRPC,
@@ -39,14 +40,21 @@ from .ietf_network_slice import register_ietf_nss 
from .qkd_app import register_qkd_app

from .restconf_root import register_restconf_root

from .tfs_api import register_tfs_api

#from .topology_updates import register_topology_updates

from .vntm_recommend import register_vntm_recommend

from .well_known_meta import register_well_known





LOG_LEVEL = get_log_level()

logging.basicConfig(level=LOG_LEVEL)

logging.basicConfig(

    level=LOG_LEVEL,

    format="[Worker-%(process)d][%(asctime)s] %(levelname)s:%(name)s:%(message)s",

)

logging.getLogger('socketio.server').setLevel(logging.WARNING)

LOGGER = logging.getLogger(__name__)



LOGGER.info('Starting...')



wait_for_environment_variables([

    get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_HOST     ),

    get_env_var_name(ServiceNameEnum.CONTEXT, ENVVAR_SUFIX_SERVICE_PORT_GRPC),
@@ -58,6 +66,10 @@ wait_for_environment_variables([ 


BASE_URL = get_service_baseurl_http(ServiceNameEnum.NBI) or ''



LOGGER.info('Creating missing Kafka topics...')

KafkaTopic.create_all_topics()

LOGGER.info('Created required Kafka topics')



nbi_app = NbiApplication(base_url=BASE_URL)

register_health_probes   (nbi_app)

register_restconf_root   (nbi_app)
@@ -71,10 +83,15 @@ register_ietf_network (nbi_app) 
register_ietf_nss        (nbi_app)

register_ietf_acl        (nbi_app)

register_qkd_app         (nbi_app)

#register_topology_updates(nbi_app) # does not work; check if eventlet-grpc side effects

register_vntm_recommend  (nbi_app)

LOGGER.info('All connectors registered')



nbi_app.dump_configuration()

app = nbi_app.get_flask_app()



LOGGER.info('Initialization completed!')



if __name__ == '__main__':

    # Only used to run it locally during development stage;

    # otherwise, app is directly launched by gunicorn.