Commit 0c5d2856 authored by Waleed Akbar's avatar Waleed Akbar
Browse files

Analytics Backend Initial Version 

- Added CRDB secret.
- Added test script.
- Added `main` and `__init__.py` files.
- Added Backend Service file.
- Added Spark Streamer file (successfully consuming streams from Kafka Topics).
- Added Tests folder and files.
- Added requirements file.
- Added ANALYTICS Kafka topics in `common.tools.kafka.variables` file.
parent 28bf80d3

deploy/tfs.sh

+12 −0
Original line number Diff line number Diff line
@@ -179,6 +179,18 @@ kubectl create secret generic crdb-telemetry --namespace ${TFS_K8S_NAMESPACE} -- 
    --from-literal=CRDB_DATABASE=${CRDB_DATABASE_TELEMETRY} \

    --from-literal=CRDB_USERNAME=${CRDB_USERNAME} \

    --from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \

    --from-literSLMODE=require

printf "\n"



echo "Create secret with CockroachDB data for Analytics microservices"

CRDB_SQL_PORT=$(kubectl --namespace ${CRDB_NAMESPACE} get service cockroachdb-public -o 'jsonpath={.spec.ports[?(@.name=="sql")].port}')

CRDB_DATABASE_ANALYTICS="tfs_analytics"  # TODO: change by specific configurable environment variable

kubectl create secret generic crdb-analytics --namespace ${TFS_K8S_NAMESPACE} --type='Opaque' \

    --from-literal=CRDB_NAMESPACE=${CRDB_NAMESPACE} \

    --from-literal=CRDB_SQL_PORT=${CRDB_SQL_PORT} \

    --from-literal=CRDB_DATABASE=${CRDB_DATABASE_ANALYTICS} \

    --from-literal=CRDB_USERNAME=${CRDB_USERNAME} \

    --from-literal=CRDB_PASSWORD=${CRDB_PASSWORD} \

    --from-literal=CRDB_SSLMODE=require

printf "\n"

scripts/run_tests_locally-analytics-backend.sh

0 → 100755
+22 −0
Original line number Diff line number Diff line 
#!/bin/bash

# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.





PROJECTDIR=`pwd`



cd $PROJECTDIR/src

RCFILE=$PROJECTDIR/coverage/.coveragerc

python3 -m pytest --log-level=DEBUG --log-cli-level=DEBUG --verbose \

    analytics/backend/tests/test_backend.py

src/analytics/backend/__init__.py

0 → 100644
+13 −0
Original line number Diff line number Diff line 
# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

src/analytics/backend/requirements.in

0 → 100644
+17 −0
Original line number Diff line number Diff line 
# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



java==11.0.*

pyspark==3.5.2

confluent-kafka==2.3.*
 
 No newline at end of file

src/analytics/backend/service/AnalyticsBackendService.py

0 → 100755
+29 −0
Original line number Diff line number Diff line 
# Copyright 2022-2024 ETSI OSG/SDG TeraFlowSDN (TFS) (https://tfs.etsi.org/)

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



import threading



from common.tools.service.GenericGrpcService import GenericGrpcService

from analytics.backend.service.SparkStreaming import SparkStreamer





class AnalyticsBackendService(GenericGrpcService):

    """

    Class listens for ...

    """

    def __init__(self, cls_name : str = __name__) -> None:

        pass



    def RunSparkStreamer(self):

        threading.Thread(target=SparkStreamer).start()