Commit 47e4a51d authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Merge branch 'fix/common-proto' into 'develop' 

Adapted various components to common proto folder and corrected various errors

See merge request teraflow-h2020/controller!129
parents ac377681 7c8c203a

.gitlab-ci.yml

+1 −1
Original line number Diff line number Diff line
@@ -35,7 +35,7 @@ include: 
  - local: '/src/opticalcentralizedattackdetector/.gitlab-ci.yml'

  - local: '/src/automation/.gitlab-ci.yml'

  - local: '/src/policy/.gitlab-ci.yml'

  #- local: '/src/webui/.gitlab-ci.yml'

  - local: '/src/webui/.gitlab-ci.yml'

  #- local: '/src/l3_distributedattackdetector/.gitlab-ci.yml'

  #- local: '/src/l3_centralizedattackdetector/.gitlab-ci.yml'

  #- local: '/src/l3_attackmitigator/.gitlab-ci.yml'

manifests/webuiservice.yaml

+2 −0
Original line number Diff line number Diff line
@@ -39,6 +39,8 @@ spec: 
        env:

        - name: LOG_LEVEL

          value: "DEBUG"

        - name: WEBUISERVICE_SERVICE_BASEURL_HTTP

          value: "/webui"

        readinessProbe:

          httpGet:

            path: /healthz/ready

src/common/Constants.py

+3 −0
Original line number Diff line number Diff line
@@ -47,6 +47,7 @@ class ServiceNameEnum(Enum): 
    CYBERSECURITY = 'cybersecurity'

    INTERDOMAIN   = 'interdomain'

    PATHCOMP      = 'pathcomp'

    WEBUI         = 'webui'



# Default gRPC service ports

DEFAULT_SERVICE_GRPC_PORTS = {
@@ -68,10 +69,12 @@ DEFAULT_SERVICE_GRPC_PORTS = { 
DEFAULT_SERVICE_HTTP_PORTS = {

    ServiceNameEnum.CONTEXT   .value : 8080,

    ServiceNameEnum.COMPUTE   .value : 8080,

    ServiceNameEnum.WEBUI     .value : 8004,

}



# Default HTTP/REST-API service base URLs

DEFAULT_SERVICE_HTTP_BASEURLS = {

    ServiceNameEnum.CONTEXT   .value : '/api',

    ServiceNameEnum.COMPUTE   .value : '/restconf/data',

    ServiceNameEnum.WEBUI     .value : None,

}

src/compute/.gitlab-ci.yml

+6 −3
Original line number Diff line number Diff line
@@ -21,16 +21,17 @@ build compute: 
  before_script:

    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY

  script:

    - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile ./src/

    - docker build -t "$IMAGE_NAME:$IMAGE_TAG" -f ./src/$IMAGE_NAME/Dockerfile .

    - docker tag "$IMAGE_NAME:$IMAGE_TAG" "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

    - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

  coverage: '/TOTAL\s+\d+\s+\d+\s+(\d+%)/'

  after_script:

    - docker images --filter="dangling=true" --quiet | xargs -r docker rmi

  rules:

    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'

    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"'

    - changes:

      - src/common/**/*.py

      - proto/*.proto

      - src/$IMAGE_NAME/**/*.{py,in,yml}

      - src/$IMAGE_NAME/Dockerfile

      - src/$IMAGE_NAME/tests/*.py
@@ -65,6 +66,8 @@ unit test compute: 
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH)'

    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "develop"'

    - changes:

      - src/common/**/*.py

      - proto/*.proto

      - src/$IMAGE_NAME/**/*.{py,in,yml}

      - src/$IMAGE_NAME/Dockerfile

      - src/$IMAGE_NAME/tests/*.py

src/compute/Dockerfile

+35 −16
Original line number Diff line number Diff line
@@ -12,7 +12,7 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



FROM python:3-slim

FROM python:3.9-slim



# Install dependencies

RUN apt-get --yes --quiet --quiet update && \
@@ -28,25 +28,44 @@ RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \ 
    chmod +x /bin/grpc_health_probe



# Get generic Python packages

RUN python3 -m pip install --upgrade pip setuptools wheel pip-tools

RUN python3 -m pip install --upgrade pip

RUN python3 -m pip install --upgrade setuptools wheel

RUN python3 -m pip install --upgrade pip-tools



# Set working directory

# Get common Python packages

# Note: this step enables sharing the previous Docker build steps among all the Python components

WORKDIR /var/teraflow

COPY common_requirements.in common_requirements.in

RUN pip-compile --quiet --output-file=common_requirements.txt common_requirements.in

RUN python3 -m pip install -r common_requirements.txt



# Create module sub-folders

RUN mkdir -p /var/teraflow/compute

# Add common files into working directory

WORKDIR /var/teraflow/common

COPY src/common/. ./

RUN rm -rf proto



# Create proto sub-folder, copy .proto files, and generate Python code

RUN mkdir -p /var/teraflow/common/proto

WORKDIR /var/teraflow/common/proto

RUN touch __init__.py

COPY proto/*.proto ./

RUN python3 -m grpc_tools.protoc -I=. --python_out=. --grpc_python_out=. *.proto

RUN rm *.proto

RUN find . -type f -exec sed -i -E 's/(import\ .*)_pb2/from . \1_pb2/g' {} \;



# Get Python packages per module

COPY compute/requirements.in compute/requirements.in

RUN pip-compile --output-file=compute/requirements.txt compute/requirements.in

RUN python3 -m pip install -r compute/requirements.txt

# Create component sub-folder, get specific Python packages

RUN mkdir -p /var/teraflow/compute

WORKDIR /var/teraflow/compute

COPY src/compute/requirements.in requirements.in

RUN pip-compile --quiet --output-file=requirements.txt requirements.in

RUN python3 -m pip install -r requirements.txt



# Add files into working directory

COPY common/. common

COPY compute/. compute

COPY context/. context

COPY service/. service

COPY slice/. slice

# Add component files into working directory

WORKDIR /var/teraflow

COPY src/compute/. compute/

COPY src/context/. context/

COPY src/service/. service/

COPY src/slice/. slice/



# Start compute service

# Start the service

ENTRYPOINT ["python", "-m", "compute.service"]