Added changes to yaml file

deploy/nats.sh

@@ -53,7 +53,7 @@ function nats_deploy_single() {
         echo ">>> NATS is present; skipping step."
     else
         echo ">>> Deploy NATS"
-        helm3 install ${NATS_NAMESPACE} nats/nats --namespace ${NATS_NAMESPACE} --set nats.image.tag=2.9-alpine
+        helm3 install ${NATS_NAMESPACE} nats/nats --namespace ${NATS_NAMESPACE} --set nats.image=nats:2.9-alpine
 
         echo ">>> Waiting NATS statefulset to be created..."
         while ! kubectl get --namespace ${NATS_NAMESPACE} statefulset/${NATS_NAMESPACE} &> /dev/null; do

manifests/contextservice.yaml

@@ -20,7 +20,7 @@ spec:
   selector:
     matchLabels:
       app: contextservice
-  replicas: 1
+  #replicas: 1
   template:
     metadata:
       labels:
@@ -52,11 +52,11 @@ spec:
             command: ["/bin/grpc_health_probe", "-addr=:1010"]
         resources:
           requests:
-            cpu: 50m
+            cpu: 75m
             memory: 64Mi
           limits:
-            cpu: 500m
-            memory: 512Mi
+            cpu: 100m
+            memory: 128Mi
 ---
 apiVersion: v1
 kind: Service
@@ -77,3 +77,25 @@ spec:
     protocol: TCP
     port: 9192
     targetPort: 9192
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: contextservice-hpa
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: contextservice
+  minReplicas: 1
+  maxReplicas: 20
+  metrics:
+  - type: Resource
+    resource:
+      name: cpu
+      target:
+        type: Utilization
+        averageUtilization: 80
+  #behavior:
+  #  scaleDown:
+  #    stabilizationWindowSeconds: 30

manifests/deviceservice.yaml

@@ -45,11 +45,11 @@ spec:
             command: ["/bin/grpc_health_probe", "-addr=:2020"]
         resources:
           requests:
-            cpu: 50m
+            cpu: 128m
             memory: 64Mi
           limits:
-            cpu: 500m
-            memory: 512Mi
+            cpu: 256m
+            memory: 128Mi
 ---
 apiVersion: v1
 kind: Service

manifests/l3_centralizedattackdetectorservice.yaml

@@ -61,3 +61,26 @@ spec:
   - name: grpc
     port: 10001
     targetPort: 10001
+
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: l3-centralizedattackdetectorservice-hpa
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: l3-centralizedattackdetectorservice
+  minReplicas: 1
+  maxReplicas: 10
+  metrics:
+  - type: Resource
+    resource:
+      name: cpu
+      target:
+        type: Utilization
+        averageUtilization: 80
+  behavior:
+    scaleDown:
+      stabilizationWindowSeconds: 120

manifests/load_generatorservice.yaml

@@ -44,11 +44,11 @@ spec:
             command: ["/bin/grpc_health_probe", "-addr=:50052"]
         resources:
           requests:
-            cpu: 50m
+            cpu: 256m
             memory: 64Mi
           limits:
-            cpu: 500m
-            memory: 512Mi
+            cpu: 512m
+            memory: 128Mi
 ---
 apiVersion: v1
 kind: Service

manifests/pathcompservice.yaml

@@ -20,7 +20,7 @@ spec:
   selector:
     matchLabels:
       app: pathcompservice
-  replicas: 1
+  #replicas: 1
   template:
     metadata:
       labels:
@@ -53,6 +53,8 @@ spec:
       - name: backend
         image: labs.etsi.org:5050/tfs/controller/pathcomp-backend:latest
         imagePullPolicy: Always
+        ports:
+        - containerPort: 8081
         #readinessProbe:
         #  httpGet:
         #    path: /health
@@ -96,3 +98,25 @@ spec:
     protocol: TCP
     port: 9192
     targetPort: 9192
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: pathcompservice-hpa
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: pathcompservice
+  minReplicas: 1
+  maxReplicas: 20
+  metrics:
+  - type: Resource
+    resource:
+      name: cpu
+      target:
+        type: Utilization
+        averageUtilization: 80
+  #behavior:
+  #  scaleDown:
+  #    stabilizationWindowSeconds: 30

manifests/serviceservice.yaml

@@ -20,7 +20,7 @@ spec:
   selector:
     matchLabels:
       app: serviceservice
-  replicas: 1
+  #replicas: 1
   template:
     metadata:
       labels:
@@ -45,11 +45,11 @@ spec:
             command: ["/bin/grpc_health_probe", "-addr=:3030"]
         resources:
           requests:
-            cpu: 50m
-            memory: 64Mi
+            cpu: 32m
+            memory: 32Mi
           limits:
-            cpu: 500m
-            memory: 512Mi
+            cpu: 128m
+            memory: 64Mi
 ---
 apiVersion: v1
 kind: Service
@@ -70,3 +70,25 @@ spec:
     protocol: TCP
     port: 9192
     targetPort: 9192
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: serviceservice-hpa
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: serviceservice
+  minReplicas: 1
+  maxReplicas: 20
+  metrics:
+  - type: Resource
+    resource:
+      name: cpu
+      target:
+        type: Utilization
+        averageUtilization: 80
+  #behavior:
+  #  scaleDown:
+  #    stabilizationWindowSeconds: 30

manifests/sliceservice.yaml

@@ -20,7 +20,7 @@ spec:
   selector:
     matchLabels:
       app: sliceservice
-  replicas: 1
+  #replicas: 1
   template:
     metadata:
       labels:
@@ -50,11 +50,11 @@ spec:
             command: ["/bin/grpc_health_probe", "-addr=:4040"]
         resources:
           requests:
-            cpu: 50m
-            memory: 64Mi
+            cpu: 32m
+            memory: 128Mi
           limits:
-            cpu: 500m
-            memory: 512Mi
+            cpu: 128m
+            memory: 256Mi
 ---
 apiVersion: v1
 kind: Service
@@ -75,3 +75,25 @@ spec:
     protocol: TCP
     port: 9192
     targetPort: 9192
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: sliceservice-hpa
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: sliceservice
+  minReplicas: 1
+  maxReplicas: 20
+  metrics:
+  - type: Resource
+    resource:
+      name: cpu
+      target:
+        type: Utilization
+        averageUtilization: 80
+  #behavior:
+  #  scaleDown:
+  #    stabilizationWindowSeconds: 30

my_deploy.sh

@@ -3,7 +3,15 @@ export TFS_REGISTRY_IMAGES="http://localhost:32000/tfs/"
 export TFS_COMPONENTS="context device automation monitoring pathcomp service slice compute webui load_generator l3_attackmitigator l3_centralizedattackdetector"
 export TFS_IMAGE_TAG="dev"
 export TFS_K8S_NAMESPACE="tfs"
+<<<<<<< HEAD
 export TFS_EXTRA_MANIFESTS="manifests/nginx_ingress_http.yaml"
+=======
+
+# Set additional manifest files to be applied after the deployment
+export TFS_EXTRA_MANIFESTS="manifests/nginx_ingress_http.yaml manifests/servicemonitors.yaml"
+
+# Set the new Grafana admin password
+>>>>>>> origin/develop
 export TFS_GRAFANA_PASSWORD="admin123+"
 export TFS_SKIP_BUILD=""
 

proto/context.proto

@@ -40,7 +40,7 @@ service ContextService {
   rpc SetDevice          (Device        ) returns (       DeviceId        ) {}
   rpc RemoveDevice       (DeviceId      ) returns (       Empty           ) {}
   rpc GetDeviceEvents    (Empty         ) returns (stream DeviceEvent     ) {}
-
+  rpc SelectDevice       (DeviceFilter  ) returns (       DeviceList      ) {}
   rpc ListEndPointNames  (EndPointIdList) returns (       EndPointNameList) {}
 
   rpc ListLinkIds        (Empty         ) returns (       LinkIdList      ) {}
@@ -57,6 +57,7 @@ service ContextService {
   rpc UnsetService       (Service       ) returns (       ServiceId       ) {}
   rpc RemoveService      (ServiceId     ) returns (       Empty           ) {}
   rpc GetServiceEvents   (Empty         ) returns (stream ServiceEvent    ) {}
+  rpc SelectService      (ServiceFilter ) returns (       ServiceList     ) {}
 
   rpc ListSliceIds       (ContextId     ) returns (       SliceIdList     ) {}
   rpc ListSlices         (ContextId     ) returns (       SliceList       ) {}
@@ -65,6 +66,7 @@ service ContextService {
   rpc UnsetSlice         (Slice         ) returns (       SliceId         ) {}
   rpc RemoveSlice        (SliceId       ) returns (       Empty           ) {}
   rpc GetSliceEvents     (Empty         ) returns (stream SliceEvent      ) {}
+  rpc SelectSlice        (SliceFilter   ) returns (       SliceList       ) {}
 
   rpc ListConnectionIds  (ServiceId     ) returns (       ConnectionIdList) {}
   rpc ListConnections    (ServiceId     ) returns (       ConnectionList  ) {}
@@ -208,6 +210,13 @@ message DeviceList {
   repeated Device devices = 1;
 }
 
+message DeviceFilter {
+  DeviceIdList device_ids = 1;
+  bool include_endpoints = 2;
+  bool include_config_rules = 3;
+  bool include_components = 4;
+}
+
 message DeviceEvent {
   Event event = 1;
   DeviceId device_id = 2;
@@ -288,6 +297,13 @@ message ServiceList {
   repeated Service services = 1;
 }
 
+message ServiceFilter {
+  ServiceIdList service_ids = 1;
+  bool include_endpoint_ids = 2;
+  bool include_constraints = 3;
+  bool include_config_rules = 4;
+}
+
 message ServiceEvent {
   Event event = 1;
   ServiceId service_id = 2;
@@ -342,6 +358,15 @@ message SliceList {
   repeated Slice slices = 1;
 }
 
+message SliceFilter {
+  SliceIdList slice_ids = 1;
+  bool include_endpoint_ids = 2;
+  bool include_constraints = 3;
+  bool include_service_ids = 4;
+  bool include_subslice_ids = 5;
+  bool include_config_rules = 6;
+}
+
 message SliceEvent {
   Event event = 1;
   SliceId slice_id = 2;

src/context/client/ContextClient.py

@@ -21,11 +21,11 @@ from common.tools.grpc.Tools import grpc_message_to_json_string
 from common.proto.context_pb2 import (
     Connection, ConnectionEvent, ConnectionId, ConnectionIdList, ConnectionList,
     Context, ContextEvent, ContextId, ContextIdList, ContextList,
-    Device, DeviceEvent, DeviceId, DeviceIdList, DeviceList,
+    Device, DeviceEvent, DeviceFilter, DeviceId, DeviceIdList, DeviceList,
     Empty, EndPointIdList, EndPointNameList,
     Link, LinkEvent, LinkId, LinkIdList, LinkList,
-    Service, ServiceEvent, ServiceId, ServiceIdList, ServiceList,
-    Slice, SliceEvent, SliceId, SliceIdList, SliceList,
+    Service, ServiceEvent, ServiceFilter, ServiceId, ServiceIdList, ServiceList,
+    Slice, SliceEvent, SliceFilter, SliceId, SliceIdList, SliceList,
     Topology, TopologyDetails, TopologyEvent, TopologyId, TopologyIdList, TopologyList)
 from common.proto.context_pb2_grpc import ContextServiceStub
 from common.proto.context_policy_pb2_grpc import ContextPolicyServiceStub
@@ -185,6 +185,13 @@ class ContextClient:
         LOGGER.debug('RemoveDevice result: {:s}'.format(grpc_message_to_json_string(response)))
         return response
 
+    @RETRY_DECORATOR
+    def SelectDevice(self, request: DeviceFilter) -> DeviceList:
+        LOGGER.debug('SelectDevice request: {:s}'.format(grpc_message_to_json_string(request)))
+        response = self.stub.SelectDevice(request)
+        LOGGER.debug('SelectDevice result: {:s}'.format(grpc_message_to_json_string(response)))
+        return response
+
     @RETRY_DECORATOR
     def GetDeviceEvents(self, request: Empty) -> Iterator[DeviceEvent]:
         LOGGER.debug('GetDeviceEvents request: {:s}'.format(grpc_message_to_json_string(request)))
@@ -283,6 +290,13 @@ class ContextClient:
         LOGGER.debug('RemoveService result: {:s}'.format(grpc_message_to_json_string(response)))
         return response
 
+    @RETRY_DECORATOR
+    def SelectService(self, request: ServiceFilter) -> ServiceList:
+        LOGGER.debug('SelectService request: {:s}'.format(grpc_message_to_json_string(request)))
+        response = self.stub.SelectService(request)
+        LOGGER.debug('SelectService result: {:s}'.format(grpc_message_to_json_string(response)))
+        return response
+
     @RETRY_DECORATOR
     def GetServiceEvents(self, request: Empty) -> Iterator[ServiceEvent]:
         LOGGER.debug('GetServiceEvents request: {:s}'.format(grpc_message_to_json_string(request)))
@@ -332,6 +346,13 @@ class ContextClient:
         LOGGER.debug('RemoveSlice result: {:s}'.format(grpc_message_to_json_string(response)))
         return response
 
+    @RETRY_DECORATOR
+    def SelectSlice(self, request: SliceFilter) -> SliceList:
+        LOGGER.debug('SelectSlice request: {:s}'.format(grpc_message_to_json_string(request)))
+        response = self.stub.SelectSlice(request)
+        LOGGER.debug('SelectSlice result: {:s}'.format(grpc_message_to_json_string(response)))
+        return response
+
     @RETRY_DECORATOR
     def GetSliceEvents(self, request: Empty) -> Iterator[SliceEvent]:
         LOGGER.debug('GetSliceEvents request: {:s}'.format(grpc_message_to_json_string(request)))

src/context/service/ContextServiceServicerImpl.py

@@ -18,11 +18,11 @@ from common.message_broker.MessageBroker import MessageBroker
 from common.proto.context_pb2 import (
     Connection, ConnectionEvent, ConnectionId, ConnectionIdList, ConnectionList,
     Context, ContextEvent, ContextId, ContextIdList, ContextList,
-    Device, DeviceEvent, DeviceId, DeviceIdList, DeviceList,
+    Device, DeviceEvent, DeviceFilter, DeviceId, DeviceIdList, DeviceList,
     Empty, EndPointIdList, EndPointNameList, EventTypeEnum,
     Link, LinkEvent, LinkId, LinkIdList, LinkList,
-    Service, ServiceEvent, ServiceId, ServiceIdList, ServiceList,
-    Slice, SliceEvent, SliceId, SliceIdList, SliceList,
+    Service, ServiceEvent, ServiceFilter, ServiceId, ServiceIdList, ServiceList,
+    Slice, SliceEvent, SliceFilter, SliceId, SliceIdList, SliceList,
     Topology, TopologyDetails, TopologyEvent, TopologyId, TopologyIdList, TopologyList)
 from common.proto.policy_pb2 import PolicyRuleIdList, PolicyRuleId, PolicyRuleList, PolicyRule
 from common.proto.context_pb2_grpc import ContextServiceServicer
@@ -31,13 +31,13 @@ from common.method_wrappers.Decorator import MetricsPool, safe_and_metered_rpc_m
 from .database.Connection import (
     connection_delete, connection_get, connection_list_ids, connection_list_objs, connection_set)
 from .database.Context import context_delete, context_get, context_list_ids, context_list_objs, context_set
-from .database.Device import device_delete, device_get, device_list_ids, device_list_objs, device_set
+from .database.Device import device_delete, device_get, device_list_ids, device_list_objs, device_select, device_set
 from .database.EndPoint import endpoint_list_names
 from .database.Link import link_delete, link_get, link_list_ids, link_list_objs, link_set
 from .database.PolicyRule import (
     policyrule_delete, policyrule_get, policyrule_list_ids, policyrule_list_objs, policyrule_set)
-from .database.Service import service_delete, service_get, service_list_ids, service_list_objs, service_set
-from .database.Slice import slice_delete, slice_get, slice_list_ids, slice_list_objs, slice_set, slice_unset
+from .database.Service import service_delete, service_get, service_list_ids, service_list_objs, service_select, service_set
+from .database.Slice import slice_delete, slice_get, slice_list_ids, slice_list_objs, slice_select, slice_set, slice_unset
 from .database.Topology import (
     topology_delete, topology_get, topology_get_details, topology_list_ids, topology_list_objs, topology_set)
 from .Events import (
@@ -161,6 +161,10 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer
             notify_event(self.messagebroker, TOPIC_DEVICE, event_type, {'device_id': device_id})
         return Empty()
 
+    @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
+    def SelectDevices(self, request : DeviceFilter, context : grpc.ServicerContext) -> DeviceList:
+        return DeviceList(devices=device_select(self.db_engine, request))
+
     @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
     def GetDeviceEvents(self, request : Empty, context : grpc.ServicerContext) -> Iterator[DeviceEvent]:
         for message in self.messagebroker.consume({TOPIC_DEVICE}, consume_timeout=CONSUME_TIMEOUT):
@@ -235,6 +239,10 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer
             notify_event(self.messagebroker, TOPIC_SERVICE, event_type, {'service_id': service_id})
         return Empty()
 
+    @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
+    def SelectService(self, request : ServiceFilter, context : grpc.ServicerContext) -> ServiceList:
+        return ServiceList(services=service_select(self.db_engine, request))
+
     @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
     def GetServiceEvents(self, request : Empty, context : grpc.ServicerContext) -> Iterator[ServiceEvent]:
         for message in self.messagebroker.consume({TOPIC_SERVICE}, consume_timeout=CONSUME_TIMEOUT):
@@ -278,6 +286,10 @@ class ContextServiceServicerImpl(ContextServiceServicer, ContextPolicyServiceSer
             notify_event(self.messagebroker, TOPIC_SLICE, event_type, {'slice_id': slice_id})
         return Empty()
 
+    @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
+    def SelectSlice(self, request : SliceFilter, context : grpc.ServicerContext) -> SliceList:
+        return SliceList(slices=slice_select(self.db_engine, request))
+
     @safe_and_metered_rpc_method(METRICS_POOL, LOGGER)
     def GetSliceEvents(self, request : Empty, context : grpc.ServicerContext) -> Iterator[SliceEvent]:
         for message in self.messagebroker.consume({TOPIC_SLICE}, consume_timeout=CONSUME_TIMEOUT):

src/context/service/database/Connection.py

@@ -16,7 +16,7 @@ import datetime, logging, re
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.engine import Engine
 from sqlalchemy.exc import IntegrityError
-from sqlalchemy.orm import Session, sessionmaker
+from sqlalchemy.orm import Session, selectinload, sessionmaker
 from sqlalchemy_cockroachdb import run_transaction
 from typing import Dict, List, Optional, Tuple
 from common.proto.context_pb2 import Connection, ConnectionId, ServiceId
@@ -40,7 +40,11 @@ def connection_list_ids(db_engine : Engine, request : ServiceId) -> List[Dict]:
 def connection_list_objs(db_engine : Engine, request : ServiceId) -> List[Dict]:
     _,service_uuid = service_get_uuid(request, allow_random=False)
     def callback(session : Session) -> List[Dict]:
-        obj_list : List[ConnectionModel] = session.query(ConnectionModel).filter_by(service_uuid=service_uuid).all()
+        obj_list : List[ConnectionModel] = session.query(ConnectionModel)\
+            .options(selectinload(ConnectionModel.connection_service))\
+            .options(selectinload(ConnectionModel.connection_endpoints))\
+            .options(selectinload(ConnectionModel.connection_subservices))\
+            .filter_by(service_uuid=service_uuid).all()
         return [obj.dump() for obj in obj_list]
     return run_transaction(sessionmaker(bind=db_engine), callback)
 
@@ -48,6 +52,9 @@ def connection_get(db_engine : Engine, request : ConnectionId) -> Dict:
     connection_uuid = connection_get_uuid(request, allow_random=False)
     def callback(session : Session) -> Optional[Dict]:
         obj : Optional[ConnectionModel] = session.query(ConnectionModel)\
+            .options(selectinload(ConnectionModel.connection_service))\
+            .options(selectinload(ConnectionModel.connection_endpoints))\
+            .options(selectinload(ConnectionModel.connection_subservices))\
             .filter_by(connection_uuid=connection_uuid).one_or_none()
         return None if obj is None else obj.dump()
     obj = run_transaction(sessionmaker(bind=db_engine), callback)

src/context/service/database/Context.py

@@ -15,7 +15,7 @@
 import datetime, logging
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.engine import Engine
-from sqlalchemy.orm import Session, sessionmaker
+from sqlalchemy.orm import Session, selectinload, sessionmaker
 from sqlalchemy_cockroachdb import run_transaction
 from typing import Dict, List, Optional, Tuple
 from common.proto.context_pb2 import Context, ContextId
@@ -34,14 +34,22 @@ def context_list_ids(db_engine : Engine) -> List[Dict]:
 
 def context_list_objs(db_engine : Engine) -> List[Dict]:
     def callback(session : Session) -> List[Dict]:
-        obj_list : List[ContextModel] = session.query(ContextModel).all()
+        obj_list : List[ContextModel] = session.query(ContextModel)\
+            .options(selectinload(ContextModel.topologies))\
+            .options(selectinload(ContextModel.services))\
+            .options(selectinload(ContextModel.slices))\
+            .all()
         return [obj.dump() for obj in obj_list]
     return run_transaction(sessionmaker(bind=db_engine), callback)
 
 def context_get(db_engine : Engine, request : ContextId) -> Dict:
     context_uuid = context_get_uuid(request, allow_random=False)
     def callback(session : Session) -> Optional[Dict]:
-        obj : Optional[ContextModel] = session.query(ContextModel).filter_by(context_uuid=context_uuid).one_or_none()
+        obj : Optional[ContextModel] = session.query(ContextModel)\
+            .options(selectinload(ContextModel.topologies))\
+            .options(selectinload(ContextModel.services))\
+            .options(selectinload(ContextModel.slices))\
+            .filter_by(context_uuid=context_uuid).one_or_none()
         return None if obj is None else obj.dump()
     obj = run_transaction(sessionmaker(bind=db_engine), callback)
     if obj is None:

src/context/service/database/Device.py

@@ -15,12 +15,12 @@
 import datetime, logging
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.engine import Engine
-from sqlalchemy.orm import Session, sessionmaker
+from sqlalchemy.orm import Session, selectinload, sessionmaker
 from sqlalchemy_cockroachdb import run_transaction
 from typing import Dict, List, Optional, Set, Tuple
 from common.method_wrappers.ServiceExceptions import InvalidArgumentException, NotFoundException
-from common.proto.context_pb2 import Device, DeviceId, TopologyId
-from common.tools.grpc.Tools import grpc_message_to_json_string
+from common.proto.context_pb2 import Device, DeviceFilter, DeviceId, TopologyId
+#from common.tools.grpc.Tools import grpc_message_to_json_string
 from common.tools.object_factory.Device import json_device_id
 from context.service.database.uuids.Topology import topology_get_uuid
 from .models.DeviceModel import DeviceModel
@@ -43,14 +43,22 @@ def device_list_ids(db_engine : Engine) -> List[Dict]:
 
 def device_list_objs(db_engine : Engine) -> List[Dict]:
     def callback(session : Session) -> List[Dict]:
-        obj_list : List[DeviceModel] = session.query(DeviceModel).all()
+        obj_list : List[DeviceModel] = session.query(DeviceModel)\
+            .options(selectinload(DeviceModel.endpoints))\
+            .options(selectinload(DeviceModel.config_rules))\
+            .all()
+            #.options(selectinload(DeviceModel.components))\
         return [obj.dump() for obj in obj_list]
     return run_transaction(sessionmaker(bind=db_engine), callback)
 
 def device_get(db_engine : Engine, request : DeviceId) -> Dict:
     device_uuid = device_get_uuid(request, allow_random=False)
     def callback(session : Session) -> Optional[Dict]:
-        obj : Optional[DeviceModel] = session.query(DeviceModel).filter_by(device_uuid=device_uuid).one_or_none()
+        obj : Optional[DeviceModel] = session.query(DeviceModel)\
+            .options(selectinload(DeviceModel.endpoints))\
+            .options(selectinload(DeviceModel.config_rules))\
+            .filter_by(device_uuid=device_uuid).one_or_none()
+            #.options(selectinload(DeviceModel.components))\
         return None if obj is None else obj.dump()
     obj = run_transaction(sessionmaker(bind=db_engine), callback)
     if obj is None:
@@ -163,7 +171,9 @@ def device_set(db_engine : Engine, request : Device) -> Tuple[Dict, bool]:
             endpoint_updates = session.execute(stmt).fetchall()
             updated_endpoints = any([(updated_at > created_at) for created_at,updated_at in endpoint_updates])
 
-        if len(related_topologies) > 0:
+        if not updated or len(related_topologies) > 1:
+            # Only update topology-device relations when device is created (not updated) or when endpoints are
+            # modified (len(related_topologies) > 1).
             session.execute(insert(TopologyDeviceModel).values(related_topologies).on_conflict_do_nothing(
                 index_elements=[TopologyDeviceModel.topology_uuid, TopologyDeviceModel.device_uuid]
             ))
@@ -182,3 +192,22 @@ def device_delete(db_engine : Engine, request : DeviceId) -> Tuple[Dict, bool]:
         return num_deleted > 0
     deleted = run_transaction(sessionmaker(bind=db_engine), callback)
     return json_device_id(device_uuid),deleted
+
+def device_select(db_engine : Engine, request : DeviceFilter) -> List[Dict]:
+    device_uuids = [
+        device_get_uuid(device_id, allow_random=False)
+        for device_id in request.device_ids.device_ids
+    ]
+    dump_params = dict(
+        include_endpoints   =request.include_endpoints,
+        include_config_rules=request.include_config_rules,
+        include_components  =request.include_components,
+    )
+    def callback(session : Session) -> List[Dict]:
+        query = session.query(DeviceModel)
+        if request.include_endpoints   : query = query.options(selectinload(DeviceModel.endpoints))
+        if request.include_config_rules: query = query.options(selectinload(DeviceModel.config_rules))
+        #if request.include_components  : query = query.options(selectinload(DeviceModel.components))
+        obj_list : List[DeviceModel] = query.filter(DeviceModel.device_uuid.in_(device_uuids)).all()
+        return [obj.dump(**dump_params) for obj in obj_list]
+    return run_transaction(sessionmaker(bind=db_engine), callback)

src/context/service/database/EndPoint.py

@@ -14,7 +14,7 @@
 
 import logging
 from sqlalchemy.engine import Engine
-from sqlalchemy.orm import Session, sessionmaker
+from sqlalchemy.orm import Session, selectinload, sessionmaker
 from sqlalchemy_cockroachdb import run_transaction
 from typing import Dict, List
 from common.proto.context_pb2 import EndPointIdList
@@ -29,7 +29,8 @@ def endpoint_list_names(db_engine : Engine, request : EndPointIdList) -> List[Di
         for endpoint_id in request.endpoint_ids
     }
     def callback(session : Session) -> List[Dict]:
-        obj_list : List[EndPointModel] = \
-            session.query(EndPointModel).filter(EndPointModel.endpoint_uuid.in_(endpoint_uuids)).all()
+        obj_list : List[EndPointModel] = session.query(EndPointModel)\
+            .options(selectinload(EndPointModel.device))\
+            .filter(EndPointModel.endpoint_uuid.in_(endpoint_uuids)).all()
         return [obj.dump_name() for obj in obj_list]
     return run_transaction(sessionmaker(bind=db_engine), callback)

src/context/service/database/Link.py

@@ -15,7 +15,7 @@
 import datetime, logging
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.engine import Engine
-from sqlalchemy.orm import Session, sessionmaker
+from sqlalchemy.orm import Session, selectinload, sessionmaker
 from sqlalchemy_cockroachdb import run_transaction
 from typing import Dict, List, Optional, Set, Tuple
 from common.proto.context_pb2 import Link, LinkId
@@ -36,14 +36,18 @@ def link_list_ids(db_engine : Engine) -> List[Dict]:
 
 def link_list_objs(db_engine : Engine) -> List[Dict]:
     def callback(session : Session) -> List[Dict]:
-        obj_list : List[LinkModel] = session.query(LinkModel).all()
+        obj_list : List[LinkModel] = session.query(LinkModel)\
+            .options(selectinload(LinkModel.link_endpoints))\
+            .all()
         return [obj.dump() for obj in obj_list]
     return run_transaction(sessionmaker(bind=db_engine), callback)
 
 def link_get(db_engine : Engine, request : LinkId) -> Dict:
     link_uuid = link_get_uuid(request, allow_random=False)
     def callback(session : Session) -> Optional[Dict]:
-        obj : Optional[LinkModel] = session.query(LinkModel).filter_by(link_uuid=link_uuid).one_or_none()
+        obj : Optional[LinkModel] = session.query(LinkModel)\
+            .options(selectinload(LinkModel.link_endpoints))\
+            .filter_by(link_uuid=link_uuid).one_or_none()
         return None if obj is None else obj.dump()
     obj = run_transaction(sessionmaker(bind=db_engine), callback)
     if obj is None:

src/context/service/database/PolicyRule.py

@@ -15,7 +15,7 @@
 import datetime, json
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.engine import Engine
-from sqlalchemy.orm import Session, sessionmaker
+from sqlalchemy.orm import Session, selectinload, sessionmaker
 from sqlalchemy_cockroachdb import run_transaction
 from typing import Dict, List, Optional, Set, Tuple
 from common.proto.policy_pb2 import PolicyRule, PolicyRuleId, PolicyRuleIdList, PolicyRuleList
@@ -31,14 +31,15 @@ from .uuids.Service import service_get_uuid
 def policyrule_list_ids(db_engine : Engine) -> List[Dict]:
     def callback(session : Session) -> List[Dict]:
         obj_list : List[PolicyRuleModel] = session.query(PolicyRuleModel).all()
-        #.options(selectinload(PolicyRuleModel.topology)).filter_by(context_uuid=context_uuid).one_or_none()
         return [obj.dump_id() for obj in obj_list]
     return run_transaction(sessionmaker(bind=db_engine), callback)
 
 def policyrule_list_objs(db_engine : Engine) -> List[Dict]:
     def callback(session : Session) -> List[Dict]:
-        obj_list : List[PolicyRuleModel] = session.query(PolicyRuleModel).all()
-        #.options(selectinload(PolicyRuleModel.topology)).filter_by(context_uuid=context_uuid).one_or_none()
+        obj_list : List[PolicyRuleModel] = session.query(PolicyRuleModel)\
+            .options(selectinload(PolicyRuleModel.policyrule_service))\
+            .options(selectinload(PolicyRuleModel.policyrule_devices))\
+            .all()
         return [obj.dump() for obj in obj_list]
     return run_transaction(sessionmaker(bind=db_engine), callback)
 
@@ -46,6 +47,8 @@ def policyrule_get(db_engine : Engine, request : PolicyRuleId) -> PolicyRule:
     policyrule_uuid = policyrule_get_uuid(request, allow_random=False)
     def callback(session : Session) -> Optional[Dict]:
         obj : Optional[PolicyRuleModel] = session.query(PolicyRuleModel)\
+            .options(selectinload(PolicyRuleModel.policyrule_service))\
+            .options(selectinload(PolicyRuleModel.policyrule_devices))\
             .filter_by(policyrule_uuid=policyrule_uuid).one_or_none()
         return None if obj is None else obj.dump()
     obj = run_transaction(sessionmaker(bind=db_engine), callback)

src/context/service/database/Service.py

@@ -15,10 +15,10 @@
 import datetime, logging
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.engine import Engine
-from sqlalchemy.orm import Session, sessionmaker
+from sqlalchemy.orm import Session, selectinload, sessionmaker
 from sqlalchemy_cockroachdb import run_transaction
 from typing import Dict, List, Optional, Tuple
-from common.proto.context_pb2 import ContextId, Service, ServiceId
+from common.proto.context_pb2 import ContextId, Service, ServiceFilter, ServiceId
 from common.method_wrappers.ServiceExceptions import InvalidArgumentException, NotFoundException
 from common.tools.object_factory.Context import json_context_id
 from common.tools.object_factory.Service import json_service_id
@@ -43,14 +43,22 @@ def service_list_ids(db_engine : Engine, request : ContextId) -> List[Dict]:
 def service_list_objs(db_engine : Engine, request : ContextId) -> List[Dict]:
     context_uuid = context_get_uuid(request, allow_random=False)
     def callback(session : Session) -> List[Dict]:
-        obj_list : List[ServiceModel] = session.query(ServiceModel).filter_by(context_uuid=context_uuid).all()
+        obj_list : List[ServiceModel] = session.query(ServiceModel)\
+            .options(selectinload(ServiceModel.service_endpoints))\
+            .options(selectinload(ServiceModel.constraints))\
+            .options(selectinload(ServiceModel.config_rules))\
+            .filter_by(context_uuid=context_uuid).all()
         return [obj.dump() for obj in obj_list]
     return run_transaction(sessionmaker(bind=db_engine), callback)
 
 def service_get(db_engine : Engine, request : ServiceId) -> Dict:
     _,service_uuid = service_get_uuid(request, allow_random=False)
     def callback(session : Session) -> Optional[Dict]:
-        obj : Optional[ServiceModel] = session.query(ServiceModel).filter_by(service_uuid=service_uuid).one_or_none()
+        obj : Optional[ServiceModel] = session.query(ServiceModel)\
+            .options(selectinload(ServiceModel.service_endpoints))\
+            .options(selectinload(ServiceModel.constraints))\
+            .options(selectinload(ServiceModel.config_rules))\
+            .filter_by(service_uuid=service_uuid).one_or_none()
         return None if obj is None else obj.dump()
     obj = run_transaction(sessionmaker(bind=db_engine), callback)
     if obj is None:
@@ -145,3 +153,22 @@ def service_delete(db_engine : Engine, request : ServiceId) -> Tuple[Dict, bool]
         return num_deleted > 0
     deleted = run_transaction(sessionmaker(bind=db_engine), callback)
     return json_service_id(service_uuid, json_context_id(context_uuid)),deleted
+
+def service_select(db_engine : Engine, request : ServiceFilter) -> List[Dict]:
+    service_uuids = [
+        service_get_uuid(service_id, allow_random=False)[1]
+        for service_id in request.service_ids.service_ids
+    ]
+    dump_params = dict(
+        include_endpoint_ids=request.include_endpoint_ids,
+        include_constraints =request.include_constraints,
+        include_config_rules=request.include_config_rules,
+    )
+    def callback(session : Session) -> List[Dict]:
+        query = session.query(ServiceModel)
+        if request.include_endpoint_ids: query = query.options(selectinload(ServiceModel.service_endpoints))
+        if request.include_constraints : query = query.options(selectinload(ServiceModel.constraints))
+        if request.include_config_rules: query = query.options(selectinload(ServiceModel.config_rules))
+        obj_list : List[ServiceModel] = query.filter(ServiceModel.service_uuid.in_(service_uuids)).all()
+        return [obj.dump(**dump_params) for obj in obj_list]
+    return run_transaction(sessionmaker(bind=db_engine), callback)

src/context/service/database/Slice.py

@@ -16,10 +16,10 @@ import datetime, logging
 from sqlalchemy import and_
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.engine import Engine
-from sqlalchemy.orm import Session, sessionmaker
+from sqlalchemy.orm import Session, selectinload, sessionmaker
 from sqlalchemy_cockroachdb import run_transaction
 from typing import Dict, List, Optional, Set, Tuple
-from common.proto.context_pb2 import ContextId, Slice, SliceId
+from common.proto.context_pb2 import ContextId, Slice, SliceFilter, SliceId
 from common.method_wrappers.ServiceExceptions import InvalidArgumentException, NotFoundException
 from common.tools.object_factory.Context import json_context_id
 from common.tools.object_factory.Slice import json_slice_id
@@ -44,14 +44,26 @@ def slice_list_ids(db_engine : Engine, request : ContextId) -> List[Dict]:
 def slice_list_objs(db_engine : Engine, request : ContextId) -> List[Dict]:
     context_uuid = context_get_uuid(request, allow_random=False)
     def callback(session : Session) -> List[Dict]:
-        obj_list : List[SliceModel] = session.query(SliceModel).filter_by(context_uuid=context_uuid).all()
+        obj_list : List[SliceModel] = session.query(SliceModel)\
+            .options(selectinload(SliceModel.slice_endpoints))\
+            .options(selectinload(SliceModel.slice_services))\
+            .options(selectinload(SliceModel.slice_subslices))\
+            .options(selectinload(SliceModel.constraints))\
+            .options(selectinload(SliceModel.config_rules))\
+            .filter_by(context_uuid=context_uuid).all()
         return [obj.dump() for obj in obj_list]
     return run_transaction(sessionmaker(bind=db_engine), callback)
 
 def slice_get(db_engine : Engine, request : SliceId) -> Dict:
     _,slice_uuid = slice_get_uuid(request, allow_random=False)
     def callback(session : Session) -> Optional[Dict]:
-        obj : Optional[SliceModel] = session.query(SliceModel).filter_by(slice_uuid=slice_uuid).one_or_none()
+        obj : Optional[SliceModel] = session.query(SliceModel)\
+            .options(selectinload(SliceModel.slice_endpoints))\
+            .options(selectinload(SliceModel.slice_services))\
+            .options(selectinload(SliceModel.slice_subslices))\
+            .options(selectinload(SliceModel.constraints))\
+            .options(selectinload(SliceModel.config_rules))\
+            .filter_by(slice_uuid=slice_uuid).one_or_none()
         return None if obj is None else obj.dump()
     obj = run_transaction(sessionmaker(bind=db_engine), callback)
     if obj is None:
@@ -240,3 +252,26 @@ def slice_delete(db_engine : Engine, request : SliceId) -> Tuple[Dict, bool]:
         return num_deleted > 0
     deleted = run_transaction(sessionmaker(bind=db_engine), callback)
     return json_slice_id(slice_uuid, json_context_id(context_uuid)),deleted
+
+def slice_select(db_engine : Engine, request : SliceFilter) -> List[Dict]:
+    slice_uuids = [
+        slice_get_uuid(slice_id, allow_random=False)[1]
+        for slice_id in request.slice_ids.slice_ids
+    ]
+    dump_params = dict(
+        include_endpoint_ids=request.include_endpoint_ids,
+        include_constraints =request.include_constraints,
+        include_service_ids =request.include_service_ids,
+        include_subslice_ids=request.include_subslice_ids,
+        include_config_rules=request.include_config_rules,
+    )
+    def callback(session : Session) -> List[Dict]:
+        query = session.query(SliceModel)
+        if request.include_endpoint_ids: query = query.options(selectinload(SliceModel.slice_endpoints))
+        if request.include_service_ids : query = query.options(selectinload(SliceModel.slice_services))
+        if request.include_subslice_ids: query = query.options(selectinload(SliceModel.slice_subslices))
+        if request.include_constraints : query = query.options(selectinload(SliceModel.constraints))
+        if request.include_config_rules: query = query.options(selectinload(SliceModel.config_rules))
+        obj_list : List[SliceModel] = query.filter(SliceModel.slice_uuid.in_(slice_uuids)).all()
+        return [obj.dump(**dump_params) for obj in obj_list]
+    return run_transaction(sessionmaker(bind=db_engine), callback)