te: Add support for protocol buffer

manifests/teservice.yaml

@@ -47,15 +47,11 @@ spec:
             fieldRef:
               fieldPath: status.podIP
         readinessProbe:
-          initialDelaySeconds: 10
-          periodSeconds: 10
           exec:
             command: ["/tfte/bin/tfte", "status"]
         livenessProbe:
-          initialDelaySeconds: 10
-          periodSeconds: 10
-          exec:
-            command: ["/tfte/bin/tfte", "status"]
+          grpc:
+            port: 11010
         resources:
           requests:
             cpu: 250m

proto/.gitignore

@@ -3,5 +3,8 @@ src/*/*
 # used to prevent breaking symbolic links from source code folders
 !src/*/.gitignore
 !src/python/__init__.py
+!src/erlang/rebar.config
+!src/erlang/rebar.lock
+!src/erlang/src/tfpb.app.src
 
 uml/generated

proto/generate_code_erlang.sh

+#!/bin/bash -eu
+# Copyright 2021-2023 H2020 TeraFlow (https://www.teraflow-h2020.eu/)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+FORCE=0
+DEFAULT_ACTION="generate"
+
+usage() {
+    echo "Usage: $0 [-f] [clean|generate]" 1>&2
+    echo "Options:"
+    echo "  -f: Force regeneration of all protocol buffers"
+    exit 1;
+}
+
+while getopts "fc" o; do
+    case "${o}" in
+        f)
+            FORCE=1
+            ;;
+        *)
+            usage
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+ACTION=${1:-$DEFAULT_ACTION}
+cd $(dirname $0)
+ROOT=$(pwd)
+ERLANG_PROTO_DIR="$ROOT/src/erlang"
+BUILD_CHECK="$ERLANG_PROTO_DIR/.generated"
+
+tfpb_clean() {
+    rm -f "$BUILD_CHECK"
+    rm -rf "$ERLANG_PROTO_DIR/src/"*.erl
+    rm -rf "$ERLANG_PROTO_DIR/src/erlang/_build"
+}
+
+tfpb_generate() {
+    if [[ -f "$BUILD_CHECK" && $FORCE != 1 ]]; then
+        echo "Protocol buffer code for Erlang already generated, use -f to force"
+        exit 0
+    fi
+
+    tfpb_clean
+    mkdir -p "$ERLANG_PROTO_DIR"
+    cd "$ERLANG_PROTO_DIR"
+    rebar3 compile
+    rebar3 grpc gen
+    rebar3 compile
+    touch "$BUILD_CHECK"
+
+    echo "Protocol buffer code for Erlang generated"
+}
+
+case "$ACTION" in
+    clean) tfpb_clean;;
+    generate) tfpb_generate;;
+    *) usage;;
+esac
+

proto/src/erlang/.gitignore

+*
+!rebar.config
+!rebar.lock
+!src/tfpb.app.src

proto/src/erlang/rebar.config

+{erl_opts, [debug_info]}.
+{deps, [grpcbox]}.
+
+{grpc, [{protos, "../.."},
+        {gpb_opts, [{i, "../.."}, {strbin, true}, {descriptor, true}, {module_name_suffix, "_pb"}]}]}.
+
+{plugins, [grpcbox_plugin]}.

proto/src/erlang/rebar.lock

+{"1.2.0",
+[{<<"acceptor_pool">>,{pkg,<<"acceptor_pool">>,<<"1.0.0">>},1},
+ {<<"chatterbox">>,{pkg,<<"ts_chatterbox">>,<<"0.12.0">>},1},
+ {<<"ctx">>,{pkg,<<"ctx">>,<<"0.6.0">>},1},
+ {<<"gproc">>,{pkg,<<"gproc">>,<<"0.8.0">>},1},
+ {<<"grpcbox">>,{pkg,<<"grpcbox">>,<<"0.15.0">>},0},
+ {<<"hpack">>,{pkg,<<"hpack_erl">>,<<"0.2.3">>},2}]}.
+[
+{pkg_hash,[
+ {<<"acceptor_pool">>, <<"43C20D2ACAE35F0C2BCD64F9D2BDE267E459F0F3FD23DAB26485BF518C281B21">>},
+ {<<"chatterbox">>, <<"4E54F199E15C0320B85372A24E35554A2CCFC4342E0B7CD8DAED9A04F9B8EF4A">>},
+ {<<"ctx">>, <<"8FF88B70E6400C4DF90142E7F130625B82086077A45364A78D208ED3ED53C7FE">>},
+ {<<"gproc">>, <<"CEA02C578589C61E5341FCE149EA36CCEF236CC2ECAC8691FBA408E7EA77EC2F">>},
+ {<<"grpcbox">>, <<"97C7126296A091602D372EBF5860A04F7BC795B45B33A984CAD2B8E362774FD8">>},
+ {<<"hpack">>, <<"17670F83FF984AE6CD74B1C456EDDE906D27FF013740EE4D9EFAA4F1BF999633">>}]},
+{pkg_hash_ext,[
+ {<<"acceptor_pool">>, <<"0CBCD83FDC8B9AD2EEE2067EF8B91A14858A5883CB7CD800E6FCD5803E158788">>},
+ {<<"chatterbox">>, <<"6478C161BC60244F41CD5847CC3ACCD26D997883E9F7FACD36FF24533B2FA579">>},
+ {<<"ctx">>, <<"A14ED2D1B67723DBEBBE423B28D7615EB0BDCBA6FF28F2D1F1B0A7E1D4AA5FC2">>},
+ {<<"gproc">>, <<"580ADAFA56463B75263EF5A5DF4C86AF321F68694E7786CB057FD805D1E2A7DE">>},
+ {<<"grpcbox">>, <<"161ABE9E17E7D1982EFA6488ADEAA13C3E847A07984A6E6B224E553368918647">>},
+ {<<"hpack">>, <<"06F580167C4B8B8A6429040DF36CC93BBA6D571FAEAEC1B28816523379CBB23A">>}]}
+].

src/te/Dockerfile

@@ -17,14 +17,16 @@
 # Build stage 0
 FROM erlang:24.3-alpine
 
-RUN mkdir /buildroot
-WORKDIR /buildroot
+RUN apk add --no-cache bash
 
-# Copy our Erlang application
-COPY . tfte
+RUN mkdir /var/teraflow
+WORKDIR /var/teraflow
 
-# And build the release
-WORKDIR tfte
+COPY proto proto
+RUN bash -c proto/generate_code_erlang.sh
+RUN mkdir src
+COPY src/te src/te
+WORKDIR src/te
 RUN rebar3 as prod release
 
 # Build stage 1
@@ -36,10 +38,10 @@ RUN apk add --no-cache libgcc libstdc++ && \
     apk add --no-cache ncurses-libs
 
 # Install the released application
-COPY --from=0 /buildroot/tfte/_build/prod/rel/tfte /tfte
+COPY --from=0 /var/teraflow/src/te/_build/prod/rel/tfte /tfte
 
 # Expose relevant ports
-#EXPOSE ????
+EXPOSE 11010
 
 ARG ERLANG_LOGGER_LEVEL_DEFAULT=debug
 ARG ERLANG_COOKIE_DEFAULT=tfte-unsafe-cookie

src/te/README.md

@@ -6,6 +6,12 @@ The Traffic Engineering service is tested on Ubuntu 20.04. Follow the instructio
 
 ## Build
 
+First the TeraFlow protocol buffer code must have been generated:
+
+    $ ../../proto/generate_code_erlang.sh
+
+Then the TE service can be built:
+
     $ rebar3 compile
 
 
@@ -25,26 +31,42 @@ Then you  can start the service in console mode:
     $ rebar3 shell
 
 
-## Build Docker Image
+## Docker
+
+### Build Image
+
+The docker image must be built from the root of the Teraflow project:
+
+    $ docker build -t te:dev -f src/te/Dockerfile .
 
-    $ docker build -t te:dev .
 
+### Run a shell from inside the container
 
-## Run Docker Container
+    $ docker run -ti --rm --entrypoint sh te:dev
+
+
+### Run Docker Container
 
     $ docker run -d --name te --init te:dev
 
 
-## Open a Console to a Docker Container
+### Open a Console to a Docker Container's Service
 
     $ docker exec -it te /tfte/bin/tfte remote_console
 
 
-## Open a Console to a Kubernetes Pod
+### Show Logs
+
+    $ docker logs te
+
+
+## Kubernetes
+
+### Open a Console
 
     $ kubectl --namespace tfs exec -ti $(kubectl --namespace tfs get pods --selector=app=teservice -o name) -- /tfte/bin/tfte remote_console
 
 
 ## Show Logs
 
-    $ docker logs te
+    $ kubectl --namespace tfs logs $(kubectl --namespace tfs get pods --selector=app=teservice -o name)
\ No newline at end of file

src/te/apps/tfte/src/tfte.app.src

@@ -5,7 +5,8 @@
   {mod, {tfte_app, []}},
   {applications,
    [kernel,
-    stdlib
+    stdlib,
+    tfpb
    ]},
   {env,[]},
   {modules, []},

src/te/apps/tfte/src/tfte_app.erl

-%%%-------------------------------------------------------------------
+%%%-----------------------------------------------------------------------------
 %% @doc tfte public API
 %% @end
-%%%-------------------------------------------------------------------
+%%%-----------------------------------------------------------------------------
 
 -module(tfte_app).
 
 -behaviour(application).
 
+
+%--- Includes ------------------------------------------------------------------
+
+-include_lib("kernel/include/logger.hrl").
+
+
+%--- Exports -------------------------------------------------------------------
+
+% Behaviour application callback functions
 -export([start/2, stop/1]).
 
+
+%--- Behaviour application Callback Functions ----------------------------------
+
 start(_StartType, _StartArgs) ->
-    tfte_sup:start_link().
+    case tfte_sup:start_link() of
+        {ok, Pid} ->
+            add_services(),
+            {ok, Pid};
+        Other ->
+            Other
+    end.
 
 stop(_State) ->
     ok.
 
-%% internal functions
+
+%--- Internal Functions --------------------------------------------------------
+
+add_services() ->
+    case application:get_env(tfte, services) of
+        {ok, Services} -> add_services(Services);
+        _ -> ok
+    end.
+
+add_services([]) -> ok;
+add_services([{Name, EndpointsSpecs, GrpcOpts} | Rest]) ->
+    try resolve_endpoints(Name, EndpointsSpecs, []) of
+        Endpoints ->
+            case grpcbox_channel_sup:start_child(Name, Endpoints, GrpcOpts) of
+                {ok, _Pid} ->
+                    ?LOG_INFO("GRPC channel to ~s service started", [Name]),
+                    ok;
+                {error, Reason} ->
+                    ?LOG_WARNING("GRPC channel to ~s service failed to start: ~p",
+                                 [Name, Reason]),
+                    ok
+            end
+    catch
+        throw:{Name, Reason, Extra} ->
+            ?LOG_WARNING("Failed to resolve ~s service configuration: ~s ~p ~p",
+                         [Name, Reason, Extra])
+    end,
+    add_services(Rest).
+
+resolve_endpoints(_Name, [], Acc) ->
+    lists:reverse(Acc);
+resolve_endpoints(Name, [{Transport, HostSpec, PortSpec, SslOpts} | Rest], Acc) ->
+    Acc2 = [{Transport, resolve_host_spec(Name, HostSpec),
+             resolve_port_spec(Name, PortSpec), SslOpts} | Acc],
+    resolve_endpoints(Name, Rest, Acc2).
+
+resolve_host_spec(_Name, Hostname) when is_list(Hostname) -> Hostname;
+resolve_host_spec(Name, {env, Key}) when is_list(Key) ->
+    ?LOG_DEBUG("????? HOST ~s ~s -> ~p", [Name, Key, os:getenv(Key)]),
+    try os:getenv(Key) of
+        false -> throw({Name, service_hostname_not_found, Key});
+        Hostname -> Hostname
+    catch
+        _:Reason ->
+            throw({Name, service_hostname_error, Reason})
+    end.
+
+resolve_port_spec(_Name, Port) when is_integer(Port) -> Port;
+resolve_port_spec(Name, {env, Key}) when is_list(Key) ->
+    ?LOG_DEBUG("????? PORT ~s ~s -> ~p", [Name, Key, os:getenv(Key)]),
+    try os:getenv(Key) of
+        false -> throw({Name, service_port_not_found, Key});
+        PortStr ->
+            try list_to_integer(PortStr) of
+                Port -> Port
+            catch
+                _:Reason ->
+                    throw({Name, service_port_error, Reason})
+            end
+    catch
+        _:Reason ->
+            throw({Name, service_port_error, Reason})
+    end.

src/te/apps/tfte/src/tfte_sup.erl

-%%%-------------------------------------------------------------------
+%%%-----------------------------------------------------------------------------
 %% @doc tfte top level supervisor.
 %% @end
-%%%-------------------------------------------------------------------
+%%%-----------------------------------------------------------------------------
 
 -module(tfte_sup).
 
 -behaviour(supervisor).
 
+
+%--- Exports -------------------------------------------------------------------
+
+% API Functions
 -export([start_link/0]).
 
+% Behaviour supervisor callback functions
 -export([init/1]).
 
+
+%--- Macros --------------------------------------------------------------------
+
 -define(SERVER, ?MODULE).
 
+
+%--- API Functions -------------------------------------------------------------
+
 start_link() ->
     supervisor:start_link({local, ?SERVER}, ?MODULE, []).
 
-%% sup_flags() = #{strategy => strategy(),         % optional
-%%                 intensity => non_neg_integer(), % optional
-%%                 period => pos_integer()}        % optional
-%% child_spec() = #{id => child_id(),       % mandatory
-%%                  start => mfargs(),      % mandatory
-%%                  restart => restart(),   % optional
-%%                  shutdown => shutdown(), % optional
-%%                  type => worker(),       % optional
-%%                  modules => modules()}   % optional
+
+%--- Behaviour supervisor Callback Functions -----------------------------------
+
 init([]) ->
     SupFlags = #{strategy => one_for_all,
                  intensity => 0,
                  period => 1},
     ChildSpecs = [],
     {ok, {SupFlags, ChildSpecs}}.
-
-%% internal functions

src/te/apps/tfte/src/tfte_te_service.erl

+-module(tfte_te_service).
+
+-behaviour(te_te_service_bhvr).
+
+
+%--- Includes ------------------------------------------------------------------
+
+-include_lib("grpcbox/include/grpcbox.hrl").
+
+
+%--- Exports -------------------------------------------------------------------
+
+% Behaviour te_te_service_bhvr callback functions
+-export([request_lsp/2]).
+-export([update_lsp/2]).
+-export([delete_lsp/2]).
+
+
+%--- Behaviour te_te_service_bhvr Callback Functions ---------------------------
+
+request_lsp(_Ctx, _Service) ->
+    {error, {?GRPC_STATUS_UNIMPLEMENTED, <<"Not yet implemented">>},
+             #{headers => #{}, trailers => #{}}}.
+
+update_lsp(_Ctx, _Service) ->
+    {error, {?GRPC_STATUS_UNIMPLEMENTED, <<"Not yet implemented">>},
+             #{headers => #{}, trailers => #{}}}.
+
+delete_lsp(_Ctx, _Service) ->
+    {error, {?GRPC_STATUS_UNIMPLEMENTED, <<"Not yet implemented">>},
+             #{headers => #{}, trailers => #{}}}.

src/te/config/dev.config.template

 [
     {tfte, [
+        {services, [
+            {te, [{http, "localhost", 11010, []}], #{}},
+    ]},
+
+    {grpcbox, [
+        {servers, [#{
+            grpc_opts => #{
+                service_protos => [te_pb, grpcbox_health_pb, grpcbox_reflection_pb],
+                %client_cert_dir => "",
+                services => #{
+                    'te.TEService' => tfte_te_service,
+                    'grpc.health.v1.Health' => grpcbox_health_service,
+                    'grpc.reflection.v1alpha.ServerReflection' => grpcbox_reflection_service
+                }
+            },
+            transport_opts => #{
+                ssl => false
+                %keyfile => "",
+                %certfile => "",
+                %cacertfile => ""
+            },
+            listen_opts => #{
+                port => 11010,
+                ip => {0,0,0,0}
+            },
+            pool_opts => #{
+                size => 10
+            },
+            server_opts => #{
+                header_table_size => 4096,
+                enable_push => 1,
+                max_concurrent_streams => unlimited,
+                initial_window_size => 65535,
+                max_frame_size => 16384,
+                max_header_list_size => unlimited
+            }
+        }]}
     ]},
 
     {kernel, [

src/te/config/sys.config.src

 [
     {tfte, [
+        {services, [
+            {te, [
+                {http, {env, "TESERVICE_SERVICE_HOST"}, {env, "TESERVICE_SERVICE_PORT_GRPC"}, []}
+            ], #{}},
+            {service, [
+                {http, {env, "SERVICESERVICE_SERVICE_HOST"}, {env, "SERVICESERVICE_SERVICE_PORT_GRPC"}, []}
+            ], #{}},
+            {monitoring, [
+                {http, {env, "MONITORINGSERVICE_SERVICE_HOST"}, {env, "MONITORINGSERVICE_SERVICE_PORT_GRPC"}, []}
+            ], #{}},
+            {compute, [
+                {http, {env, "COMPUTESERVICE_SERVICE_HOST"}, {env, "COMPUTESERVICE_SERVICE_PORT_GRPC"}, []}
+            ], #{}},
+            {device, [
+                {http, {env, "DEVICESERVICE_SERVICE_HOST"}, {env, "DEVICESERVICE_SERVICE_PORT_GRPC"}, []}
+            ], #{}},
+            {context, [
+                {http, {env, "CONTEXTSERVICE_SERVICE_HOST"}, {env, "CONTEXTSERVICE_SERVICE_PORT_GRPC"}, []}
+            ], #{}},
+            {automation, [
+                {http, {env, "AUTOMATIONSERVICE_SERVICE_HOST"}, {env, "AUTOMATIONSERVICE_SERVICE_PORT_GRPC"}, []}
+            ], #{}}
+        ]}
+    ]},
+
+    {grpcbox, [
+        {servers, [#{
+            grpc_opts => #{
+                service_protos => [te_pb, grpcbox_health_pb, grpcbox_reflection_pb],
+                %client_cert_dir => "",
+                services => #{
+                    'te.TEService' => tfte_te_service,
+                    'grpc.health.v1.Health' => grpcbox_health_service,
+                    'grpc.reflection.v1alpha.ServerReflection' => grpcbox_reflection_service
+                }
+            },
+            transport_opts => #{
+                ssl => false
+                %keyfile => "",
+                %certfile => "",
+                %cacertfile => ""
+            },
+            listen_opts => #{
+                port => 11010,
+                ip => {0,0,0,0}
+            },
+            pool_opts => #{
+                size => 10
+            },
+            server_opts => #{
+                header_table_size => 4096,
+                enable_push => 1,
+                max_concurrent_streams => unlimited,
+                initial_window_size => 65535,
+                max_frame_size => 16384,
+                max_header_list_size => unlimited
+            }
+        }]}
     ]},
 
     {kernel, [

src/te/rebar.config

 {erl_opts, [debug_info]}.
 
-{deps, [
-]}.
+{deps, [grpcbox]}.
 
 {shell, [
     {config, "config/dev.config"},
-    {apps, [tfte]}
+    {apps, [tfte, tfpb, grpcbox]}
 ]}.
 
+{project_app_dirs, ["apps/*", "../../proto/src/erlang"]}.
+
 {relx, [
-    {release, {tfte, "1.0.0"}, [tfte]},
+    {release, {tfte, "1.0.0"}, [tfte, tfpb, grpcbox]},
     {vm_args_src, "config/vm.args.src"},
     {sys_config_src, "config/sys.config.src"},
     {dev_mode, true},

src/te/rebar.lock

-[].
+{"1.2.0",
+[{<<"acceptor_pool">>,{pkg,<<"acceptor_pool">>,<<"1.0.0">>},1},
+ {<<"chatterbox">>,{pkg,<<"ts_chatterbox">>,<<"0.12.0">>},1},
+ {<<"ctx">>,{pkg,<<"ctx">>,<<"0.6.0">>},1},
+ {<<"gproc">>,{pkg,<<"gproc">>,<<"0.8.0">>},1},
+ {<<"grpcbox">>,{pkg,<<"grpcbox">>,<<"0.15.0">>},0},
+ {<<"hpack">>,{pkg,<<"hpack_erl">>,<<"0.2.3">>},2}]}.
+[
+{pkg_hash,[
+ {<<"acceptor_pool">>, <<"43C20D2ACAE35F0C2BCD64F9D2BDE267E459F0F3FD23DAB26485BF518C281B21">>},
+ {<<"chatterbox">>, <<"4E54F199E15C0320B85372A24E35554A2CCFC4342E0B7CD8DAED9A04F9B8EF4A">>},
+ {<<"ctx">>, <<"8FF88B70E6400C4DF90142E7F130625B82086077A45364A78D208ED3ED53C7FE">>},
+ {<<"gproc">>, <<"CEA02C578589C61E5341FCE149EA36CCEF236CC2ECAC8691FBA408E7EA77EC2F">>},
+ {<<"grpcbox">>, <<"97C7126296A091602D372EBF5860A04F7BC795B45B33A984CAD2B8E362774FD8">>},
+ {<<"hpack">>, <<"17670F83FF984AE6CD74B1C456EDDE906D27FF013740EE4D9EFAA4F1BF999633">>}]},
+{pkg_hash_ext,[
+ {<<"acceptor_pool">>, <<"0CBCD83FDC8B9AD2EEE2067EF8B91A14858A5883CB7CD800E6FCD5803E158788">>},
+ {<<"chatterbox">>, <<"6478C161BC60244F41CD5847CC3ACCD26D997883E9F7FACD36FF24533B2FA579">>},
+ {<<"ctx">>, <<"A14ED2D1B67723DBEBBE423B28D7615EB0BDCBA6FF28F2D1F1B0A7E1D4AA5FC2">>},
+ {<<"gproc">>, <<"580ADAFA56463B75263EF5A5DF4C86AF321F68694E7786CB057FD805D1E2A7DE">>},
+ {<<"grpcbox">>, <<"161ABE9E17E7D1982EFA6488ADEAA13C3E847A07984A6E6B224E553368918647">>},
+ {<<"hpack">>, <<"06F580167C4B8B8A6429040DF36CC93BBA6D571FAEAEC1B28816523379CBB23A">>}]}
+].