Initial version of Hackfest material

This is a quick guide for having fast access to the commands

################################################################################

# Preparation (already done):

################################################################################

$ cd ~/tfs-ctrl/hackfest

$ pip install pyang

$ pip install git+https://github.com/robshakir/pyangbind.git

$ pip install netconf

$ sudo apt-get install -y graphviz openjdk-11-jre

# Download plantuml.7997.jar from http://sourceforge.net/projects/plantuml/files/plantuml.7997.jar/download

cp ~/Downloads/plantuml.7997.jar ~/tfs-ctrl/hackfest/yang/plantuml.jar

sudo apt-get install -y wireshark

#   When asked "allow non-root users to capture?" answer "yes"

sudo usermod -a -G wireshark $USER

################################################################################

# YANG

################################################################################

$ cd ~/tfs-ctrl/hackfest/yang

$ cat topology.yang

$ pyang -f tree topology.yang

$ pyang -f tree topology.yang -o topology.tree

$ pyang -f sample-xml-skeleton --sample-xml-skeleton-annotations topology.yang

$ pyang -f sample-xml-skeleton --sample-xml-skeleton-annotations topology.yang -o topology-sample.xml

$ pyang -f uml topology.yang -o topology.uml

$ java -jar plantuml.jar topology.uml

PYANGBIND

---------

$ export PYBINDPLUGIN=`/usr/bin/env python3 -c \

'import pyangbind; import os; print ("{}/plugin".format(os.path.dirname(pyangbind.__file__)))'`

$ echo $PYBINDPLUGIN

$ pyang -f pybind topology.yang --plugindir $PYBINDPLUGIN -o binding_topology.py

$ python3 topology.py

EXERCISE: CONNECTION

--------------------

$ cd ~/tfs-ctrl/hackfest/yang/connection

$ export PYBINDPLUGIN=`/usr/bin/env python3 -c \

'import pyangbind; import os; print ("{}/plugin".format(os.path.dirname(pyangbind.__file__)))'`

$ pyang -f pybind connection.yang --plugindir $PYBINDPLUGIN -o binding_connection.py

$ python3 connection.py

################################################################################

# NETCONF

################################################################################

Run server:

$ cd ~/tfs-ctrl/hackfest/netconf

$ python3 server_topology.py

In another window, run client:

$ cd ~/tfs-ctrl/hackfest/netconf

$ python3 client_topology.py

EXERCISE: NETCONF EDIT-CONFIG

-----------------------------

Run server:

$ cd ~/tfs-ctrl/hackfest/netconf/connection

$ python3 server_topology_connection.py

In another window, run client:

$ cd ~/tfs-ctrl/hackfest/netconf/connection

$ python3 client_connection.py

################################################################################

# TAPI

################################################################################

RUN TAPI SERVER:

(if needed)

$ pip install -r ~/tfs-ctrl/hackfest/tapi/server/requirements.txt

$ cd ~/tfs-ctrl/hackfest/tapi/server

$ python3 tapi_server.py

RUN TAPI CLIENT (In a new window):

$ cd ~/tfs-ctrl/hackfest/tapi/client

$ curl -X GET -H "Content-Type: application/json" http://127.0.0.1:8080/restconf/data/context/

$ curl -X GET -H "Content-Type: application/json" http://127.0.0.1:8080/restconf/data/context/service-interface-point/

$ curl -X GET -H "Content-Type: application/json" http://127.0.0.1:8080/restconf/data/context/service-interface-point/sip-pe1-uni1/

$ curl -X GET -H "Content-Type: application/json" http://127.0.0.1:8080/restconf/data/context/topology/

$ curl -X GET -H "Content-Type: application/json" http://127.0.0.1:8080/restconf/data/context/topology/topo-nwk/

$ curl -X GET -H "Content-Type: application/json" http://127.0.0.1:8080/restconf/data/context/topology/topo-nwk/node/

$ curl -X GET -H "Content-Type: application/json" http://127.0.0.1:8080/restconf/data/context/topology/topo-nwk/node/node-pe-1/

$ curl -X GET -H "Content-Type: application/json" http://127.0.0.1:8080/restconf/data/context/topology/topo-nwk/node/node-pe-1/owned-node-edge-point/

$ curl -X GET -H "Content-Type: application/json" http://127.0.0.1:8080/restconf/data/context/topology/topo-nwk/node/node-pe-1/owned-node-edge-point/NEP_PE_01_UNI1/

$ curl -X GET -H "Content-Type: application/json" http://127.0.0.1:8080/restconf/data/context/topology/topo-nwk/link/PE1_NNI3_PI3_NNI1/

CONNECTIONS:

$ curl -X POST -H "Content-Type: application/json" http://127.0.0.1:8080/restconf/data/context/connectivity-service/cs1/ -d @cs1.json

$ curl -X GET -H "Content-Type: application/json" http://127.0.0.1:8080/restconf/data/context/connectivity-service/

$ curl -X GET -H "Content-Type: application/json" http://127.0.0.1:8080/restconf/data/context/connectivity-service/cs1/

$ curl -X GET -H "Content-Type: application/json" http://127.0.0.1:8080/restconf/data/context/connection/cs1/

$ curl -X DELETE -H "Content-Type: application/json" http://127.0.0.1:8080/restconf/data/context/connectivity-service/cs1/

EXERCISE: TAPI_APP

$ cd ~/tfs-ctrl/hackfest/tapi/tapi_app

$ python3 tapi_app.py

== GRPC

$ cd ~/tfs-ctrl/hackfest/grpc

COMPILE connection.proto:

$ python -m grpc_tools.protoc -I=. --python_out=connection/ connection.proto

RUN CREATE AND LIST CONNECTION

$ cd ~/tfs-ctrl/hackfest/grpc/connection

$ python3 create.py connection.txt 

$ python3 list.py connection.txt

COMPILE connectionService.proto

$ python -m grpc_tools.protoc -I=. --python_out=connectionService/ --grpc_python_out=connectionService/ connectionService.proto

RUN SERVER

$ cd ~/tfs-ctrl/hackfest/grpc/connectionService

$ python3 connectionService_server.py

RUN CLIENT (in another window)

$ cd ~/tfs-ctrl/hackfest/grpc/connectionService

$ python3 connectionService_client.py

### EXERCISE

COMPILE connectionServiceWithNotif.proto

$ cd ~/tfs-ctrl/hackfest/grpc/

$ python -m grpc_tools.protoc -I=. --python_out=connectionServiceWithNotif/ --grpc_python_out=connectionServiceWithNotif/ connectionServiceWithNotif.proto

RUN SERVER

$ cd ~/tfs-ctrl/hackfest/grpc/connectionServiceWithNotif

$ python3 connectionServiceWithNotif_server.py

RUN CLIENT (in another window)

$ cd ~/tfs-ctrl/hackfest/grpc/connectionServiceWithNotif

$ python3 connectionServiceWithNotif_client.py

== GNMI

$ cd /usr/share/gocode/src/ 

$ export GOPATH=/usr/share/gocode/

$ go run github.com/openconfig/ygot/generator/generator.go -generate_fakeroot -output_file github.com/google/gnxi/gnmi/modeldata/gostruct/generated.go -package_name gostruct github.com/rvilalta/OFC_SC472/yang/topology.yang 

$ cd /usr/share/gocode/src/github.com/google/gnxi/gnmi_target

$ go run gnmi_target.go -bind_address :10161 -config ~/tfs-ctrl/hackfest/gnmi/topology.json --notls -alsologtostderr

RUN CLIENT (in another window)

$ export GOPATH=/usr/share/gocode/

$ cd /usr/share/gocode/src/github.com/google/gnxi/gnmi_get

$ go run gnmi_get.go -notls -xpath "/topology/" -target_addr localhost:10161 -alsologtostderr 

$ go run gnmi_get.go -notls -xpath "/topology/node[node-id=A]" -target_addr localhost:10161 -alsologtostderr

USE PYTHON CLIENT

$ cd /usr/share/gocode/src/github.com/google/gnxi/gnmi_cli_py

$ python py_gnmicli.py -n -m get -t localhost -p 10161 -x /topology -u foo -pass bar

== KAFKA

$ cd ~/tfs-ctrl/hackfest/kafka

(INSTALL)

$ pip3 install kafka-python

$ wget https://ftp.cixug.es/apache/kafka/2.8.0/kafka_2.13-2.8.0.tgz

$ tar -xzf kafka_2.13-2.8.0.tgz

(RUN)

$ cd kafka_2.13-2.8.0

$ bin/zookeeper-server-start.sh config/zookeeper.properties

(In new window)

$ cd ~/tfs-ctrl/hackfest/kafka/kafka_2.13-2.8.0

$ bin/kafka-server-start.sh config/server.properties

CREATE TOPIC

(In new window)

$ cd ~/tfs-ctrl/hackfest/kafka/kafka_2.13-2.8.0

$ bin/kafka-topics.sh --create --topic my-topic --bootstrap-server localhost:9092

(In new window)

$ cd ~/tfs-ctrl/hackfest/kafka

$ python3 sub.py

(In new window)

$ cd ~/tfs-ctrl/hackfest/kafka

$ python3 pub.py

== APPENDIX: CONFD

$ cd ~/tfs-ctrl/hackfest/netconf

$ unzip confd-basic-6.4.linux.x86_64.zip  

$ cd confd-basic-6.4.linux.x86_64/

$ ./confd-basic-6.4.linux.x86_64.installer.bin /root/confd/

Data model compilation

$ cd /root/confd/bin/

$ ./confdc -c ~/tfs-ctrl/hackfest/yang/topology.yang

Start ConfD

$ ./confd --foreground -v --addloadpath .

In another terminal, use ConfD-client to populate model

$ cd /root/confd/bin/

$ ./confd_cli

> conf

> topology node node1

> exit

> commit

> exit

> exit

Use ConfD-client to show db

$ ./confd_cli

> conf

> show full-configuration

> exit

> exit

== RUN ONOS

$ cd /root/onos-2.1.0/apache-karaf-4.2.3/bin/

$ ./karaf clean

> app activate org.onosproject.openflow

> app activate org.onosproject.gui

In another terminal, run mininet:

$ mn --topo linear,3 --mac --controller=remote,ip=127.0.0.1,port=6653 --switch ovs,protocols=OpenFlow13

In another terminal, we use ONOS REST API:

$ curl -X GET -u onos:rocks --header 'Accept: application/json' http://localhost:8181/onos/v1/links | python -m json.tool

$ cd ~/tfs-ctrl/hackfest/onos_api/

$ python3 onos_topology.py 

################################################################################

# OTHER MATERIAL

################################################################################

== RESTCONF

=== YANG2SWAGGER

$ cd ~/tfs-ctrl/hackfest/restconf

(if needed)

$ wget https://github.com/bartoszm/yang2swagger/releases/download/1.1.11/swagger-generator-cli-1.1.11-executable.jar

Generate swagger files:

$ java -jar swagger-generator-cli-1.1.11-executable.jar -yang-dir ../yang/ -output topology.yaml topology

$ java -jar swagger-generator-cli-1.1.11-executable.jar -yang-dir ../yang/ -output connection.yaml connection

=== SERVER

$ cd ~/tfs-ctrl/hackfest/restconf

(if needed)

$ wget https://repo1.maven.org/maven2/io/swagger/codegen/v3/swagger-codegen-cli/3.0.11/swagger-codegen-cli-3.0.11.jar -O swagger-codegen-cli.jar

Create the server:

$ mkdir ~/tfs-ctrl/hackfest/restconf/server

$ java -jar swagger-codegen-cli.jar generate -i connection.yaml -l python-flask -o server

Run the server:

$ cd ~/tfs-ctrl/hackfest/restconf/server

$ pip3 install -r requirements.txt

(Open ~/tfs-ctrl/hackfest/restconf/server/swagger_server/swagger/swagger.yaml and modify all: "name: connection_id" for "name: connection-id")

RUN AUTOGENERATED SERVER

$ python3 -m swagger_server

(you have the solution in ~/tfs-ctrl/hackfest/restconf/connectionserver )

RUN CURL AS CLIENT (In another window)

$ cd ~/tfs-ctrl/hackfest/restconf/

$ curl -X POST -H "Content-Type: application/yang-data+json" http://127.0.0.1:8080/data/connection/ -d@conn1.json

$ curl -X GET -H "Content-Type: application/yang-data+json" http://127.0.0.1:8080/data/connection=0/

$ curl -X DELETE -H "Content-Type: application/yang-data+json" http://127.0.0.1:8080/data/connection=0/

=== Exercise: RESTCONF TOPOLOGY ===

$ mkdir ~/tfs-ctrl/hackfest/restconf/topologyserver

$ java -jar swagger-codegen-cli.jar generate -i topology.yaml -l python-flask -o topologyserver

$ cd ~/tfs-ctrl/hackfest/restconf/topologyserver

(Open ~/tfs-ctrl/hackfest/restconf/topologyserver/swagger_server/swagger/swagger.yaml and modify all: "name: link_id" for "name: link-id", same for node and port)

$ python3 -m swagger_server

RUN CURL AS CLIENT (In another window)

$curl -X GET -H "Content-Type: application/yang-data+json" http://127.0.0.1:8080/data/topology/

{

  "topology" : {

     "node" : [

       { "node-id" : "A" , "port" : [ { "port-id" : "portA1" } ] },

       { "node-id" : "B" , "port" : [ { "port-id" : "portB1" } ] }

     ]

  }

}

//Example of connection

syntax = "proto3";

package connection;

message Connection {

  string connectionId = 1;

  string sourceNode = 2;  

  string targetNode = 3;

  string sourcePort = 4;  

  string targetPort = 5;  

  uint32 bandwidth = 6;

  enum LayerProtocolName {

    ETH = 0;

    OPTICAL = 1;

  }

  LayerProtocolName layerProtocolName = 7;

}

message ConnectionList {

  repeated Connection connection = 1;

}

# Generated by the protocol buffer compiler.  DO NOT EDIT!

# source: connection.proto

import sys

_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))

from google.protobuf import descriptor as _descriptor

from google.protobuf import message as _message

from google.protobuf import reflection as _reflection

from google.protobuf import symbol_database as _symbol_database

# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()

DESCRIPTOR = _descriptor.FileDescriptor(

  name='connection.proto',

  package='connection',

  syntax='proto3',

  serialized_options=None,

  serialized_pb=_b('\n\x10\x63onnection.proto\x12\nconnection\"\xf5\x01\n\nConnection\x12\x14\n\x0c\x63onnectionId\x18\x01 \x01(\t\x12\x12\n\nsourceNode\x18\x02 \x01(\t\x12\x12\n\ntargetNode\x18\x03 \x01(\t\x12\x12\n\nsourcePort\x18\x04 \x01(\t\x12\x12\n\ntargetPort\x18\x05 \x01(\t\x12\x11\n\tbandwidth\x18\x06 \x01(\r\x12\x43\n\x11layerProtocolName\x18\x07 \x01(\x0e\x32(.connection.Connection.LayerProtocolName\")\n\x11LayerProtocolName\x12\x07\n\x03\x45TH\x10\x00\x12\x0b\n\x07OPTICAL\x10\x01\"<\n\x0e\x43onnectionList\x12*\n\nconnection\x18\x01 \x03(\x0b\x32\x16.connection.Connectionb\x06proto3')

)

_CONNECTION_LAYERPROTOCOLNAME = _descriptor.EnumDescriptor(

  name='LayerProtocolName',

  full_name='connection.Connection.LayerProtocolName',

  filename=None,

  file=DESCRIPTOR,

  values=[

    _descriptor.EnumValueDescriptor(

      name='ETH', index=0, number=0,

      serialized_options=None,

      type=None),

    _descriptor.EnumValueDescriptor(

      name='OPTICAL', index=1, number=1,

      serialized_options=None,

      type=None),

  ],

  containing_type=None,

  serialized_options=None,

  serialized_start=237,

  serialized_end=278,

)

_sym_db.RegisterEnumDescriptor(_CONNECTION_LAYERPROTOCOLNAME)

_CONNECTION = _descriptor.Descriptor(

  name='Connection',

  full_name='connection.Connection',

  filename=None,

  file=DESCRIPTOR,

  containing_type=None,

  fields=[

    _descriptor.FieldDescriptor(

      name='connectionId', full_name='connection.Connection.connectionId', index=0,

      number=1, type=9, cpp_type=9, label=1,

      has_default_value=False, default_value=_b("").decode('utf-8'),

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      serialized_options=None, file=DESCRIPTOR),

    _descriptor.FieldDescriptor(

      name='sourceNode', full_name='connection.Connection.sourceNode', index=1,

      number=2, type=9, cpp_type=9, label=1,

      has_default_value=False, default_value=_b("").decode('utf-8'),

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      serialized_options=None, file=DESCRIPTOR),

    _descriptor.FieldDescriptor(

      name='targetNode', full_name='connection.Connection.targetNode', index=2,

      number=3, type=9, cpp_type=9, label=1,

      has_default_value=False, default_value=_b("").decode('utf-8'),

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      serialized_options=None, file=DESCRIPTOR),

    _descriptor.FieldDescriptor(

      name='sourcePort', full_name='connection.Connection.sourcePort', index=3,

      number=4, type=9, cpp_type=9, label=1,

      has_default_value=False, default_value=_b("").decode('utf-8'),

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      serialized_options=None, file=DESCRIPTOR),

    _descriptor.FieldDescriptor(

      name='targetPort', full_name='connection.Connection.targetPort', index=4,

      number=5, type=9, cpp_type=9, label=1,

      has_default_value=False, default_value=_b("").decode('utf-8'),

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      serialized_options=None, file=DESCRIPTOR),

    _descriptor.FieldDescriptor(

      name='bandwidth', full_name='connection.Connection.bandwidth', index=5,

      number=6, type=13, cpp_type=3, label=1,

      has_default_value=False, default_value=0,

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      serialized_options=None, file=DESCRIPTOR),

    _descriptor.FieldDescriptor(

      name='layerProtocolName', full_name='connection.Connection.layerProtocolName', index=6,

      number=7, type=14, cpp_type=8, label=1,

      has_default_value=False, default_value=0,

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      serialized_options=None, file=DESCRIPTOR),

  ],

  extensions=[

  ],

  nested_types=[],

  enum_types=[

    _CONNECTION_LAYERPROTOCOLNAME,

  ],

  serialized_options=None,

  is_extendable=False,

  syntax='proto3',

  extension_ranges=[],

  oneofs=[

  ],

  serialized_start=33,

  serialized_end=278,

)

_CONNECTIONLIST = _descriptor.Descriptor(

  name='ConnectionList',

  full_name='connection.ConnectionList',

  filename=None,

  file=DESCRIPTOR,

  containing_type=None,

  fields=[

    _descriptor.FieldDescriptor(

      name='connection', full_name='connection.ConnectionList.connection', index=0,

      number=1, type=11, cpp_type=10, label=3,

      has_default_value=False, default_value=[],

      message_type=None, enum_type=None, containing_type=None,

      is_extension=False, extension_scope=None,

      serialized_options=None, file=DESCRIPTOR),

  ],

  extensions=[

  ],

  nested_types=[],

  enum_types=[

  ],

  serialized_options=None,

  is_extendable=False,

  syntax='proto3',

  extension_ranges=[],

  oneofs=[

  ],

  serialized_start=280,

  serialized_end=340,

)

_CONNECTION.fields_by_name['layerProtocolName'].enum_type = _CONNECTION_LAYERPROTOCOLNAME

_CONNECTION_LAYERPROTOCOLNAME.containing_type = _CONNECTION

_CONNECTIONLIST.fields_by_name['connection'].message_type = _CONNECTION

DESCRIPTOR.message_types_by_name['Connection'] = _CONNECTION

DESCRIPTOR.message_types_by_name['ConnectionList'] = _CONNECTIONLIST

_sym_db.RegisterFileDescriptor(DESCRIPTOR)

Connection = _reflection.GeneratedProtocolMessageType('Connection', (_message.Message,), dict(

  DESCRIPTOR = _CONNECTION,

  __module__ = 'connection_pb2'

  # @@protoc_insertion_point(class_scope:connection.Connection)

  ))

_sym_db.RegisterMessage(Connection)

ConnectionList = _reflection.GeneratedProtocolMessageType('ConnectionList', (_message.Message,), dict(

  DESCRIPTOR = _CONNECTIONLIST,

  __module__ = 'connection_pb2'

  # @@protoc_insertion_point(class_scope:connection.ConnectionList)

  ))

_sym_db.RegisterMessage(ConnectionList)

# @@protoc_insertion_point(module_scope)

#! /usr/bin/env python3

import connection_pb2

import sys

try:

  raw_input          # Python 2

except NameError:

  raw_input = input  # Python 3

# This function fills in a Connection message based on user input.

def PromptForConnection(connection):

  connection.connectionId = raw_input("Enter connectionID: ")

  connection.sourceNode = raw_input("Enter sourceNode: ")  

  connection.targetNode = raw_input("Enter targetNode: ")

  connection.sourcePort = raw_input("Enter sourcePort: ")  

  connection.targetPort = raw_input("Enter targetPort: ") 

  connection.bandwidth = int( raw_input("Enter bandwidth: ") )

  type = raw_input("Is this a eth or optical connection? ")

  if type == "eth":

    connection.layerProtocolName = connection_pb2.Connection.ETH

  elif type == "optical":

    connection.layerProtocolName = connection_pb2.Connection.OPTICAL

  else:

    print("Unknown layerProtocolName type; leaving as default value.")

if __name__ == '__main__':

  if len(sys.argv) != 2:

    print("Usage:", sys.argv[0], "CONNECTION_FILE")

    sys.exit(-1)

  connectionList = connection_pb2.ConnectionList()

  # Read the existing address book.

  try:

    with open(sys.argv[1], "rb") as f:

      connectionList.ParseFromString(f.read())

  except IOError:

    print(sys.argv[1] + ": File not found.  Creating a new file.")

  # Add an address.

  PromptForConnection(connectionList.connection.add())

  # Write the new address book back to disk.

  with open(sys.argv[1], "wb") as f:

    f.write(connectionList.SerializeToString())