index.html


<!doctype html>
<html lang="en" class="no-js">
  <head>
    
      <meta charset="utf-8">
      <meta name="viewport" content="width=device-width,initial-scale=1">
      
        <meta name="description" content="ETSI SDG TFS Documentation page">
      
      
        <meta name="author" content="TeraflowSDN by ETSI">
      
      
        <link rel="canonical" href="https://tfs.etsi.org/v3.0.0/deployment_guide/">
      
      
        <link rel="prev" href="..">
      
      
        <link rel="next" href="../development_guide/">
      
      
      <link rel="icon" href="../images/favicon.png">
      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.44">
    
    
        <title>1. Deployment Guide - ETSI SDG TFS Documentation</title>
      
    
      <link rel="stylesheet" href="../assets/stylesheets/main.0253249f.min.css">
      
        
        <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
      
      
        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
      
    
      <link rel="stylesheet" href="../stylesheets/extra.css">
    
    <script>__md_scope=new URL("..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
    
      
  </head>
  
  
    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="0D47A1" data-md-color-accent="1565C0">
  
    
    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
    <label class="md-overlay" for="__drawer"></label>
    <div data-md-component="skip">
      
        
        <a href="#11-configure-your-machine" class="md-skip">
          Skip to content
        </a>
      
    </div>
    <div data-md-component="announce">
      
    </div>
    
      <div data-md-color-scheme="default" data-md-component="outdated" hidden>
        
      </div>
    
    
<header class="md-header md-header--shadow" data-md-component="header">
  <nav class="md-header__inner md-grid" aria-label="Header">
    <a href=".." title="ETSI SDG TFS Documentation" class="md-header__button md-logo" aria-label="ETSI SDG TFS Documentation" data-md-component="logo">
      
  <img src="../images/logos/TeraFlow SDN Logo Screen_B_W NEG.png" alt="logo">

    </a>
    <label class="md-header__button md-icon" for="__drawer">
      
      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
    </label>
    <div class="md-header__title" data-md-component="header-title">
      <div class="md-header__ellipsis">
        <div class="md-header__topic">
          <span class="md-ellipsis">
            ETSI SDG TFS Documentation
          </span>
        </div>
        <div class="md-header__topic" data-md-component="header-topic">
          <span class="md-ellipsis">
            
              1. Deployment Guide
            
          </span>
        </div>
      </div>
    </div>
    
      
      <label class="md-header__button md-icon" for="__search">
        
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
      </label>
      <div class="md-search" data-md-component="search" role="dialog">
  <label class="md-search__overlay" for="__search"></label>
  <div class="md-search__inner" role="search">
    <form class="md-search__form" name="search">
      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
      <label class="md-search__icon md-icon" for="__search">
        
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
        
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
      </label>
      <nav class="md-search__options" aria-label="Search">
        
        <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
          
          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
        </button>
      </nav>
      
    </form>
    <div class="md-search__output">
      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
        <div class="md-search-result" data-md-component="search-result">
          <div class="md-search-result__meta">
            Initializing search
          </div>
          <ol class="md-search-result__list" role="presentation"></ol>
        </div>
      </div>
    </div>
  </div>
</div>
    
    
      <div class="md-header__source">
        <a href="https://labs.etsi.org/rep/tfs" title="Go to repository" class="md-source" data-md-component="source">
  <div class="md-source__icon md-icon">
    
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="m503.5 204.6-.7-1.8-69.7-181.78c-1.4-3.57-3.9-6.59-7.2-8.64-2.4-1.55-5.1-2.515-8-2.81s-5.7.083-8.4 1.11c-2.7 1.02-5.1 2.66-7.1 4.78-1.9 2.12-3.3 4.67-4.1 7.44l-47 144H160.8l-47.1-144c-.8-2.77-2.2-5.31-4.1-7.43-2-2.12-4.4-3.75-7.1-4.77a18.1 18.1 0 0 0-8.38-1.113 18.4 18.4 0 0 0-8.04 2.793 18.1 18.1 0 0 0-7.16 8.64L9.267 202.8l-.724 1.8a129.57 129.57 0 0 0-3.52 82c7.747 26.9 24.047 50.7 46.447 67.6l.27.2.59.4 105.97 79.5 52.6 39.7 32 24.2c3.7 1.9 8.3 4.3 13 4.3s9.3-2.4 13-4.3l32-24.2 52.6-39.7 106.7-79.9.3-.3c22.4-16.9 38.7-40.6 45.6-67.5 8.6-27 7.4-55.8-2.6-82"/></svg>
  </div>
  <div class="md-source__repository">
    TFS GitLab
  </div>
</a>
      </div>
    
  </nav>
  
</header>
    
    <div class="md-container" data-md-component="container">
      
      
      <main class="md-main" data-md-component="main">
        <div class="md-main__inner md-grid">
          
            
              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">
                    

<nav class="md-nav md-nav--primary md-nav--integrated" aria-label="Navigation" data-md-level="0">
  <label class="md-nav__title" for="__drawer">
    <a href=".." title="ETSI SDG TFS Documentation" class="md-nav__button md-logo" aria-label="ETSI SDG TFS Documentation" data-md-component="logo">
      
  <img src="../images/logos/TeraFlow SDN Logo Screen_B_W NEG.png" alt="logo">

    </a>
    ETSI SDG TFS Documentation
  </label>
  
    <div class="md-nav__source">
      <a href="https://labs.etsi.org/rep/tfs" title="Go to repository" class="md-source" data-md-component="source">
  <div class="md-source__icon md-icon">
    
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="m503.5 204.6-.7-1.8-69.7-181.78c-1.4-3.57-3.9-6.59-7.2-8.64-2.4-1.55-5.1-2.515-8-2.81s-5.7.083-8.4 1.11c-2.7 1.02-5.1 2.66-7.1 4.78-1.9 2.12-3.3 4.67-4.1 7.44l-47 144H160.8l-47.1-144c-.8-2.77-2.2-5.31-4.1-7.43-2-2.12-4.4-3.75-7.1-4.77a18.1 18.1 0 0 0-8.38-1.113 18.4 18.4 0 0 0-8.04 2.793 18.1 18.1 0 0 0-7.16 8.64L9.267 202.8l-.724 1.8a129.57 129.57 0 0 0-3.52 82c7.747 26.9 24.047 50.7 46.447 67.6l.27.2.59.4 105.97 79.5 52.6 39.7 32 24.2c3.7 1.9 8.3 4.3 13 4.3s9.3-2.4 13-4.3l32-24.2 52.6-39.7 106.7-79.9.3-.3c22.4-16.9 38.7-40.6 45.6-67.5 8.6-27 7.4-55.8-2.6-82"/></svg>
  </div>
  <div class="md-source__repository">
    TFS GitLab
  </div>
</a>
    </div>
  
  <ul class="md-nav__list" data-md-scrollfix>
    
      
    <li class="md-nav__item">
      <a href=".." class="md-nav__link">
        
  
  <span class="md-ellipsis">
    0. Home
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item md-nav__item--active">
      
      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
      
      
        <label class="md-nav__link md-nav__link--active" for="__toc">
          
  
  <span class="md-ellipsis">
    1. Deployment Guide
  </span>
  

          <span class="md-nav__icon md-icon"></span>
        </label>
      
      <a href="./" class="md-nav__link md-nav__link--active">
        
  
  <span class="md-ellipsis">
    1. Deployment Guide
  </span>
  

      </a>
      
        
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
  
  
    <label class="md-nav__title" for="__toc">
      <span class="md-nav__icon md-icon"></span>
      Table of contents
    </label>
    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
      
        <li class="md-nav__item">
  <a href="#11-configure-your-machine" class="md-nav__link">
    <span class="md-ellipsis">
      1.1. Configure your Machine
    </span>
  </a>
  
    <nav class="md-nav" aria-label="1.1. Configure your Machine">
      <ul class="md-nav__list">
        
          <li class="md-nav__item">
  <a href="#111-physical-server" class="md-nav__link">
    <span class="md-ellipsis">
      1.1.1. Physical Server
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#112-oracle-virtual-box" class="md-nav__link">
    <span class="md-ellipsis">
      1.1.2. Oracle Virtual Box
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#113-vmware-fusion" class="md-nav__link">
    <span class="md-ellipsis">
      1.1.3. VMWare Fusion
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#114-openstack" class="md-nav__link">
    <span class="md-ellipsis">
      1.1.4. OpenStack
    </span>
  </a>
  
</li>
        
          <li class="md-nav__item">
  <a href="#115-vagrant-box" class="md-nav__link">
    <span class="md-ellipsis">
      1.1.5. Vagrant Box
    </span>
  </a>
  
</li>
        
      </ul>
    </nav>
  
</li>
      
        <li class="md-nav__item">
  <a href="#12-install-microk8s" class="md-nav__link">
    <span class="md-ellipsis">
      1.2. Install MicroK8s
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#13-deploy-teraflowsdn" class="md-nav__link">
    <span class="md-ellipsis">
      1.3. Deploy TeraFlowSDN
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#14-webui-and-grafana-dashboards" class="md-nav__link">
    <span class="md-ellipsis">
      1.4. WebUI and Grafana Dashboards
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#15-show-deployment-and-logs" class="md-nav__link">
    <span class="md-ellipsis">
      1.5. Show Deployment and Logs
    </span>
  </a>
  
</li>
      
    </ul>
  
</nav>
      
    </li>
  

    <li class="md-nav__item">
      <a href="../development_guide/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    2. Development Guide
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../run_experiments/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    3. Run Experiments
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../features_and_bugs/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    4. Feature and bugs
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../supported_sbis_and_network_elements/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    5. Supported SBIs and Network Elements
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../supported_nbis/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    6. Supported NBIs
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../supported_service_handlers/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    7. Supported Service Handlers
  </span>
  

      </a>
    </li>
  

    <li class="md-nav__item">
      <a href="../troubleshooting/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    8. Troubleshooting
  </span>
  

      </a>
    </li>
  

  </ul>
</nav>
                  </div>
                </div>
              </div>
            
            
            <div class="md-content" data-md-component="content">
              <article class="md-content__inner md-typeset">
                
                  
  <h1>1. Deployment Guide</h1>

<p>This section walks you through the process of deploying TeraFlowSDN on top of a machine running <a href="https://microk8s.io">MicroK8s Kubernetes platform</a>.
The guide includes the details on configuring and installing the machine, installing and 
configuring MicroK8s, and deploying and reporting the status of the TeraFlowSDN 
controller.</p>
<h2 id="11-configure-your-machine"><strong>1.1. Configure your Machine</strong></h2>
<p>In this section, we describe how to configure a machine (physical or virtual) to be used as the deployment, execution, and development environment for the ETSI TeraFlowSDN controller. Choose your preferred environment below and follow the instructions provided.</p>
<p><strong>NOTE</strong>: If you already have a remote physical server fitting the requirements specified in this section feel free to use it instead of deploying a local VM. Check <a href="#111-physical-server">1.1.1. Physical Server</a> for further details.</p>
<p>Virtualization platforms tested are:</p>
<ul>
<li><a href="#111-physical-server">Physical Server</a></li>
<li><a href="#112-oracle-virtual-box">Oracle Virtual Box</a></li>
<li><a href="#113-vmware-fusion">VMWare Fusion</a></li>
<li><a href="#114-openstack">OpenStack</a></li>
<li><a href="#115-vagrant-box">Vagrant Box</a></li>
</ul>
<h3 id="111-physical-server"><strong>1.1.1. Physical Server</strong></h3>
<p>This section describes how to configure a physical server for running ETSI TeraFlowSDN(TFS) controller.</p>
<h3><u>Server Specifications</u></h3>

<p><strong>Minimum Server Specifications for development and basic deployment</strong></p>
<ul>
<li>CPU: 4 cores</li>
<li>RAM: 8 GB</li>
<li>Disk: 60 GB</li>
<li>1 GbE NIC</li>
</ul>
<p><strong>Recommended Server Specifications for development and basic deployment</strong></p>
<ul>
<li>CPU: 6 cores</li>
<li>RAM: 12 GB</li>
<li>Disk: 80 GB</li>
<li>1 GbE NIC</li>
</ul>
<p><strong>Server Specifications for best development and deployment experience</strong></p>
<ul>
<li>CPU: 8 cores</li>
<li>RAM: 32 GB</li>
<li>Disk: 120 GB</li>
<li>1 GbE NIC</li>
</ul>
<p><strong>NOTE</strong>: the specifications listed above are provided as a reference. They depend also on the CPU clock frequency, the RAM memory, the disk technology and speed, etc.</p>
<p>For development purposes, it is recommended to run the VSCode IDE (or the IDE of your choice) in a more powerful server, for instance, the recommended server specifications for development and basic deployment.</p>
<p>Given that TeraFlowSDN follows a micro-services architecture, for the deployment, it might be better to use many clusterized servers with many slower cores than a single server with few highly performant cores.</p>
<h3><u>Clusterized Deployment</u></h3>
<p>You might consider creating a cluster of machines each featuring, at least, the minimum server specifications. That solution brings you scalability in the future.</p>
<h3><u>Networking</u></h3>
<p>No explicit indications are given in terms of networking besides that servers need access to the Internet for downloading dependencies, binaries, and packages while building and deploying the TeraFlowSDN components.</p>
<p>Besides that, the network requirements are essentially the same than that required for running a classical Kubernetes environment. To facilitate the deployment, we extensively use <a href="https://microk8s.io/">MicroK8s</a>, thus the network requirements are, essentially, the same demanded by MicroK8s, especially, if you consider creating a Kubernetes cluster.</p>
<p>As a reference, the other deployment solutions based on VMs assume the VM is connected to a virtual network configured with the IP range <code>10.0.2.0/24</code> and have the gateway at IP <code>10.0.2.1</code>. The VMs have the IP address <code>10.0.2.10</code>.</p>
<p>The minimum required ports to be accessible are:</p>
<ul>
<li>22/SSH     : for management purposes</li>
<li>80/HTTP    : for the TeraFlowSDN WebUI and Grafana dashboard</li>
<li>8081/HTTPS : for the CockroachDB WebUI</li>
</ul>
<p>Other ports might be required if you consider to deploy addons such as Kubernetes observability, etc. The details on these ports are left appart given they might vary depending on the Kubernetes environment you use.</p>
<h3><u>Operating System</u></h3>

<p>The recommended Operating System for deploying TeraFlowSDN is <a href="https://releases.ubuntu.com/jammy/">Ubuntu Server 22.04 LTS</a> or <a href="https://releases.ubuntu.com/focal/">Ubuntu Server 20.04 LTS</a>. Other version might work, but we have not tested them. We strongly recommend using Long Term Support (LTS) versions as they provide better stability.</p>
<p>Below we provide some installation guidelines:</p>
<ul>
<li>Installation Language: English</li>
<li>Autodetect your keyboard</li>
<li>If asked, select "Ubuntu Server" (do not select "Ubuntu Server (minimized)").</li>
<li>Configure static network specifications (adapt them based on your particular setup):</li>
</ul>
<table>
<thead>
<tr>
<th>Interface</th>
<th>IPv4 Method</th>
<th>Subnet</th>
<th>Address</th>
<th>Gateway</th>
<th>Name servers</th>
<th>Search domains</th>
</tr>
</thead>
<tbody>
<tr>
<td>enp0s3</td>
<td>Manual</td>
<td>10.0.2.0/24</td>
<td>10.0.2.10</td>
<td>10.0.2.1</td>
<td>8.8.8.8,8.8.4.4</td>
<td><empty></td>
</tr>
</tbody>
</table>
<ul>
<li>Leave proxy and mirror addresses as they are</li>
<li>Let the installer self-upgrade (if asked).</li>
<li>Use an entire disk for the installation<ul>
<li>Disable setup of the disk as LVM group</li>
<li>Double check that NO swap space is allocated in the partition table. Kubernetes does not work properly with SWAP.</li>
</ul>
</li>
<li>Configure your user and system names:<ul>
<li>User name: <code>TeraFlowSDN</code></li>
<li>Server's name: <code>tfs-vm</code></li>
<li>Username: <code>tfs</code></li>
<li>Password: <code>tfs123</code></li>
</ul>
</li>
<li>Install Open SSH Server<ul>
<li>Import SSH keys, if any.</li>
</ul>
</li>
<li>Featured Server Snaps<ul>
<li>Do not install featured server snaps. It will be done manually later to illustrate how to uninstall and reinstall them in case of trouble with.</li>
</ul>
</li>
<li>Let the system install and upgrade the packages.<ul>
<li>This operation might take some minutes depending on how old is the Optical Drive ISO image you use and your Internet connection speed.</li>
</ul>
</li>
<li>Restart the VM when the installation is completed.</li>
</ul>
<h3><u>Upgrade the Ubuntu distribution</u></h3>

<pre><code class="language-bash">sudo apt-get update -y
sudo apt-get dist-upgrade -y
</code></pre>
<ul>
<li>If asked to restart services, restart the default ones proposed.</li>
<li>Restart the VM when the installation is completed.</li>
</ul>
<h3 id="112-oracle-virtual-box"><strong>1.1.2. Oracle Virtual Box</strong></h3>
<p>This section describes how to configure a VM for running ETSI TeraFlowSDN(TFS) controller using <a href="https://www.virtualbox.org/">Oracle VirtualBox</a>. It has been tested with VirtualBox up to version 6.1.40 r154048.</p>
<h3><u>Create a NAT Network in VirtualBox</u></h3>
<p>In "Oracle VM VirtualBox Manager", Menu "File &gt; Preferences... &gt; Network", create a NAT 
network with the following specifications:</p>
<table>
<thead>
<tr>
<th>Name</th>
<th>CIDR</th>
<th>DHCP</th>
<th>IPv6</th>
</tr>
</thead>
<tbody>
<tr>
<td>TFS-NAT-Net</td>
<td>10.0.2.0/24</td>
<td>Disabled</td>
<td>Disabled</td>
</tr>
</tbody>
</table>
<p>Within the newly created "TFS-NAT-Net" NAT network, configure the following IPv4 
forwarding rules:</p>
<table>
<thead>
<tr>
<th>Name</th>
<th>Protocol</th>
<th>Host IP</th>
<th>Host Port</th>
<th>Guest IP</th>
<th>Guest Port</th>
</tr>
</thead>
<tbody>
<tr>
<td>SSH</td>
<td>TCP</td>
<td>127.0.0.1</td>
<td>2200</td>
<td>10.0.2.10</td>
<td>22</td>
</tr>
<tr>
<td>HTTP</td>
<td>TCP</td>
<td>127.0.0.1</td>
<td>8080</td>
<td>10.0.2.10</td>
<td>80</td>
</tr>
</tbody>
</table>
<p><strong>Note</strong>: IP address 10.0.2.10 is the one that will be assigned to the VM.</p>
<h3><u>Create VM in VirtualBox:</u></h3>

<ul>
<li>Name: TFS-VM</li>
<li>Type/Version: Linux / Ubuntu (64-bit)</li>
<li>CPU (*): 4 vCPUs @ 100% execution capacity</li>
<li>RAM: 8 GB</li>
<li>Disk: 60 GB, Virtual Disk Image (VDI), Dynamically allocated</li>
<li>Optical Drive ISO Image: "ubuntu-22.04.X-live-server-amd64.iso"<ul>
<li>Download the latest Long Term Support (LTS) version of the <em>Ubuntu Server</em> image from <a href="https://releases.ubuntu.com/22.04/">Ubuntu 22.04 LTS</a>, e.g., "ubuntu-22.04.X-live-server-amd64.iso".</li>
<li><strong>Note</strong>: use Ubuntu Server image instead of Ubuntu Desktop to create a lightweight VM.</li>
</ul>
</li>
<li>Network Adapter 1 (*): enabled, attached to NAT Network "TFS-NAT-Net"</li>
<li>Minor adjustments (*):<ul>
<li>Audio: disabled</li>
<li>Boot order: disable "Floppy"</li>
</ul>
</li>
</ul>
<p><strong>Note</strong>: (*) settings to be editing after the VM is created.</p>
<h3><u>Install Ubuntu 22.04 LTS Operating System</u></h3>

<p>In "Oracle VM VirtualBox Manager", start the VM in normal mode, and follow the 
installation procedure.
Below we provide some installation guidelines:</p>
<ul>
<li>Installation Language: English</li>
<li>Autodetect your keyboard</li>
<li>If asked, select "Ubuntu Server" (do not select "Ubuntu Server (minimized)").</li>
<li>Configure static network specifications:</li>
</ul>
<table>
<thead>
<tr>
<th>Interface</th>
<th>IPv4 Method</th>
<th>Subnet</th>
<th>Address</th>
<th>Gateway</th>
<th>Name servers</th>
<th>Search domains</th>
</tr>
</thead>
<tbody>
<tr>
<td>enp0s3</td>
<td>Manual</td>
<td>10.0.2.0/24</td>
<td>10.0.2.10</td>
<td>10.0.2.1</td>
<td>8.8.8.8,8.8.4.4</td>
<td><empty></td>
</tr>
</tbody>
</table>
<ul>
<li>Leave proxy and mirror addresses as they are</li>
<li>Let the installer self-upgrade (if asked).</li>
<li>Use an entire disk for the installation<ul>
<li>Disable setup of the disk as LVM group</li>
<li>Double check that NO swap space is allocated in the partition table. Kubernetes does not work properly with SWAP.</li>
</ul>
</li>
<li>Configure your user and system names:<ul>
<li>User name: TeraFlowSDN</li>
<li>Server's name: tfs-vm</li>
<li>Username: tfs</li>
<li>Password: tfs123</li>
</ul>
</li>
<li>Install Open SSH Server<ul>
<li>Import SSH keys, if any.</li>
</ul>
</li>
<li>Featured Server Snaps<ul>
<li>Do not install featured server snaps. It will be done manually later to illustrate how to uninstall and reinstall them in case of trouble with.</li>
</ul>
</li>
<li>Let the system install and upgrade the packages.<ul>
<li>This operation might take some minutes depending on how old is the Optical Drive ISO image you use and your Internet connection speed.</li>
</ul>
</li>
<li>Restart the VM when the installation is completed.</li>
</ul>
<p><strong>Upgrade the Ubuntu distribution</strong></p>
<pre><code class="language-bash">sudo apt-get update -y
sudo apt-get dist-upgrade -y
</code></pre>
<ul>
<li>If asked to restart services, restart the default ones proposed.</li>
<li>Restart the VM when the installation is completed.</li>
</ul>
<p><strong>Install VirtualBox Guest Additions</strong></p>
<p>On VirtualBox Manager, open the VM main screen. If you are running the VM in headless 
mode, right click over the VM in the VirtualBox Manager window and click "Show".
If a dialog informing about how to leave the interface of the VM is shown, confirm 
pressing "Switch" button. The interface of the VM should appear.</p>
<p>Click menu "Device &gt; Insert Guest Additions CD image..."</p>
<p>On the VM terminal, type:</p>
<pre><code class="language-bash">sudo apt-get install -y linux-headers-$(uname -r) build-essential dkms
  # This command might take some minutes depending on your VM specs and your Internet access speed.
sudo mount /dev/cdrom /mnt/
cd /mnt/
sudo ./VBoxLinuxAdditions.run
  # This command might take some minutes depending on your VM specs.
sudo reboot
</code></pre>
<h3 id="113-vmware-fusion"><strong>1.1.3. VMWare Fusion</strong></h3>
<p>This section describes how to configure a VM for running ETSI TeraFlowSDN(TFS) controller using <a href="https://www.vmware.com/products/fusion.html">VMWare Fusion</a>. It has been tested with VMWare Fusion version 12 and 13.</p>
<h3><u>Create VM in VMWare Fusion:</u></h3>

<p>In "VMWare Fusion" manager, create a new network from the "Settings/Network" menu.</p>
<ul>
<li>Unlock to make changes</li>
<li>Press the + icon and create a new network</li>
<li>Change the name to TFS-NAT-Net</li>
<li>Check "Allow virtual machines on this network to connect to external network (NAT)"</li>
<li>Do not check "Enable IPv6"</li>
<li>Add port forwarding for HTTP and SSH</li>
<li>Uncheck "Provide address on this network via DHCP"</li>
</ul>
<p>Create a new VM an Ubuntu 22.04.1 ISO:</p>
<ul>
<li>Display Name: TeraFlowSDN</li>
<li>Username: tfs</li>
<li>Password: tfs123</li>
</ul>
<p>On the next screen press "Customize Settings", save the VM and in "Settings" change:</p>
<ul>
<li>Change to use 4 CPUs</li>
<li>Change to access 8 GB of RAM</li>
<li>Change disk to size 60 GB</li>
<li>Change the network interface to use the previously created TFS-NAT-Net</li>
</ul>
<p>Run the VM to start the installation.</p>
<h3><u>Install Ubuntu 22.04.1 LTS Operating System</u></h3>

<p>The installation will be automatic, without any configuration required.</p>
<ul>
<li>
<p>Configure the guest IP, gateway and DNS:</p>
<p>Using the Network Settings for the wired connection, set the IP to 10.0.2.10,
the mask to 255.255.255.0, the gateway to 10.0.2.2 and the DNS to 10.0.2.2.</p>
</li>
<li>
<p>Disable and remove swap file:</p>
<p>$ sudo swapoff -a
$ sudo rm /swapfile</p>
<p>Then you can remove or comment the /swapfile entry in /etc/fstab</p>
</li>
<li>
<p>Install Open SSH Server</p>
<ul>
<li>Import SSH keys, if any.</li>
</ul>
</li>
<li>
<p>Restart the VM when the installation is completed.</p>
</li>
</ul>
<h3><u>Upgrade the Ubuntu distribution</u></h3>

<pre><code class="language-bash">sudo apt-get update -y
sudo apt-get dist-upgrade -y
</code></pre>
<h3 id="114-openstack"><strong>1.1.4. OpenStack</strong></h3>
<p>This section describes how to configure a VM for running ETSI TeraFlowSDN(TFS) controller using <a href="https://www.openstack.org/">OpenStack</a>. It has been tested with OpenStack Kolla up to Yoga version. </p>
<h3><u>Create a Security Group in OpenStack</h3>
<p></u>
In OpenStack, go to Project - Network - Security Groups - Create Security Group with name TFS</p>
<p>Add the following rules:</p>
<table>
<thead>
<tr>
<th>Direction</th>
<th>Ether Type</th>
<th>IP Protocol</th>
<th>Port Range</th>
<th>Remote IP Prefix</th>
</tr>
</thead>
<tbody>
<tr>
<td>Ingress</td>
<td>IPv4</td>
<td>TCP</td>
<td>22 (SSH)</td>
<td>0.0.0.0/0</td>
</tr>
<tr>
<td>Ingress</td>
<td>IPv4</td>
<td>TCP</td>
<td>2200</td>
<td>0.0.0.0/0</td>
</tr>
<tr>
<td>Ingress</td>
<td>IPv4</td>
<td>TCP</td>
<td>8080</td>
<td>0.0.0.0/0</td>
</tr>
<tr>
<td>Ingress</td>
<td>IPv4</td>
<td>TCP</td>
<td>80</td>
<td>0.0.0.0/0</td>
</tr>
<tr>
<td>Egress</td>
<td>IPv4</td>
<td>Any</td>
<td>Any</td>
<td>0.0.0.0/0</td>
</tr>
<tr>
<td>Egress</td>
<td>IPv6</td>
<td>Any</td>
<td>Any</td>
<td>::/0</td>
</tr>
</tbody>
</table>
<p><strong>Note</strong>: The IP address will be assigned depending on the network you have configured inside OpenStack. This IP will have to be modified in TeraFlow configuration files which by default use IP 10.0.2.10</p>
<h3><u>Create a flavour</h3>
<p></u></p>
<p><strong>From dashboard (Horizon)</strong></p>
<p>Go to Admin - Compute - Flavors and press Create Flavor</p>
<ul>
<li>Name: TFS</li>
<li>VCPUs: 4</li>
<li>RAM (MB): 8192</li>
<li>Root Disk (GB): 60</li>
</ul>
<p><strong>From CLI</strong></p>
<pre><code> openstack flavor create TFS --id auto --ram 8192 --disk 60 --vcpus 8
</code></pre>
<h3><u>Create an instance in OpenStack:</h3>
<p></u></p>
<ul>
<li>Instance name: TFS-VM</li>
<li>Origin: [Ubuntu-22.04 cloud image] (https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img)</li>
<li>Create new volume: No</li>
<li>Flavor: TFS</li>
<li>Networks: extnet </li>
<li>Security Groups: TFS</li>
<li>Configuration: Include the following cloud-config</li>
</ul>
<pre><code>#cloud-config
# Modifies the password for the VM instance
username: ubuntu
password: &lt;your-password&gt;
chpasswd: { expire: False }
ssh_pwauth: True
</code></pre>
<h3><u>Upgrade the Ubuntu distribution</h3>
<p></u></p>
<pre><code class="language-bash">sudo apt-get update -y
sudo apt-get dist-upgrade -y
</code></pre>
<ul>
<li>If asked to restart services, restart the default ones proposed.</li>
<li>Restart the VM when the installation is completed.</li>
</ul>
<h3 id="115-vagrant-box"><strong>1.1.5. Vagrant Box</strong></h3>
<p>This section describes how to create a Vagrant Box, using the base virtual machine configured in <a href="#112-oracle-virtual-box">Oracle Virtual Box</a>.</p>
<h3><u>Virtual Machine specifications</h3>
<p></u></p>
<p>Most of the specifications can be as specified in the <a href="#112-oracle-virtual-box">Oracle Virtual Box</a> page, however, there are a few particularities to Vagrant that must be accommodated, such as:</p>
<ul>
<li>Virtual Hard Disk<ul>
<li>Size: 60GB (at least)</li>
<li><strong>Type</strong>: VMDK</li>
</ul>
</li>
</ul>
<p><img alt="Screenshottt_from_2024-10-21_18-13-43" src="../images/deployment_guide/01_vagrant_box.jpg" /></p>
<p>Also, before initiating the VM and installing the OS, we'll need to:</p>
<ul>
<li>Disable Floppy in the 'Boot Order'</li>
<li>Disable audio</li>
<li>Disable USB</li>
<li>Ensure Network Adapter 1 is set to NAT</li>
</ul>
<h3><u>Network configurations</h3>
<p></u>
At Network Adapt 1, the following port-forwarding rule must be set.</p>
<table>
<thead>
<tr>
<th>Name</th>
<th>Protocol</th>
<th>Host IP</th>
<th>Host Port</th>
<th>Guest IP</th>
<th>Guest Port</th>
</tr>
</thead>
<tbody>
<tr>
<td>SSH</td>
<td>TCP</td>
<td></td>
<td><strong>2222</strong></td>
<td></td>
<td>22</td>
</tr>
</tbody>
</table>
<p><img alt="Screenshot_from_2023-07-10_18-25-18" src="../images/deployment_guide/02_vagrant_box.jpg" /></p>
<h3><u>Installing the OS</h3>
<p></u></p>
<p>For a Vagrant Box, it is generally suggested that the ISO's server version is used, as it is intended to be used via SSH, and any web GUI is expected to be forwarded to the host.</p>
<p><img alt="Screenshot_from_2023-07-10_18-41-49" src="../images/deployment_guide/03_vagrant_box.jpg" /></p>
<p><img alt="Screenshot_from_2023-07-10_18-42-30" src="../images/deployment_guide/04_vagrant_box.jpg" /></p>
<p><img alt="Screenshot_from_2023-07-10_18-42-45" src="../images/deployment_guide/05_vagrant_box.jpg" /></p>
<p>Make sure the disk is not configured as an LVM group!</p>
<p><img alt="Screenshot_from_2023-07-10_18-43-16" src="../images/deployment_guide/06_vagrant_box.jpg" /></p>
<h3><u>Vagrant ser</h3>
<p></u>
Vagrant expects by default, that in the box's OS exists the user <code>vagrant</code> with the password also being <code>vagrant</code>.</p>
<p><img alt="Screenshot_from_2023-07-10_18-54-12" src="../images/deployment_guide/07_vagrant_box.jpg" /></p>
<h3><u>SSH</h3>
<p></u></p>
<p>Vagrant uses SSH to connect to the boxes, so installing it now will save the hassle of doing it later.</p>
<p><img alt="Screenshot_from_2023-07-10_18-54-48" src="../images/deployment_guide/08_vagrant_box.jpg" /></p>
<h3><u>Features server snaps</h3>
<p></u></p>
<p>Do not install featured server snaps. It will be done manually <a href="#12-install-microk8s">later</a> to illustrate how to uninstall and reinstall them in case of trouble with.</p>
<h3><u>Updates</h3>
<p></u></p>
<p>Let the system install and upgrade the packages. This operation might take some minutes depending on how old is the Optical Drive ISO image you use and your Internet connection speed.</p>
<h3><u>Upgrade the Ubuntu distribution</h3>
<p></u></p>
<pre><code class="language-bash">sudo apt-get update -y
sudo apt-get dist-upgrade -y
</code></pre>
<ul>
<li>If asked to restart services, restart the default ones proposed.</li>
<li>Restart the VM when the installation is completed.</li>
</ul>
<h3><u>Install VirtualBox Guest Additions</h3>
<p></u>
On VirtualBox Manager, open the VM main screen. If you are running the VM in headless 
mode, right-click over the VM in the VirtualBox Manager window, and click "Show".
If a dialog informing about how to leave the interface of the VM is shown, confirm 
by pressing the "Switch" button. The interface of the VM should appear.</p>
<p>Click the menu "Device &gt; Insert Guest Additions CD image..."</p>
<p>On the VM terminal, type:</p>
<pre><code class="language-bash">sudo apt-get install -y linux-headers-$(uname -r) build-essential dkms
  # This command might take some minutes depending on your VM specs and your Internet access speed.
sudo mount /dev/cdrom /mnt/
cd /mnt/
sudo ./VBoxLinuxAdditions.run
  # This command might take some minutes depending on your VM specs.
sudo reboot
</code></pre>
<h3><u>ETSI TFS Installation</h3>
<p></u>
After this, proceed to <a href="#12-install-microk8s">1.2. Install Microk8s</a>, after which, return to this wiki to finish the Vagrant Box creation.</p>
<h3><u>Box configuration and creation</h3>
<p></u>
Make sure the ETSI TFS controller is correctly configured. <strong>You will not be able to change it after!</strong></p>
<p>It is advisable to do the next configurations from a host's terminal, via a SSH connection.</p>
<pre><code class="language-bash">ssh -p 2222 vagrant@127.0.0.1
</code></pre>
<h3><u>Set root password</h3>
<p></u>
Set the root password to <code>vagrant</code>.</p>
<pre><code class="language-bash">sudo passwd root
</code></pre>
<h3><u>Set the superuser</h3>
<p></u>
Set up the Vagrant user so that it’s able to use sudo without being prompted for a password.
Anything in the <code>/etc/sudoers.d/*</code> directory is included in the sudoers privileges when created by the root user.
Create a new sudo file.</p>
<pre><code class="language-bash">sudo visudo -f /etc/sudoers.d/vagrant
</code></pre>
<p>and add the following lines</p>
<pre><code class="language-text"># add vagrant user
vagrant ALL=(ALL) NOPASSWD:ALL
</code></pre>
<p>You can now test that it works by running a simple command.</p>
<pre><code class="language-bash">sudo pwd
</code></pre>
<p>Issuing this command should result in an immediate response without a request for a password.</p>
<h3><u>Install the Vagrant key</h3>
<p></u>
Vagrant uses a default set of SSH keys for you to directly connect to boxes via the CLI command <code>vagrant ssh</code>, after which it creates a new set of SSH keys for your new box. Because of this, we need to load the default key to be able to access the box after created.</p>
<pre><code class="language-bash">chmod 0700 /home/vagrant/.ssh
wget --no-check-certificate https://raw.github.com/mitchellh/vagrant/master/keys/vagrant.pub -O /home/vagrant/.ssh/authorized_keys
chmod 0600 /home/vagrant/.ssh/authorized_keys
chown -R vagrant /home/vagrant/.ssh
</code></pre>
<h3><u>Configure the OpenSSH Server</h3>
<p></u>
Edit the <code>/etc/ssh/sshd_config</code> file:</p>
<pre><code class="language-bash">sudo vim /etc/ssh/sshd_config
</code></pre>
<p>And uncomment the following line:</p>
<pre><code class="language-bash">AuthorizedKeysFile %h/.ssh/authorized_keys
</code></pre>
<p>Then restart SSH.</p>
<pre><code class="language-bash">sudo service ssh restart
</code></pre>
<h3><u>Package the box</h3>
<p></u>
Before you package the box, if you intend to make your box public, it is best to clean your bash history with:</p>
<pre><code class="language-bash">history -c
</code></pre>
<p>Exit the SSH connection, and <strong>at you're host machine</strong>, package the VM:</p>
<pre><code class="language-bash">vagrant package --base teraflowsdncontroller --output teraflowsdncontroller.box
</code></pre>
<h3><u>Test run the box</h3>
<p></u>
Add the base box to you local Vagrant box list:</p>
<pre><code class="language-bash">vagrant box add --name teraflowsdncontroller ./teraflowsdncontroller.box
</code></pre>
<p>Now you should try to run it, for that you'll need to create a <strong>Vagrantfile</strong>. For a simple run, this is the minimal required code for this box:</p>
<pre><code class="language-ruby"># -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.configure(&quot;2&quot;) do |config|
  config.vm.box = &quot;teraflowsdncontroller&quot;
  config.vm.box_version = &quot;1.1.0&quot;
  config.vm.network :forwarded_port, host: 8080 ,guest: 80
end
</code></pre>
<p>Now you'll be able to spin up the virtual machine by issuing the command:</p>
<pre><code class="language-bash">vagrant up
</code></pre>
<p>And connect to the machine using:</p>
<pre><code class="language-bash">vagrant ssh
</code></pre>
<h3><u>Pre-configured boxes</h3>
<p></u></p>
<p>If you do not wish to create your own Vagrant Box, you can use one of the existing ones created by TFS contributors.
  - <a href="https://app.vagrantup.com/davidjosearaujo/boxes/teraflowsdncontroller">davidjosearaujo/teraflowsdncontroller</a>
  - ... <!-- Should create and host one at ETSI!! --></p>
<p>To use them, you simply have to create a Vagrantfile and run <code>vagrant up controller</code> in the same directory. The following example Vagrantfile already allows you to do just that, with the bonus of exposing the multiple management GUIs to your <code>localhost</code>.</p>
<pre><code class="language-ruby">Vagrant.configure(&quot;2&quot;) do |config|

  config.vm.define &quot;controller&quot; do |controller|
    controller.vm.box = &quot;davidjosearaujo/teraflowsdncontroller&quot;
    controller.vm.network &quot;forwarded_port&quot;, guest: 80, host: 8080     # WebUI
    controller.vm.network &quot;forwarded_port&quot;, guest: 8084, host: 50750  # Linkerd Viz Dashboard
    controller.vm.network &quot;forwarded_port&quot;, guest: 8081, host: 8081   # CockroachDB Dashboard
    controller.vm.network &quot;forwarded_port&quot;, guest: 8222, host: 8222   # NATS Dashboard
    controller.vm.network &quot;forwarded_port&quot;, guest: 9000, host: 9000   # QuestDB Dashboard
    controller.vm.network &quot;forwarded_port&quot;, guest: 9090, host: 9090   # Prometheus Dashboard

    # Setup Linkerd Viz reverse proxy
    ## Copy config file
    controller.vm.provision &quot;file&quot; do |f|
      f.source = &quot;./reverse-proxy-linkerdviz.sh&quot;
      f.destination = &quot;./reverse-proxy-linkerdviz.sh&quot;
    end
    ## Execute configuration file
    controller.vm.provision &quot;shell&quot; do |s|
      s.inline = &quot;chmod +x ./reverse-proxy-linkerdviz.sh &amp;&amp; ./reverse-proxy-linkerdviz.sh&quot;
    end

    # Update controller source code to the desired branch
    if ENV['BRANCH'] != nil
      controller.vm.provision &quot;shell&quot; do |s|
        s.inline = &quot;cd ./tfs-ctrl &amp;&amp; git pull &amp;&amp; git switch &quot; + ENV['BRANCH']
      end
    end

  end
end
</code></pre>
<p>This Vagrantfile also allows for <strong>optional repository updates</strong> on startup by running the command with a specified environment variable <code>BRANCH</code></p>
<pre><code class="language-bash">BRANCH=develop vagrant up controller
</code></pre>
<h3><u>Linkerd DNS rebinding bypass</h3>
<p></u>
Because of Linkerd's security measures against DNS rebinding, a reverse proxy, that modifies the request's header <code>Host</code> field, is needed to expose the GUI to the host. The previous Vagrantfile already deploys such configurations, for that, all you need to do is create the <code>reverse-proxy-linkerdviz.sh</code> file in the same directory. The content of this file is displayed below.</p>
<pre><code class="language-bash"># Install NGINX
sudo apt update &amp;&amp; sudo apt install nginx -y

# NGINX reverse proxy configuration
echo 'server {
    listen 8084;

    location / {
        proxy_pass http://127.0.0.1:50750;
        proxy_set_header Host localhost;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
    }
}' &gt; /home/vagrant/expose-linkerd

# Create symlink of the NGINX configuration file
sudo ln -s /home/vagrant/expose-linkerd /etc/nginx/sites-enabled/

# Commit the reverse proxy configurations
sudo systemctl restart nginx

# Enable start on login
echo &quot;linkerd viz dashboard &amp;&quot; &gt;&gt; .profile

# Start dashboard
linkerd viz dashboard &amp;

echo &quot;Linkerd Viz dashboard running!&quot;
</code></pre>
<h2 id="12-install-microk8s"><strong>1.2. Install MicroK8s</strong></h2>
<p>This section describes how to deploy the MicroK8s Kubernetes platform and configure it to be used with ETSI TeraFlowSDN controller. Besides, Docker is installed to build docker images for the ETSI TeraFlowSDN controller.</p>
<p>The steps described in this section might take some minutes depending on your internet connection speed and the resources assigned to your VM, or the specifications of your physical server.</p>
<p>To facilitate work, these steps are easier to be executed through an SSH connection, for instance using tools like <a href="https://www.putty.org/">PuTTY</a> or <a href="https://mobaxterm.mobatek.net/">MobaXterm</a>.</p>
<h3><u>Upgrade the Ubuntu distribution</h3>
<p></u>
Skip this step if you already did it during the creation of the VM.</p>
<pre><code class="language-bash">sudo apt-get update -y
sudo apt-get dist-upgrade -y
</code></pre>
<h3><u>Install prerequisites</h3>
<p></u></p>
<pre><code class="language-bash">sudo apt-get install -y ca-certificates curl gnupg lsb-release snapd jq
</code></pre>
<h3><u>Install Docker CE</h3>
<p></u>
Install Docker CE and Docker BuildX plugin</p>
<pre><code class="language-bash">sudo apt-get install -y docker.io docker-buildx
</code></pre>
<p><strong>NOTE</strong>: Starting from Docker v23, <a href="https://docs.docker.com/build/architecture/">Build architecture</a> has been updated and <code>docker build</code> command entered into deprecation process in favor of the new <code>docker buildx build</code> command. Package <code>docker-buildx</code> provides the new <code>docker buildx build</code> command.</p>
<p>Add key "insecure-registries" with the private repository to the daemon configuration. It is done in two commands since
sometimes read from and write to same file might cause trouble.</p>
<pre><code class="language-bash">if [ -s /etc/docker/daemon.json ]; then cat /etc/docker/daemon.json; else echo '{}'; fi \
    | jq 'if has(&quot;insecure-registries&quot;) then . else .+ {&quot;insecure-registries&quot;: []} end' -- \
    | jq '.&quot;insecure-registries&quot; |= (.+ [&quot;localhost:32000&quot;] | unique)' -- \
    | tee tmp.daemon.json
sudo mv tmp.daemon.json /etc/docker/daemon.json
sudo chown root:root /etc/docker/daemon.json
sudo chmod 600 /etc/docker/daemon.json
</code></pre>
<p>Restart the Docker daemon</p>
<pre><code class="language-bash">sudo systemctl restart docker
</code></pre>
<h3><u>Install MicroK8s</h3>
<p></u></p>
<p><strong>Important</strong>: By default, Kubernetes uses CIDR 10.1.0.0/16 for pods and CIDR 10.152.183.0/24 for services. If they conflict with your internal network CIDR, you might need to change Kubernetes CIDRs at deployment time. To do so, check links below and ask for support if needed.</p>
<ul>
<li><a href="https://microk8s.io/docs/how-to-dual-stack">MicroK8s - How to configure network Dual-stack</a></li>
<li><a href="https://microk8s.io/docs/change-cidr">MicroK8s - MicroK8s CNI Configuration</a></li>
</ul>
<pre><code class="language-bash"># Install MicroK8s
sudo snap install microk8s --classic --channel=1.24/stable

# Create alias for command &quot;microk8s.kubectl&quot; to be usable as &quot;kubectl&quot;
sudo snap alias microk8s.kubectl kubectl
</code></pre>
<p>It is important to make sure that <code>ufw</code> will not interfere with the internal pod-to-pod
and pod-to-Internet traffic.
To do so, first check the status.
If <code>ufw</code> is active, use the following command to enable the communication.</p>
<pre><code class="language-bash">
# Verify status of ufw firewall
sudo ufw status

# If ufw is active, install following rules to enable access pod-to-pod and pod-to-internet
sudo ufw allow in on cni0 &amp;&amp; sudo ufw allow out on cni0
sudo ufw default allow routed
</code></pre>
<p><strong>NOTE</strong>: MicroK8s can be used to compose a Highly Available Kubernetes cluster enabling you to construct an environment combining the CPU, RAM and storage resources of multiple machines. If you are interested in this procedure, review the official instructions in <a href="https://ubuntu.com/tutorials/getting-started-with-kubernetes-ha">How to build a highly available Kubernetes cluster with MicroK8s</a>, in particular, the step <a href="https://ubuntu.com/tutorials/getting-started-with-kubernetes-ha#4-create-a-microk8s-multinode-cluster">Create a MicroK8s multi-node cluster</a>.</p>
<p><strong>References:</strong></p>
<ul>
<li><a href="https://microk8s.io/#install-microk8s">The lightweight Kubernetes &gt; Install MicroK8s</a></li>
<li><a href="https://ubuntu.com/tutorials/install-a-local-kubernetes-with-microk8s">Install a local Kubernetes with MicroK8s</a></li>
<li><a href="https://ubuntu.com/tutorials/getting-started-with-kubernetes-ha">How to build a highly available Kubernetes cluster with MicroK8s</a></li>
<li><a href="https://microk8s.io/docs/how-to-dual-stack">MicroK8s - How to configure network Dual-stack</a></li>
<li><a href="https://microk8s.io/docs/change-cidr">MicroK8s - MicroK8s CNI Configuration</a></li>
</ul>
<h3><u>Add user to the docker and microk8s groups</h3>
<p></u></p>
<p>It is important that your user has the permission to run <code>docker</code> and <code>microk8s</code> in the 
terminal.
To allow this, you need to add your user to the <code>docker</code> and <code>microk8s</code> groups with the 
following commands:</p>
<pre><code class="language-bash">sudo usermod -a -G docker $USER
sudo usermod -a -G microk8s $USER
sudo chown -f -R $USER $HOME/.kube
sudo reboot
</code></pre>
<p>In case that you get trouble executing the following commands, might due to the .kube folder is not automatically provisioned into your home folder, you may follow the steps below:</p>
<pre><code class="language-bash">mkdir -p $HOME/.kube
sudo chown -f -R $USER $HOME/.kube
microk8s config &gt; $HOME/.kube/config
sudo reboot
</code></pre>
<h3><u id="check-status-of-kubernetes-and-addons">Check status of Kubernetes and addons</h3>
<p></u>
To retrieve the status of Kubernetes <strong>once</strong>, run the following command:</p>
<pre><code class="language-bash">microk8s.status --wait-ready
</code></pre>
<p>To retrieve the status of Kubernetes <strong>periodically</strong> (e.g., every 1 second), run the 
following command:</p>
<pre><code class="language-bash">watch -n 1 microk8s.status --wait-ready
</code></pre>
<h3><u id="check-all-resources-in-kubernetes">Check all resources in Kubernetes</h3>
<p></u>
To retrieve the status of the Kubernetes resources <strong>once</strong>, run the following command:</p>
<pre><code class="language-bash">kubectl get all --all-namespaces
</code></pre>
<p>To retrieve the status of the Kubernetes resources <strong>periodically</strong> (e.g., every 1 
second), run the following command:</p>
<pre><code class="language-bash">watch -n 1 kubectl get all --all-namespaces
</code></pre>
<h3><u>Enable addons</h3>
<p></u></p>
<p>First, we need to enable the community plugins (maintained by third parties):</p>
<pre><code class="language-bash">microk8s.enable community
</code></pre>
<p>The Addons to be enabled are:</p>
<ul>
<li><code>dns</code>: enables resolving the pods and services by name</li>
<li><code>helm3</code>: required to install NATS</li>
<li><code>hostpath-storage</code>: enables providing storage for the pods (required by <code>registry</code>)</li>
<li><code>ingress</code>: deploys an ingress controller to expose the microservices outside Kubernetes</li>
<li><code>registry</code>: deploys a private registry for the TFS controller images</li>
<li><code>linkerd</code>: deploys the <a href="https://linkerd.io">linkerd service mesh</a> used for load balancing among replicas</li>
<li><code>prometheus</code>: set of tools that enable TFS observability through per-component instrumentation</li>
<li><code>metrics-server</code>: deploys the <a href="https://github.com/kubernetes-sigs/metrics-server">Kubernetes metrics server</a> for API access to service metrics</li>
</ul>
<pre><code class="language-bash">microk8s.enable dns helm3 hostpath-storage ingress registry prometheus metrics-server linkerd
</code></pre>
<p><strong>Important</strong>: Enabling some of the addons might take few minutes. Do not proceed with next steps until the addons are ready. Otherwise, the deployment might fail.
To confirm everything is up and running:</p>
<ol>
<li>Periodically
   <a href="#12-install-microk8s">Check the status of Kubernetes</a>
   until you see the addons [dns, ha-cluster, helm3, hostpath-storage, ingress, linkerd, metrics-server, prometheus, registry, storage] in the enabled block.</li>
<li>Periodically
   <a href="">Check Kubernetes resources</a>
   until all pods are <strong>Ready</strong> and <strong>Running</strong>.</li>
<li>If it takes too long for the Pods to be ready, <strong>we observed that rebooting the machine may help</strong>.</li>
</ol>
<p>Then, create aliases to make the commands easier to access:</p>
<pre><code class="language-bash">sudo snap alias microk8s.helm3 helm3
sudo snap alias microk8s.linkerd linkerd
</code></pre>
<p>To validate that <code>linkerd</code> is working correctly, run:</p>
<pre><code class="language-bash">linkerd check
</code></pre>
<p>To validate that the <code>metrics-server</code> is working correctly, run:</p>
<pre><code class="language-bash">kubectl top pods --all-namespaces
</code></pre>
<p>and you should see a screen similar to the <code>top</code> command in Linux, showing the columns <em>namespace</em>, <em>pod name</em>, <em>CPU (cores)</em>, and <em>MEMORY (bytes)</em>.</p>
<p>In case pods are not starting, check information from pods logs. For example, linkerd is sensitive for proper /etc/resolv.conf syntax.</p>
<pre><code class="language-bash">kubectl logs &lt;podname&gt; --namespace &lt;namespace&gt;
</code></pre>
<p>If the command shows an error message, also restarting the machine might help.</p>
<h3><u>Stop, Restart, and Redeploy</h3>
<p></u>
Find below some additional commands you might need while you work with MicroK8s:</p>
<pre><code class="language-bash">microk8s.stop  # stop MicroK8s cluster (for instance, before power off your computer)
microk8s.start # start MicroK8s cluster
microk8s.reset # reset infrastructure to a clean state
</code></pre>
<p>If the following commands does not work to recover the MicroK8s cluster, you can redeploy it.</p>
<p>If you want to keep MicroK8s configuration, use:</p>
<pre><code class="language-bash">sudo snap remove microk8s
</code></pre>
<p>If you need to completely drop MicroK8s and its complete configuration, use:</p>
<pre><code class="language-bash">sudo snap remove microk8s --purge
sudo apt-get remove --purge docker.io docker-buildx
</code></pre>
<p><strong>IMPORTANT</strong>: After uninstalling MicroK8s, it is convenient to reboot the computer (the VM if you work on a VM, or the physical computer if you use a physical computer). Otherwise, there are system configurations that are not correctly cleaned. Especially in what port forwarding and firewall rules matters.</p>
<p>After the reboot, redeploy as it is described in this section.</p>
<h2 id="13-deploy-teraflowsdn"><strong>1.3. Deploy TeraFlowSDN</strong></h2>
<p>This section describes how to deploy TeraFlowSDN controller on top of MicroK8s using the environment configured in the previous sections.</p>
<h3><u>Install prerequisites</h3>
<p></u></p>
<pre><code class="language-bash">sudo apt-get install -y git curl jq
</code></pre>
<h3><u>Clone the Git repository of the TeraFlowSDN controller</h3>
<p></u>
Clone from ETSI-hosted GitLab code repository:</p>
<pre><code class="language-bash">mkdir ~/tfs-ctrl
git clone https://labs.etsi.org/rep/tfs/controller.git ~/tfs-ctrl
</code></pre>
<p><strong>Important</strong>: The original H2020-TeraFlow project hosted on GitLab.com has been 
archieved and will not receive further contributions/updates.
Please, clone from <a href="https://labs.etsi.org/rep/tfs/controller">ETSI-hosted GitLab code repository</a>.</p>
<h3><u>Checkout the appropriate Git branch</h3>
<p></u>
TeraFlowSDN controller versions can be found in the appropriate release tags and/or branches as described in <a href="https://tfs.etsi.org/news/">Home &gt; Versions</a>.</p>
<p>By default the branch <em>master</em> is checked out and points to the latest stable version of the TeraFlowSDN controller, while branch <em>develop</em> contains the latest developments and contributions under test and validation.</p>
<p>To switch to the appropriate branch run the following command, changing <code>develop</code> by the name of the branch you want to deploy:</p>
<pre><code class="language-bash">cd ~/tfs-ctrl
git checkout develop
</code></pre>
<h3><u>Prepare a deployment script with the deployment settings</h3>
<p></u>
Create a new deployment script, e.g., <code>my_deploy.sh</code>, adding the appropriate settings as 
follows.
This section provides just an overview of the available settings. An example <a href="https://labs.etsi.org/rep/tfs/controller/-/blob/master/my_deploy.sh"><code>my_deploy.sh</code></a> script is provided in the root folder of the project for your convenience with full description of all the settings.</p>
<p><strong>Note</strong>: The example <code>my_deploy.sh</code> script provides reasonable settings for deploying a functional and complete enough TeraFlowSDN controller, and a brief description of their meaning. To see extended descriptions, check scripts in the <code>deploy</code> folder.</p>
<pre><code class="language-bash">cd ~/tfs-ctrl
tee my_deploy.sh &gt;/dev/null &lt;&lt; EOF
# ----- TeraFlowSDN ------------------------------------------------------------
export TFS_REGISTRY_IMAGES=&quot;http://localhost:32000/tfs/&quot;
export TFS_COMPONENTS=&quot;context device ztp monitoring pathcomp service slice nbi webui load_generator&quot;
export TFS_IMAGE_TAG=&quot;dev&quot;
export TFS_K8S_NAMESPACE=&quot;tfs&quot;
export TFS_EXTRA_MANIFESTS=&quot;manifests/nginx_ingress_http.yaml&quot;
export TFS_GRAFANA_PASSWORD=&quot;admin123+&quot;
export TFS_SKIP_BUILD=&quot;&quot;

# ----- CockroachDB ------------------------------------------------------------
export CRDB_NAMESPACE=&quot;crdb&quot;
export CRDB_EXT_PORT_SQL=&quot;26257&quot;
export CRDB_EXT_PORT_HTTP=&quot;8081&quot;
export CRDB_USERNAME=&quot;tfs&quot;
export CRDB_PASSWORD=&quot;tfs123&quot;
export CRDB_DATABASE=&quot;tfs&quot;
export CRDB_DEPLOY_MODE=&quot;single&quot;
export CRDB_DROP_DATABASE_IF_EXISTS=&quot;YES&quot;
export CRDB_REDEPLOY=&quot;&quot;

# ----- NATS -------------------------------------------------------------------
export NATS_NAMESPACE=&quot;nats&quot;
export NATS_EXT_PORT_CLIENT=&quot;4222&quot;
export NATS_EXT_PORT_HTTP=&quot;8222&quot;
export NATS_REDEPLOY=&quot;&quot;

# ----- QuestDB ----------------------------------------------------------------
export QDB_NAMESPACE=&quot;qdb&quot;
export QDB_EXT_PORT_SQL=&quot;8812&quot;
export QDB_EXT_PORT_ILP=&quot;9009&quot;
export QDB_EXT_PORT_HTTP=&quot;9000&quot;
export QDB_USERNAME=&quot;admin&quot;
export QDB_PASSWORD=&quot;quest&quot;
export QDB_TABLE_MONITORING_KPIS=&quot;tfs_monitoring_kpis&quot;
export QDB_TABLE_SLICE_GROUPS=&quot;tfs_slice_groups&quot;
export QDB_DROP_TABLES_IF_EXIST=&quot;YES&quot;
export QDB_REDEPLOY=&quot;&quot;

EOF
</code></pre>
<p>The settings are organized in 4 sections:</p>
<ul>
<li>Section <code>TeraFlowSDN</code>:<ul>
<li><code>TFS_REGISTRY_IMAGE</code> enables to specify the private Docker registry to be used, by default, we assume to use the Docker respository enabled in MicroK8s.</li>
<li><code>TFS_COMPONENTS</code> specifies the components their Docker image will be rebuilt, uploaded to the private Docker registry, and deployed in Kubernetes.</li>
<li><code>TFS_IMAGE_TAG</code> defines the tag to be used for Docker images being rebuilt and uploaded to the private Docker registry.</li>
<li><code>TFS_K8S_NAMESPACE</code> specifies the name of the Kubernetes namespace to be used for deploying the TFS components.</li>
<li><code>TFS_EXTRA_MANIFESTS</code> enables to provide additional manifests to be applied into the Kubernetes environment during the deployment. Typical use case is to deploy ingress controllers, service monitors for Prometheus, etc.</li>
<li><code>TFS_GRAFANA_PASSWORD</code> lets you specify the password you want to use for the <code>admin</code> user of the Grafana instance being deployed and linked to the Monitoring component.</li>
<li><code>TFS_SKIP_BUILD</code>, if set to <code>YES</code>, prevents rebuilding the Docker images. That means, the deploy script will redeploy existing Docker images without rebuilding/updating them.</li>
</ul>
</li>
<li>Section <code>CockroachDB</code>: enables to configure the deployment of the backend <a href="https://www.cockroachlabs.com/">CockroachDB</a> database.<ul>
<li>Check example script <a href="https://labs.etsi.org/rep/tfs/controller/-/blob/master/my_deploy.sh"><code>my_deploy.sh</code></a> for further details.</li>
</ul>
</li>
<li>Section <code>NATS</code>: enables to configure the deployment of the backend <a href="https://nats.io/">NATS</a> message broker.<ul>
<li>Check example script <a href="https://labs.etsi.org/rep/tfs/controller/-/blob/master/my_deploy.sh"><code>my_deploy.sh</code></a> for further details.</li>
</ul>
</li>
<li>Section <code>QuestDB</code>: enables to configure the deployment of the backend <a href="https://questdb.io/">QuestDB</a> timeseries database.<ul>
<li>Check example script <a href="https://labs.etsi.org/rep/tfs/controller/-/blob/master/my_deploy.sh"><code>my_deploy.sh</code></a> for further details.</li>
</ul>
</li>
</ul>
<h3><u>Confirm that MicroK8s is running</h3>
<p></u></p>
<p>Run the following command:</p>
<pre><code class="language-bash">microk8s status
</code></pre>
<p>If it is reported <code>microk8s is not running, try microk8s start</code>, run the following command to start MicroK8s:</p>
<pre><code class="language-bash">microk8s start
</code></pre>
<p>Confirm everything is up and running:</p>
<ol>
<li>Periodically <a href="#check-status-of-kubernetes-and-addons">Check the status of Kubernetes</a> until you see the addons [dns, ha-cluster, helm3, hostpath-storage, ingress, registry, storage] in the enabled block.</li>
<li>Periodically <a href="#check-all-resources-in-kubernetes">Check Kubernetes resources</a> until all pods are <strong>Ready</strong> and <strong>Running</strong>.</li>
</ol>
<h3><u id="deploy-tfs-controller">Deploy TFS controller</h3>
<p></u>
First, source the deployment settings defined in the previous section.
This way, you do not need to specify the environment variables in each and every command you execute to operate the TFS controller.
Be aware to re-source the file if you open new terminal sessions.
Then, run the following command to deploy TeraFlowSDN controller on top of the MicroK8s Kubernetes platform.</p>
<pre><code class="language-bash">cd ~/tfs-ctrl
source my_deploy.sh
./deploy/all.sh
</code></pre>
<p>The script performs the following steps:</p>
<ul>
<li>Executes script <code>./deploy/crdb.sh</code> to automate deployment of CockroachDB database used by Context component.<ul>
<li>The script automatically checks if CockroachDB is already deployed.</li>
<li>If there are settings instructing to drop the database and/or redeploy CockroachDB, it does the appropriate actions to honor them as defined in previous section.</li>
</ul>
</li>
<li>Executes script <code>./deploy/nats.sh</code> to automate deployment of NATS message broker used by Context component.<ul>
<li>The script automatically checks if NATS is already deployed.</li>
<li>If there are settings instructing to redeploy the message broker, it does the appropriate actions to honor them as defined in previous section.</li>
</ul>
</li>
<li>Executes script <code>./deploy/qdb.sh</code> to automate deployment of QuestDB timeseries database used by Monitoring component.<ul>
<li>The script automatically checks if QuestDB is already deployed.</li>
<li>If there are settings instructing to redeploy the timeseries database, it does the appropriate actions to honor them as defined in previous section.</li>
</ul>
</li>
<li>Executes script <code>./deploy/tfs.sh</code> to automate deployment of TeraFlowSDN.<ul>
<li>Creates the namespace defined in <code>TFS_K8S_NAMESPACE</code></li>
<li>Creates secrets for CockroachDB, NATS, and QuestDB to be used by Context and Monitoring components.</li>
<li>Builds the Docker images for the components defined in <code>TFS_COMPONENTS</code></li>
<li>Tags the Docker images with the value of <code>TFS_IMAGE_TAG</code></li>
<li>Pushes the Docker images to the repository defined in <code>TFS_REGISTRY_IMAGE</code></li>
<li>Deploys the components defined in <code>TFS_COMPONENTS</code></li>
<li>Creates the file <code>tfs_runtime_env_vars.sh</code> with the environment variables for the components defined in <code>TFS_COMPONENTS</code> defining their local host addresses and their port numbers.</li>
<li>Applies extra manifests defined in <code>TFS_EXTRA_MANIFESTS</code> such as:</li>
<li>Creating an ingress controller listening at port 80 for HTTP connections to enable external access to the TeraFlowSDN WebUI, Grafana Dashboards, and Compute NBI interfaces.</li>
<li>Deploying service monitors to enable monitoring the performance of the components, device drivers and service handlers.</li>
<li>Initialize and configure the Grafana dashboards (if Monitoring component is deployed)</li>
</ul>
</li>
<li>Report a summary of the deployment<ul>
<li>See <a href="#15-show-deployment-and-logs">Show Deployment and Logs</a></li>
</ul>
</li>
</ul>
<h2 id="14-webui-and-grafana-dashboards"><strong>1.4. WebUI and Grafana Dashboards</strong></h2>
<p>This section describes how to get access to the TeraFlowSDN controller WebUI and the monitoring Grafana dashboards.</p>
<h3><u>Access the TeraFlowSDN WebUI</h3>
<p></u>
If you followed the installation steps based on MicroK8s, you got an ingress controller installed that exposes on TCP port 80.</p>
<p>Besides, the ingress controller defines the following reverse proxy paths (on your local machine):</p>
<ul>
<li><code>http://127.0.0.1/webui</code>: points to the WebUI of TeraFlowSDN.</li>
<li><code>http://127.0.0.1/grafana</code>: points to the Grafana dashboards.
  This endpoint brings access to the monitoring dashboards of TeraFlowSDN.
  The credentials for the <code>admin</code>user are those defined in the <code>my_deploy.sh</code> script, in the <code>TFS_GRAFANA_PASSWORD</code> variable.</li>
<li><code>http://127.0.0.1/restconf</code>: points to the Compute component NBI based on RestCONF. 
  This endpoint enables connecting external software, such as ETSI OpenSourceMANO NFV Orchestrator, to TeraFlowSDN.</li>
</ul>
<p><strong>Note</strong>: In the creation of the VM, a forward from host TCP port 8080 to VM's TCP port 80 is configured, so the WebUIs and REST APIs of TeraFlowSDN should be exposed on the endpoint <code>127.0.0.1:8080</code> of your local machine instead of <code>127.0.0.1:80</code>.</p>
<h2 id="15-show-deployment-and-logs"><strong>1.5. Show Deployment and Logs</strong></h2>
<p>This section presents some helper scripts to inspect the status of the deployment and 
the logs of the components.
These scripts are particularly helpful for troubleshooting during execution of 
experiments, development, and debugging.</p>
<h3><u>Report the deployment of the TFS controller</h3>
<p></u></p>
<p>The summary report given at the end of the <a href="#deploy-tfs-controller">Deploy TFS controller</a>
procedure can be generated manually at any time by running the following command.
You can avoid sourcing <code>my_deploy.sh</code> if it has been already done.</p>
<pre><code class="language-bash">cd ~/tfs-ctrl
source my_deploy.sh
./deploy/show.sh
</code></pre>
<p>Use this script to validate that all the pods, deployments, replica sets, ingress 
controller, etc. are ready and have the appropriate state, e.g., <em>running</em> for Pods, and 
the services are deployed and have appropriate IP addresses and port numbers.</p>
<h3><u>Report the log of a specific TFS controller component</h3>
<p></u></p>
<p>A number of scripts are pre-created in the <code>scripts</code> folder to facilitate the inspection 
of the component logs.
For instance, to dump the log of the Context component, run the following command.
You can avoid sourcing <code>my_deploy.sh</code> if it has been already done.</p>
<pre><code class="language-bash">source my_deploy.sh
./scripts/show_logs_context.sh
</code></pre>


              </article>
            </div>
          
          
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
        </div>
        
          <button type="button" class="md-top md-icon" data-md-component="top" hidden>
  
  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8z"/></svg>
  Back to top
</button>
        
      </main>
      
        <footer class="md-footer">
  
    
      <nav class="md-footer__inner md-grid" aria-label="Footer" >
        
          
          <a href=".." class="md-footer__link md-footer__link--prev" aria-label="Previous: 0. Home">
            <div class="md-footer__button md-icon">
              
              <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
            </div>
            <div class="md-footer__title">
              <span class="md-footer__direction">
                Previous
              </span>
              <div class="md-ellipsis">
                0. Home
              </div>
            </div>
          </a>
        
        
          <a href="../development_guide/" class="md-footer__link md-footer__link--next" aria-label="Next: 2. Development Guide">
            <div class="md-footer__title">
              <span class="md-footer__direction">
                Next
              </span>
              <div class="md-ellipsis">
                2. Development Guide
              </div>
            </div>
            <div class="md-footer__button md-icon">
              
              <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 11v2h12l-5.5 5.5 1.42 1.42L19.84 12l-7.92-7.92L10.5 5.5 16 11z"/></svg>
            </div>
          </a>
        
      </nav>
    
  
  <div class="md-footer-meta md-typeset">
    <div class="md-footer-meta__inner md-grid">
      <div class="md-copyright">
  
    <div class="md-copyright__highlight">
      Copyright &copy; 2019-2024 TeraflowSDN Project
    </div>
  
  
    Made with
    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
      Material for MkDocs
    </a>
  
</div>
      
        <div class="md-social">
  
    
    <a href="https://tfs.etsi.org/" target="_blank" rel="noopener" title="tfs.etsi.org" class="md-social__link">
      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M352 256c0 22.2-1.2 43.6-3.3 64H163.4c-2.2-20.4-3.3-41.8-3.3-64s1.2-43.6 3.3-64h185.3c2.2 20.4 3.3 41.8 3.3 64m28.8-64h123.1c5.3 20.5 8.1 41.9 8.1 64s-2.8 43.5-8.1 64H380.8c2.1-20.6 3.2-42 3.2-64s-1.1-43.4-3.2-64m112.6-32H376.7c-10-63.9-29.8-117.4-55.3-151.6 78.3 20.7 142 77.5 171.9 151.6zm-149.1 0H167.7c6.1-36.4 15.5-68.6 27-94.7 10.5-23.6 22.2-40.7 33.5-51.5C239.4 3.2 248.7 0 256 0s16.6 3.2 27.8 13.8c11.3 10.8 23 27.9 33.5 51.5 11.6 26 20.9 58.2 27 94.7m-209 0H18.6c30-74.1 93.6-130.9 172-151.6-25.5 34.2-45.3 87.7-55.3 151.6M8.1 192h123.1c-2.1 20.6-3.2 42-3.2 64s1.1 43.4 3.2 64H8.1C2.8 299.5 0 278.1 0 256s2.8-43.5 8.1-64m186.6 254.6c-11.6-26-20.9-58.2-27-94.6h176.6c-6.1 36.4-15.5 68.6-27 94.6-10.5 23.6-22.2 40.7-33.5 51.5-11.2 10.7-20.5 13.9-27.8 13.9s-16.6-3.2-27.8-13.8c-11.3-10.8-23-27.9-33.5-51.5zM135.3 352c10 63.9 29.8 117.4 55.3 151.6-78.4-20.7-142-77.5-172-151.6zm358.1 0c-30 74.1-93.6 130.9-171.9 151.6 25.5-34.2 45.2-87.7 55.3-151.6h116.7z"/></svg>
    </a>
  
    
    <a href="https://labs.etsi.org/rep/tfs" target="_blank" rel="noopener" title="labs.etsi.org" class="md-social__link">
      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="m503.5 204.6-.7-1.8-69.7-181.78c-1.4-3.57-3.9-6.59-7.2-8.64-2.4-1.55-5.1-2.515-8-2.81s-5.7.083-8.4 1.11c-2.7 1.02-5.1 2.66-7.1 4.78-1.9 2.12-3.3 4.67-4.1 7.44l-47 144H160.8l-47.1-144c-.8-2.77-2.2-5.31-4.1-7.43-2-2.12-4.4-3.75-7.1-4.77a18.1 18.1 0 0 0-8.38-1.113 18.4 18.4 0 0 0-8.04 2.793 18.1 18.1 0 0 0-7.16 8.64L9.267 202.8l-.724 1.8a129.57 129.57 0 0 0-3.52 82c7.747 26.9 24.047 50.7 46.447 67.6l.27.2.59.4 105.97 79.5 52.6 39.7 32 24.2c3.7 1.9 8.3 4.3 13 4.3s9.3-2.4 13-4.3l32-24.2 52.6-39.7 106.7-79.9.3-.3c22.4-16.9 38.7-40.6 45.6-67.5 8.6-27 7.4-55.8-2.6-82"/></svg>
    </a>
  
    
    <a href="https://www.linkedin.com/company/teraflowsdn/" target="_blank" rel="noopener" title="www.linkedin.com" class="md-social__link">
      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M416 32H31.9C14.3 32 0 46.5 0 64.3v383.4C0 465.5 14.3 480 31.9 480H416c17.6 0 32-14.5 32-32.3V64.3c0-17.8-14.4-32.3-32-32.3M135.4 416H69V202.2h66.5V416zm-33.2-243c-21.3 0-38.5-17.3-38.5-38.5S80.9 96 102.2 96c21.2 0 38.5 17.3 38.5 38.5 0 21.3-17.2 38.5-38.5 38.5m282.1 243h-66.4V312c0-24.8-.5-56.7-34.5-56.7-34.6 0-39.9 27-39.9 54.9V416h-66.4V202.2h63.7v29.2h.9c8.9-16.8 30.6-34.5 62.9-34.5 67.2 0 79.7 44.3 79.7 101.9z"/></svg>
    </a>
  
    
    <a href="https://twitter.com/TeraflowSDN" target="_blank" rel="noopener" title="twitter.com" class="md-social__link">
      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M389.2 48h70.6L305.6 224.2 487 464H345L233.7 318.6 106.5 464H35.8l164.9-188.5L26.8 48h145.6l100.5 132.9zm-24.8 373.8h39.1L151.1 88h-42z"/></svg>
    </a>
  
</div>
      
    </div>
  </div>
</footer>
      
    </div>
    <div class="md-dialog" data-md-component="dialog">
      <div class="md-dialog__inner md-typeset"></div>
    </div>
    
      <div class="md-progress" data-md-component="progress" role="progressbar"></div>
    
    
    <script id="__config" type="application/json">{"base": "..", "features": ["navigation.instant", "navigation.instant.progress", "navigation.top", "navigation.footer", "navigation.path", "search", "search.highlight", "toc.integrate"], "search": "../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": {"provider": "mike"}}</script>
    
    
      <script src="../assets/javascripts/bundle.83f73b43.min.js"></script>
      
    
  </body>
</html>