From cb145c41bd62f790fb2f94769d3216a8cc6bc4aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Ara=C3=BAjo?= Date: Mon, 8 Jul 2024 12:16:27 +0100 Subject: [PATCH 1/6] Vagrant environment --- ansible/Vagrantfile | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 ansible/Vagrantfile diff --git a/ansible/Vagrantfile b/ansible/Vagrantfile new file mode 100644 index 000000000..6dd86b0ee --- /dev/null +++ b/ansible/Vagrantfile @@ -0,0 +1,42 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +Vagrant.configure("2") do |config| + config.vm.define "hub" do |hub| + hub.vm.box = "ubuntu/mantic64" + hub.vm.hostname = "hub" + hub.vm.network "private_network", ip: "192.168.56.10" + hub.vm.provider "virtualbox" do |v| + # This is high because in this scenarion, the hub will be responsible form runnin, both, TFS and Containerlab + # Other configurations between the nodes can be used, but that will require to also change the playbook + v.memory = 16384 + v.cpus = 6 + end + hub.vm.provision "file", source: "~/.ssh/id_rsa.pub", destination: "/home/vagrant/.ssh/host.pub" + hub.vm.provision "shell", inline: "cat /home/vagrant/.ssh/host.pub >> /home/vagrant/.ssh/authorized_keys" + end + + config.vm.define "spoke1" do |spoke1| + spoke1.vm.box = "ubuntu/mantic64" + spoke1.vm.hostname = "spoke1" + spoke1.vm.network "private_network", ip: "192.168.56.11" + spoke1.vm.provider "virtualbox" do |v| + v.memory = 2048 + v.cpus = 1 + end + spoke1.vm.provision "file", source: "~/.ssh/id_rsa.pub", destination: "/home/vagrant/.ssh/host.pub" + spoke1.vm.provision "shell", inline: "cat /home/vagrant/.ssh/host.pub >> /home/vagrant/.ssh/authorized_keys" + end + + config.vm.define "spoke2" do |spoke2| + spoke2.vm.box = "ubuntu/mantic64" + spoke2.vm.hostname = "spoke2" + spoke2.vm.network "private_network", ip: "192.168.56.12" + spoke2.vm.provider "virtualbox" do |v| + v.memory = 2048 + v.cpus = 1 + end + spoke2.vm.provision "file", source: "~/.ssh/id_rsa.pub", destination: "/home/vagrant/.ssh/host.pub" + spoke2.vm.provision "shell", inline: "cat /home/vagrant/.ssh/host.pub >> /home/vagrant/.ssh/authorized_keys" + end +end -- GitLab From bf0f34039a6e6a4a798a46e15bbd5124cc02fad8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Ara=C3=BAjo?= Date: Mon, 8 Jul 2024 12:16:43 +0100 Subject: [PATCH 2/6] Shell script for deploying the environment --- ansible/run_vagrant.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100755 ansible/run_vagrant.sh diff --git a/ansible/run_vagrant.sh b/ansible/run_vagrant.sh new file mode 100755 index 000000000..148259f00 --- /dev/null +++ b/ansible/run_vagrant.sh @@ -0,0 +1,34 @@ +# Copyright 2024 David Araújo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "[!] Destroying existing Vagrant machines" +vagrant destroy -f --parallel + +echo -e "\n[!] Removing destroyed hosts from known hosts" +ssh-keygen -f "/home/davidjosearaujo/.ssh/known_hosts" -R "192.168.56.10" +ssh-keygen -f "/home/davidjosearaujo/.ssh/known_hosts" -R "192.168.56.11" +ssh-keygen -f "/home/davidjosearaujo/.ssh/known_hosts" -R "192.168.56.12" + +echo -e "\n[+] Creating new hosts" +vagrant up + +AVAILABLE=1 +while [ $AVAILABLE -ne 0 ] +do + ansible -o -i inventory.yml -m ping nodes 2>&1 >/dev/null + AVAILABLE=$? +done +echo -e "\n[!] Hosts ready, deploying configurations..." + +ansible-playbook -i inventory.yml playbook.yml \ No newline at end of file -- GitLab From f641cab5ab4f777b035208bb91af7731ecf33b5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Ara=C3=BAjo?= Date: Mon, 8 Jul 2024 12:17:14 +0100 Subject: [PATCH 3/6] Ansible inventory and playbook --- ansible/inventory.yml | 31 +++++ ansible/playbook.yml | 279 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 310 insertions(+) create mode 100644 ansible/inventory.yml create mode 100644 ansible/playbook.yml diff --git a/ansible/inventory.yml b/ansible/inventory.yml new file mode 100644 index 000000000..5f9038a74 --- /dev/null +++ b/ansible/inventory.yml @@ -0,0 +1,31 @@ +--- +# Copyright 2024 David Araújo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nodes: + hosts: + hub: + ansible_host: 192.168.56.10 + containerlab_host: true # Mandatory + microk8s_master: true # Mandatory + tfs_branch: develop # Optional + spoke1: + ansible_host: 192.168.56.11 + spoke2: + ansible_host: 192.168.56.12 + vars: + ansible_user: vagrant + ansible_ssh_private_key_file: ~/.ssh/id_rsa + # Prevents from having to interact to approve a new remote host fingerprint + ansible_ssh_common_args: -o StrictHostKeyChecking=accept-new diff --git a/ansible/playbook.yml b/ansible/playbook.yml new file mode 100644 index 000000000..2c49a10bf --- /dev/null +++ b/ansible/playbook.yml @@ -0,0 +1,279 @@ +--- +# Copyright 2024 David Araújo +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: TFS Cluster platform + hosts: nodes + become: true + remote_user: "{{ ansible_user }}" + + tasks: + - name: Wait for /var/lib/dpkg/lock-frontend to be released + ansible.builtin.shell: while lsof /var/lib/dpkg/lock-frontend; do sleep 10; done + register: out + changed_when: out.rc != 0 + + - name: Configure all packages + ansible.builtin.command: dpkg --configure -a + register: out + changed_when: out.rc != 0 + + - name: Update repositories + ansible.builtin.apt: + update_cache: true + + - name: Fix any broken packages + ansible.builtin.apt: + name: "*" + state: fixed + + - name: Upgrade the OS (apt-get dist-upgrade) + ansible.builtin.apt: + upgrade: dist + + - name: Install dependencies + ansible.builtin.apt: + pkg: + - ca-certificates + - curl + - gnupg + - lsb-release + - snapd + - jq + - docker.io + - docker-buildx + state: fixed + update_cache: true + + - name: Update Docker daemon.json with insecure-registries + ansible.builtin.shell: | + if [ -s /etc/docker/daemon.json ]; then sudo cat /etc/docker/daemon.json; else echo '{}'; fi \ + | jq 'if has("insecure-registries") then . else .+ {"insecure-registries": []} end' -- \ + | jq '."insecure-registries" |= (.+ ["localhost:32000"] | unique)' -- \ + | tee tmp.daemon.json + register: out + changed_when: out.rc != 0 + + - name: Copy tmp.daemon.json + ansible.builtin.copy: + remote_src: true + src: tmp.daemon.json + dest: /etc/docker/daemon.json + owner: root + group: root + mode: "600" + + - name: Remove workdir tmp.daemon.json file + ansible.builtin.file: + path: tmp.daemon.json + state: absent + + - name: Restart docker service + ansible.builtin.service: + name: docker + state: restarted + + - name: Add address of all hosts to all hosts + ansible.builtin.lineinfile: + dest: /etc/hosts + regexp: .*{{ item }}$ + line: "{{ hostvars[item].ansible_host }} {{ item }}" + state: present + when: hostvars[item].ansible_host is defined + with_items: "{{ groups.all }}" + + - name: Find the Containerlab host IP + ansible.builtin.set_fact: + containerlab_host_address: "{{ hostvars[item].ansible_host }}" + when: hostvars[item].containerlab_host is defined + with_items: "{{ groups['all'] }}" + run_once: true + + - name: Specify ip route to Containerlab network + ansible.builtin.shell: sudo ip route add 172.100.100.0/24 via {{ containerlab_host_address }} + when: hostvars[inventory_hostname].containerlab_host is not defined + register: out + changed_when: out.rc != 0 + + - name: Install Containerlab + ansible.builtin.shell: | + curl -sL https://containerlab.dev/setup \ + | sudo bash -s "all" + register: out + changed_when: out.rc != 0 + when: containerlab_host is defined + + - name: Install gnmic + ansible.builtin.shell: | + curl -sL https://get-gnmic.kmrd.dev \ + | sudo bash + register: out + changed_when: out.rc != 0 + + - name: Install MicroK8s + community.general.snap: + name: microk8s + classic: true + channel: 1.24/stable + + - name: Create kubectl alias + community.general.snap_alias: + name: microk8s.kubectl + alias: kubectl + + - name: Ensure group "microk8s" exists + ansible.builtin.group: + name: microk8s + state: present + + - name: Ensure group "docker" exists + ansible.builtin.group: + name: docker + state: present + + - name: Add user to docker and microk8s groups + ansible.builtin.user: + name: "{{ ansible_user }}" + groups: docker, microk8s + + - name: Create .kube directory + ansible.builtin.file: + path: /home/{{ ansible_user }}/.kube + state: directory + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: "0644" + + - name: Create .kube/config file + ansible.builtin.file: + path: /home/{{ ansible_user }}/.kube/config + state: touch + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: "0644" + + - name: Get microk8s config + ansible.builtin.command: sudo microk8s config + register: microk8s_config + changed_when: microk8s_config.rc != 0 + + - name: Keep microk8s config + ansible.builtin.copy: + remote_src: true + content: "{{ microk8s_config.stdout }}" + dest: /home/{{ ansible_user }}/.kube/config + mode: "0644" + register: out + + - name: Add IP address to microk8s certificate template + ansible.builtin.lineinfile: + dest: /var/snap/microk8s/current/certs/csr.conf.template + search_string: "#MOREIPS" + line: IP.3 = {{ ansible_host }} + state: present + + - name: Start MicroK8s + ansible.builtin.command: microk8s start + register: out + changed_when: out.rc != 0 + +- name: Forming MicroK8s cluster + hosts: all + become: true + remote_user: "{{ ansible_user }}" + serial: 1 + + tasks: + - name: MicroK8s add-node + ansible.builtin.shell: | + microk8s add-node \ + | grep -m 1 "{{ hostvars[item].ansible_host }}" + register: master_output + delegate_to: "{{ item }}" + with_items: "{{ groups.all }}" + when: (microk8s_master is not defined) and (hostvars[item].microk8s_master is defined) + + - name: Join command + ansible.builtin.set_fact: + join_command: "{{ (dict(master_output).results | selectattr('changed', 'true') | first).stdout }} --skip-verify" + when: microk8s_master is not defined + + - name: MicroK8s join-node + ansible.builtin.command: "{{ join_command }}" + register: out + changed_when: out.rc != 0 + when: microk8s_master is not defined + +- name: Enabling MicroK8s + hosts: all + become: true + remote_user: "{{ ansible_user }}" + + tasks: + - name: MicroK8s refresh certificate + ansible.builtin.command: microk8s refresh-certs -e ca.crt + register: out + changed_when: out.rc != 0 + when: microk8s_master is not defined + + - name: MicroK8s refresh config file + ansible.builtin.command: microk8s config > /home/$USER/.kube/config + register: out + changed_when: out.rc != 0 + when: microk8s_master is not defined + + - name: Enable add-ons + ansible.builtin.command: microk8s.enable {{ item }} + loop: + - community + - dns + - helm3 + - hostpath-storage + - ingress + - registry + - prometheus + - metrics-server + - linkerd + register: out + changed_when: out.rc != 0 + when: microk8s_master is defined + + - name: Create kubectl helm3 alias + community.general.snap_alias: + name: microk8s.helm3 + alias: helm3 + when: microk8s_master is defined + + - name: Create kubectl linkerd alias + community.general.snap_alias: + name: microk8s.linkerd + alias: linkerd + when: microk8s_master is defined + + - name: Ensure MicroK8s started + ansible.builtin.command: microk8s start + register: out + changed_when: out.rc != 0 + +- name: Cloning TFS + hosts: all + remote_user: "{{ ansible_user }}" + + tasks: + - name: Clone repository and switch to desired branch + ansible.builtin.git: + repo: 'https://labs.etsi.org/rep/tfs/controller.git' + dest: ./tfs-ctrl + version: "{{ tfs_branch | default('master') }}" + when: microk8s_master is defined -- GitLab From 0b094d757c24930937ae5c9a99d55be3ce2d7a67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Ara=C3=BAjo?= Date: Mon, 8 Jul 2024 14:12:00 +0100 Subject: [PATCH 4/6] README instructions and explanation --- ansible/README.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 ansible/README.md diff --git a/ansible/README.md b/ansible/README.md new file mode 100644 index 000000000..a130c37fe --- /dev/null +++ b/ansible/README.md @@ -0,0 +1,35 @@ + + +# TFS Cluster Auto-setup + +The Ansible playbook and inventory in this directory are set up to configure Ubuntu 22.04 hosts, preparing them for a future TeraFlowSDN deployment. + +## Host Environment + +As a sandbox example, we create a network of three hosts using Vagrant and provision them with password-less SSH public keys. + +In a production environment, various tools or platforms can be used to run these instances, such as a server with Proxmox or a cloud provider. Provisioning can be done manually or with tools like Terraform. + +## Inventory + +In the inventory, we list the hosts that will be part of the MicroK8s cluster and define their roles. + +As of this writing, only three tags are required, with one being optional. + +- **microk8s_master** - *boolean* - *mandatory* - Identifies the node that will serve as the master of the MicroK8s cluster. +- **tfs_branch** - *string* - *optional* - Specifies the branch to checkout when cloning the repository (defaults to `master`). This tag must be defined on the same host as the *microk8s_master* tag. +- **containerlab_host** - *boolean* - *mandatory* - Identifies the node where ContainerLab will be deployed. This tag enables Ansible to install ContainerLab on the correct host and set static routes on other hosts to reach nodes in the ContainerLab topology. \ No newline at end of file -- GitLab From 6e53c2d0b013dec5d776e955ee3fdc8aad4bfc3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Ara=C3=BAjo?= Date: Mon, 8 Jul 2024 14:12:10 +0100 Subject: [PATCH 5/6] Change VM requirements --- ansible/Vagrantfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/Vagrantfile b/ansible/Vagrantfile index 6dd86b0ee..9a693089b 100644 --- a/ansible/Vagrantfile +++ b/ansible/Vagrantfile @@ -7,10 +7,10 @@ Vagrant.configure("2") do |config| hub.vm.hostname = "hub" hub.vm.network "private_network", ip: "192.168.56.10" hub.vm.provider "virtualbox" do |v| - # This is high because in this scenarion, the hub will be responsible form runnin, both, TFS and Containerlab + # This is high because in this scenarion, the hub will be responsible form running, both TFS and Containerlab # Other configurations between the nodes can be used, but that will require to also change the playbook - v.memory = 16384 - v.cpus = 6 + v.memory = 8192 + v.cpus = 4 end hub.vm.provision "file", source: "~/.ssh/id_rsa.pub", destination: "/home/vagrant/.ssh/host.pub" hub.vm.provision "shell", inline: "cat /home/vagrant/.ssh/host.pub >> /home/vagrant/.ssh/authorized_keys" -- GitLab From c3a7a7e51d3402328c830e7eee2949b0331b4e59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Ara=C3=BAjo?= Date: Mon, 8 Jul 2024 15:28:04 +0100 Subject: [PATCH 6/6] Ensuring docker daemon doesn't stall --- ansible/playbook.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 2c49a10bf..c8c4b1c68 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -147,6 +147,15 @@ name: "{{ ansible_user }}" groups: docker, microk8s + - name: Reboot for groups to take effect + ansible.builtin.reboot: + + - name: Make sure docker daemon is running + ansible.builtin.systemd_service: + state: started + name: docker + enabled: true + - name: Create .kube directory ansible.builtin.file: path: /home/{{ ansible_user }}/.kube -- GitLab