From: Ruoyu Date: Mon, 18 May 2020 12:37:52 +0000 (+0800) Subject: Add e2e test scripts for sdewan X-Git-Tag: v1.0~31^2 X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F80%2F3480%2F10;p=icn%2Fsdwan.git Add e2e test scripts for sdewan * Add three vagrant vms to setup env Three vms are created for edge-a, edge-b and sdewan-hub. Each with a separate cluster. * Add test scripts for the e2e IPSec scenario Establish tunnels between edge and sdewan-hub and test the connections between two applications reside in edge-a and edge-b. Issue-ID: ICN-314 Change-Id: I0cb8d9d251f0f1cd8ad4c5d58b60e99809c02d0b Signed-off-by: Ruoyu --- diff --git a/platform/test/e2e-test/README.md b/platform/test/e2e-test/README.md new file mode 100644 index 0000000..57f17b8 --- /dev/null +++ b/platform/test/e2e-test/README.md @@ -0,0 +1,30 @@ +#SD-eWAN test scenario +[Overview] +In this test scenario, three clusters are created for edge-a, edge-b and sdewan-hub. +Two tunnels are established between the edge and the hub, and also two applications +are installed in edga-a and edge-b. Tunnels are verified thru the connection test between +the two applications. + +[Test guide] +Run the test.sh under sdwan/platform/test/e2e-test/ to invoke the vm creation and configurations. + $ ./test.sh + +Scripts description: +1. The Vagrantfile will be used to setup the base environment. +2. The installer.sh script contains the minimal Ubuntu instructions required for bringing up ICN. +3. The setup-cnf.sh script creates ovn networks, sdewan cnfs and application pods if needed. +4. The sdwan_verifier.sh script inserts configs into the sdewan cnf, including firewall and ipsec. +5. The test-connection.sh script under edge-a tests the connection between the applications +reside in edge-a and edge-b. + + +[License] + +Apache-2.0 + +[1]: https://gerrit.akraino.org/r/icn/sdwan + +[2]: https://git.onap.org/multicloud/k8s + +[3]: https://www.vagrantup.com/ + diff --git a/platform/test/e2e-test/edge-a/Vagrantfile b/platform/test/e2e-test/edge-a/Vagrantfile new file mode 100644 index 0000000..4fa25cb --- /dev/null +++ b/platform/test/e2e-test/edge-a/Vagrantfile @@ -0,0 +1,150 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +box = { + :virtualbox => { :name => 'elastic/ubuntu-18.04-x86_64', :version => '20191013.0.0'}, + :libvirt => { :name => 'intergratedcloudnative/ubuntu1804', :version => '1.0.0'} +} + +require 'yaml' +pdf = File.dirname(__FILE__) + '/config/default.yml' +if File.exist?(File.dirname(__FILE__) + '/config/pdf.yml') + pdf = File.dirname(__FILE__) + '/config/pdf.yml' +end +nodes = YAML.load_file(pdf) + +# Inventory file creation +File.open(File.dirname(__FILE__) + "/inventory/hosts.ini", "w") do |inventory_file| + inventory_file.puts("[all]") + nodes.each do |node| + inventory_file.puts("#{node['name']}\tansible_ssh_host=#{node['ip']} ansible_ssh_port=22") + end + ['kube-master', 'kube-node', 'etcd', 'ovn-central', 'ovn-controller', 'virtlet', 'cmk'].each do|group| + inventory_file.puts("\n[#{group}]") + nodes.each do |node| + if node['roles'].include?("#{group}") + inventory_file.puts(node['name']) + end + end + end + inventory_file.puts("\n[k8s-cluster:children]\nkube-node\nkube-master") +end + +provider = (ENV['VAGRANT_DEFAULT_PROVIDER'] || :libvirt).to_sym +puts "[INFO] Provider: #{provider} " + +if ENV['no_proxy'] != nil or ENV['NO_PROXY'] + $no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || "127.0.0.1,localhost" + nodes.each do |node| + $no_proxy += "," + node['ip'] + end + $subnet = "192.168.121" + if provider == :virtualbox + $subnet = "10.0.2" + end + # NOTE: This range is based on vagrant-libvirt network definition CIDR 192.168.121.0/27 + (1..31).each do |i| + $no_proxy += ",#{$subnet}.#{i}" + end +end + +Vagrant.configure("2") do |config| + config.vm.box = box[provider][:name] + config.vm.box_version = box[provider][:version] + config.ssh.insert_key = false + + if ENV['http_proxy'] != nil and ENV['https_proxy'] != nil + if Vagrant.has_plugin?('vagrant-proxyconf') + config.proxy.http = ENV['http_proxy'] || ENV['HTTP_PROXY'] || "" + config.proxy.https = ENV['https_proxy'] || ENV['HTTPS_PROXY'] || "" + config.proxy.no_proxy = $no_proxy + config.proxy.enabled = { docker: false } + end + end + config.vm.provider 'libvirt' do |v| + v.nested = true + v.cpu_mode = 'host-passthrough' + v.management_network_address = "192.168.121.0/27" + v.random_hostname = true + end + + sync_type = "virtualbox" + if provider == :libvirt + sync_type = "nfs" + end + + nodes.each do |node| + config.vm.define node['name'] do |nodeconfig| + if node['roles'].include?("kube-master") + nodeconfig.vm.synced_folder '../', '/home/vagrant/multicloud-k8s/', type: sync_type + end + if node['roles'].include?("kube-node") + nodeconfig.vm.provision 'shell', privileged: false do |sh| + sh.inline = <<-SHELL + sudo sed -i 's:GRUB_CMDLINE_LINUX=.*:GRUB_CMDLINE_LINUX="isolcpus=0-7":' /etc/default/grub + sudo update-grub + SHELL + end + end + nodeconfig.vm.hostname = node['name'] + nodeconfig.vm.network :private_network, :ip => node['ip'], :type => :static + nodeconfig.vm.provider 'virtualbox' do |v| + v.customize ["modifyvm", :id, "--memory", node['memory']] + v.customize ["modifyvm", :id, "--cpus", node['cpus']] + if node.has_key? "volumes" + node['volumes'].each do |volume| + $volume_file = "#{node['name']}-#{volume['name']}.vdi" + unless File.exist?($volume_file) + v.customize ['createmedium', 'disk', '--filename', $volume_file, '--size', volume['size']] + end + v.customize ['storageattach', :id, '--storagectl', 'IDE Controller', '--port', 1, '--device', 0, '--type', 'hdd', '--medium', $volume_file] + end + end + end + nodeconfig.vm.provider 'libvirt' do |v| + v.memory = node['memory'] + v.cpus = node['cpus'] + nodeconfig.vm.provision 'file', source: './scripts', destination: '$HOME/' + nodeconfig.vm.provision 'file', source: '../edge-scripts/setup-cnf.sh', destination: '$HOME/scripts/' + nodeconfig.vm.provision 'file', source: '../edge-scripts/sdwan_verifier.sh', destination: '$HOME/scripts/' + nodeconfig.vm.provision 'shell' do |sh| + sh.path = "node.sh" + if node.has_key? "volumes" + $volume_mounts_dict = '' + node['volumes'].each do |volume| + $volume_mounts_dict += "#{volume['name']}=#{volume['mount']}," + $volume_file = "./#{node['name']}-#{volume['name']}.qcow2" + v.storage :file, :bus => 'sata', :device => volume['name'], :size => volume['size'] + end + sh.args = ['-v', $volume_mounts_dict[0...-1]] + end + end + end + end + end + + config.vm.define :installer, primary: true, autostart: false do |installer| + installer.vm.hostname = "multicloud" + installer.vm.network :private_network, :ip => "10.10.10.2", :type => :static + installer.vm.synced_folder '../', '/home/vagrant/multicloud-k8s/', type: sync_type + installer.vm.provision 'shell', privileged: false do |sh| + sh.env = {'KUD_PLUGIN_ENABLED': 'false', 'OVN_CENTRAL_INTERFACE': 'eth1'} + sh.inline = <<-SHELL + cp /home/vagrant/multicloud-k8s/edge-a/insecure_keys/key.pub /home/vagrant/.ssh/id_rsa.pub + cp /home/vagrant/multicloud-k8s/edge-a/insecure_keys/key /home/vagrant/.ssh/id_rsa + chown vagrant /home/vagrant/.ssh/id_rsa + chmod 400 /home/vagrant/.ssh/id_rsa + sudo apt install jq -y + cd /home/vagrant/multicloud-k8s/edge-a/ && ./installer.sh | tee kud_installer.log + SHELL + end + end +end diff --git a/platform/test/e2e-test/edge-a/config/default.yml b/platform/test/e2e-test/edge-a/config/default.yml new file mode 100644 index 0000000..6f29c68 --- /dev/null +++ b/platform/test/e2e-test/edge-a/config/default.yml @@ -0,0 +1,24 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- name: "edge-a" + ip: "10.10.10.3" + memory: 18432 + cpus: 16 + volumes: + - name: sdb + size: 50 + mount: /var/lib/docker/ + roles: + - kube-master + - etcd + - ovn-central + - kube-node + - ovn-controller diff --git a/platform/test/e2e-test/edge-a/insecure_keys/key b/platform/test/e2e-test/edge-a/insecure_keys/key new file mode 100644 index 0000000..7d6a083 --- /dev/null +++ b/platform/test/e2e-test/edge-a/insecure_keys/key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI +w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP +kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2 +hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO +Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW +yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd +ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1 +Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf +TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK +iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A +sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf +4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP +cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk +EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN +CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX +3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG +YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj +3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+ +dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz +6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC +P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF +llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ +kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH ++vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ +NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s= +-----END RSA PRIVATE KEY----- diff --git a/platform/test/e2e-test/edge-a/insecure_keys/key.pub b/platform/test/e2e-test/edge-a/insecure_keys/key.pub new file mode 100644 index 0000000..18a9c00 --- /dev/null +++ b/platform/test/e2e-test/edge-a/insecure_keys/key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key diff --git a/platform/test/e2e-test/edge-a/installer.sh b/platform/test/e2e-test/edge-a/installer.sh new file mode 100755 index 0000000..fd6349f --- /dev/null +++ b/platform/test/e2e-test/edge-a/installer.sh @@ -0,0 +1,233 @@ +#!/bin/bash +#SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o errexit +set -o nounset +set -o pipefail + +INSTALLER_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" + +source ${INSTALLER_DIR}/../k8s/kud/tests/_functions.sh + +# _install_go() - Install GoLang package +function _install_go { + version=$(grep "go_version" ${kud_playbooks}/kud-vars.yml | awk -F "'" '{print $2}') + local tarball=go$version.linux-amd64.tar.gz + + #gcc is required for go apps compilation + if ! which gcc; then + sudo apt-get install -y gcc + fi + + if $(go version &>/dev/null); then + return + fi + + wget https://dl.google.com/go/$tarball + sudo tar -C /usr/local -xzf $tarball + rm $tarball + + export PATH=$PATH:/usr/local/go/bin + sudo sed -i "s|^PATH=.*|PATH=\"$PATH\"|" /etc/environment +} + +# _install_pip() - Install Python Package Manager +function _install_pip { + if $(pip --version &>/dev/null); then + sudo -E pip install --upgrade pip + else + sudo apt-get install -y python-dev + curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python + fi +} + +# _install_ansible() - Install and Configure Ansible program +function _install_ansible { + if $(ansible --version &>/dev/null); then + sudo pip uninstall -y ansible + fi + _install_pip + local version=$(grep "ansible_version" ${kud_playbooks}/kud-vars.yml | awk -F ': ' '{print $2}') + sudo mkdir -p /etc/ansible/ + sudo -E pip install ansible==$version +} + +# _install_docker() - Download and install docker-engine +function _install_docker { + local max_concurrent_downloads=${1:-3} + + if $(docker version &>/dev/null); then + return + fi + sudo apt-get install -y apt-transport-https ca-certificates curl + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + sudo apt-get update + sudo apt-get install -y docker-ce + + sudo mkdir -p /etc/systemd/system/docker.service.d + if [ ${http_proxy:-} ]; then + echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/http-proxy.conf + echo "Environment=\"HTTP_PROXY=$http_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/http-proxy.conf + fi + if [ ${https_proxy:-} ]; then + echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/https-proxy.conf + echo "Environment=\"HTTPS_PROXY=$https_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/https-proxy.conf + fi + if [ ${no_proxy:-} ]; then + echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/no-proxy.conf + echo "Environment=\"NO_PROXY=$no_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/no-proxy.conf + fi + sudo systemctl daemon-reload + echo "DOCKER_OPTS=\"-H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock --max-concurrent-downloads $max_concurrent_downloads \"" | sudo tee --append /etc/default/docker + if [[ -z $(groups | grep docker) ]]; then + sudo usermod -aG docker $USER + fi + + sudo systemctl restart docker + sleep 10 +} + +function _set_environment_file { + # By default ovn central interface is the first active network interface on localhost. If other wanted, need to export this variable in aio.sh or Vagrant file. + OVN_CENTRAL_INTERFACE="${OVN_CENTRAL_INTERFACE:-$(ip addr show | awk '/inet.*brd/{print $NF; exit}')}" + echo "export OVN_CENTRAL_INTERFACE=${OVN_CENTRAL_INTERFACE}" | sudo tee --append /etc/environment + echo "export OVN_CENTRAL_ADDRESS=$(get_ovn_central_address)" | sudo tee --append /etc/environment + echo "export KUBE_CONFIG_DIR=/opt/kubeconfig" | sudo tee --append /etc/environment + echo "export CSAR_DIR=/opt/csar" | sudo tee --append /etc/environment +} + +# install_k8s() - Install Kubernetes using kubespray tool +function install_k8s { + echo "Deploying kubernetes" + local dest_folder=/opt + version=$(grep "kubespray_version" ${kud_playbooks}/kud-vars.yml | awk -F ': ' '{print $2}') + local_release_dir=$(grep "local_release_dir" $kud_inventory_folder/group_vars/k8s-cluster.yml | awk -F "\"" '{print $2}') + local tarball=v$version.tar.gz + sudo apt-get install -y sshpass make unzip # install make to run mitogen target and unzip is mitogen playbook dependency + sudo apt-get install -y gnupg2 software-properties-common + _install_docker + _install_ansible + wget https://github.com/kubernetes-incubator/kubespray/archive/$tarball + sudo tar -C $dest_folder -xzf $tarball + sudo mv $dest_folder/kubespray-$version/ansible.cfg /etc/ansible/ansible.cfg + sudo chown -R $USER $dest_folder/kubespray-$version + sudo mkdir -p ${local_release_dir}/containers + rm $tarball + + pushd $dest_folder/kubespray-$version/ + sudo -E pip install -r ./requirements.txt + make mitogen + popd + rm -f $kud_inventory_folder/group_vars/all.yml 2> /dev/null + if [[ -n "${verbose:-}" ]]; then + echo "kube_log_level: 5" | tee $kud_inventory_folder/group_vars/all.yml + else + echo "kube_log_level: 2" | tee $kud_inventory_folder/group_vars/all.yml + fi + echo "kubeadm_enabled: true" | tee --append $kud_inventory_folder/group_vars/all.yml + if [[ -n "${http_proxy:-}" ]]; then + echo "http_proxy: \"$http_proxy\"" | tee --append $kud_inventory_folder/group_vars/all.yml + fi + if [[ -n "${https_proxy:-}" ]]; then + echo "https_proxy: \"$https_proxy\"" | tee --append $kud_inventory_folder/group_vars/all.yml + fi + ansible-playbook $verbose -i $kud_inventory $dest_folder/kubespray-$version/cluster.yml --become --become-user=root | sudo tee $log_folder/setup-kubernetes.log + + # Configure environment + mkdir -p $HOME/.kube + cp $kud_inventory_folder/artifacts/admin.conf $HOME/.kube/config + # Copy Kubespray kubectl to be usable in host running Ansible. Requires kubectl_localhost: true in inventory/group_vars/k8s-cluster.yml + sudo cp $kud_inventory_folder/artifacts/kubectl /usr/local/bin/ +} + +# install_addons() - Install Kubenertes AddOns +function install_addons { + source /etc/environment + echo "Installing Kubernetes AddOns" + _install_ansible + sudo ansible-galaxy install $verbose -r $kud_infra_folder/galaxy-requirements.yml --ignore-errors + ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-kud.yml | sudo tee $log_folder/setup-kud.log + for addon in ${KUD_ADDONS:-ovn4nfv}; do + echo "Deploying $addon using configure-$addon.yml playbook.." + ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-${addon}.yml | sudo tee $log_folder/setup-${addon}.log + done + echo "Run the test cases if testing_enabled is set to true." + if [[ "${testing_enabled}" == "true" ]]; then + for addon in ${KUD_ADDONS:-ovn4nfv}; do + pushd $kud_tests + bash ${addon}.sh + popd + done + fi + echo "Add-ons deployment complete..." +} + +# _print_kubernetes_info() - Prints the login Kubernetes information +function _print_kubernetes_info { + if ! $(kubectl version &>/dev/null); then + return + fi + # Expose Dashboard using NodePort + node_port=30080 + KUBE_EDITOR="sed -i \"s|type\: ClusterIP|type\: NodePort|g\"" kubectl -n kube-system edit service kubernetes-dashboard + KUBE_EDITOR="sed -i \"s|nodePort\: .*|nodePort\: $node_port|g\"" kubectl -n kube-system edit service kubernetes-dashboard + + master_ip=$(kubectl cluster-info | grep "Kubernetes master" | awk -F ":" '{print $2}') + + printf "Kubernetes Info\n===============\n" > $k8s_info_file + echo "Dashboard URL: https:$master_ip:$node_port" >> $k8s_info_file + echo "Admin user: kube" >> $k8s_info_file + echo "Admin password: secret" >> $k8s_info_file +} + +sudo -k # forgot sudo password +if ! sudo -n "true"; then + echo "" + echo "passwordless sudo is needed for '$(id -nu)' user." + echo "Please fix your /etc/sudoers file. You likely want an" + echo "entry like the following one..." + echo "" + echo "$(id -nu) ALL=(ALL) NOPASSWD: ALL" + exit 1 +fi + +verbose="" +if [[ -n "${KUD_DEBUG:-}" ]]; then + set -o xtrace + verbose="-vvv" +fi + +# Configuration values +log_folder=/var/log/kud +kud_folder=${INSTALLER_DIR}/../k8s/kud/hosting_providers/vagrant +kud_infra_folder=$kud_folder/../../deployment_infra +export kud_inventory_folder=$kud_folder/../../../../edge-a/inventory +kud_inventory=$kud_inventory_folder/hosts.ini +kud_playbooks=$kud_infra_folder/playbooks +kud_tests=$kud_folder/../../tests +k8s_info_file=$kud_folder/k8s_info.log +testing_enabled=${KUD_ENABLE_TESTS:-false} +sudo mkdir -p $log_folder +sudo mkdir -p /opt/csar +sudo chown -R $USER /opt/csar +# Install dependencies +# Setup proxy variables +if [ -f $kud_folder/sources.list ]; then + sudo mv /etc/apt/sources.list /etc/apt/sources.list.backup + sudo cp $kud_folder/sources.list /etc/apt/sources.list +fi +echo "Removing ppa for jonathonf/python-3.6" +sudo ls /etc/apt/sources.list.d/ || true +sudo find /etc/apt/sources.list.d -maxdepth 1 -name '*jonathonf*' -delete || true +sudo apt-get update +install_k8s +_set_environment_file +install_addons diff --git a/platform/test/e2e-test/edge-a/inventory/group_vars/all.yml b/platform/test/e2e-test/edge-a/inventory/group_vars/all.yml new file mode 100644 index 0000000..528430c --- /dev/null +++ b/platform/test/e2e-test/edge-a/inventory/group_vars/all.yml @@ -0,0 +1,2 @@ +kube_log_level: 2 +kubeadm_enabled: true diff --git a/platform/test/e2e-test/edge-a/inventory/group_vars/k8s-cluster.yml b/platform/test/e2e-test/edge-a/inventory/group_vars/k8s-cluster.yml new file mode 100644 index 0000000..ab91498 --- /dev/null +++ b/platform/test/e2e-test/edge-a/inventory/group_vars/k8s-cluster.yml @@ -0,0 +1,82 @@ +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Kubernetes configuration dirs and system namespace. +# Those are where all the additional config stuff goes +# kubernetes normally puts in /srv/kubernetes. +# This puts them in a sane location and namespace. +# Editing those values will almost surely break something. +system_namespace: kube-system + +# Logging directory (sysvinit systems) +kube_log_dir: "/var/log/kubernetes" + +kube_api_anonymous_auth: true + +# Users to create for basic auth in Kubernetes API via HTTP +# Optionally add groups for user +kube_api_pwd: "secret" +kube_users: + kube: + pass: "{{kube_api_pwd}}" + role: admin + groups: + - system:masters + +## It is possible to activate / deactivate selected authentication methods (basic auth, static token auth) +#kube_oidc_auth: false +kube_basic_auth: true +kube_token_auth: true + +# Choose network plugin (calico, contiv, weave or flannel) +# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing +kube_network_plugin: flannel + +# Make a copy of kubeconfig (admin.conf) on the host that runs Ansible to inventory/artifacts +kubeconfig_localhost: true +# Copy kubectl binary on the host that runs Ansible to inventory/artifacts +kubectl_localhost: true +# Disable nodelocal dns cache +enable_nodelocaldns: false +# Enable MountPropagation gate feature +local_volumes_enabled: true +local_volume_provisioner_enabled: true + +## Change this to use another Kubernetes version, e.g. a current beta release +kube_version: v1.14.3 + +# Helm deployment +helm_enabled: true + +# Kube-proxy proxyMode configuration. +# NOTE: Ipvs is based on netfilter hook function, but uses hash table as the underlying data structure and +# works in the kernel space +# https://kubernetes.io/docs/concepts/services-networking/service/#proxy-mode-ipvs +#kube_proxy_mode: ipvs + +# Download container images only once then push to cluster nodes in batches +download_run_once: true + +# Where the binaries will be downloaded. +# Note: ensure that you've enough disk space (about 1G) +local_release_dir: "/tmp/releases" + +# Makes the installer node a delegate for pushing images while running +# the deployment with ansible. This maybe the case if cluster nodes +# cannot access each over via ssh or you want to use local docker +# images as a cache for multiple clusters. +download_localhost: true + +# Subnet for cluster IPs +kube_service_addresses: 10.244.0.0/18 +# Subnet for Pod IPs +kube_pods_subnet: 10.244.64.0/18 + +# Add specific docker version +docker_version: 'latest' diff --git a/platform/test/e2e-test/edge-a/node.sh b/platform/test/e2e-test/edge-a/node.sh new file mode 100755 index 0000000..a51be19 --- /dev/null +++ b/platform/test/e2e-test/edge-a/node.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o nounset +set -o pipefail + +# usage() - Prints the usage of the program +function usage { + cat <> /etc/fstab +} + +while getopts "h?v:" opt; do + case $opt in + v) + dict_volumes="$OPTARG" + ;; + h|\?) + usage + exit + ;; + esac +done + +swapoff -a +if [[ -n "${dict_volumes+x}" ]]; then + for kv in ${dict_volumes//,/ } ;do + mount_external_partition ${kv%=*} ${kv#*=} + done +fi + +vendor_id=$(lscpu|grep "Vendor ID") +if [[ $vendor_id == *GenuineIntel* ]]; then + kvm_ok=$(cat /sys/module/kvm_intel/parameters/nested) + if [[ $kvm_ok == 'N' ]]; then + echo "Enable Intel Nested-Virtualization" + rmmod kvm-intel + echo 'options kvm-intel nested=y' >> /etc/modprobe.d/dist.conf + modprobe kvm-intel + echo kvm-intel >> /etc/modules + fi +else + kvm_ok=$(cat /sys/module/kvm_amd/parameters/nested) + if [[ $kvm_ok == '0' ]]; then + echo "Enable AMD Nested-Virtualization" + rmmod kvm-amd + sh -c "echo 'options kvm-amd nested=1' >> /etc/modprobe.d/dist.conf" + modprobe kvm-amd + echo kvm-amd >> /etc/modules + fi +fi +modprobe vhost_net +echo vhost_net >> /etc/modules +source /etc/os-release || source /usr/lib/os-release +case ${ID,,} in + *suse) + ;; + ubuntu|debian) + apt-get install -y cpu-checker + kvm-ok + ;; + rhel|centos|fedora) + ;; +esac diff --git a/platform/test/e2e-test/edge-a/scripts/edge-a-sdwan-config b/platform/test/e2e-test/edge-a/scripts/edge-a-sdwan-config new file mode 100644 index 0000000..1d51a76 --- /dev/null +++ b/platform/test/e2e-test/edge-a/scripts/edge-a-sdwan-config @@ -0,0 +1,13 @@ +internal_subnet=192.168.1.0/24 +redirect=" +{ + \"name\":\"SNAT\", + \"src_ip\":\"$ovnSubnet\", + \"src_dip\": \"$sdwan_pod_virtual_ip\", + \"dest\":\"wan0\", + \"dest_ip\":\"$internal_subnet\", + \"proto\":\"tcp\", + \"target\":\"SNAT\" +} +" + diff --git a/platform/test/e2e-test/edge-a/scripts/test-connection.sh b/platform/test/e2e-test/edge-a/scripts/test-connection.sh new file mode 100755 index 0000000..df8b98c --- /dev/null +++ b/platform/test/e2e-test/edge-a/scripts/test-connection.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o errexit +set -o nounset +set -o pipefail + +test -f /home/vagrant/scripts/variables && . /home/vagrant/scripts/variables + +sdewan_virtual_ip=$(kubectl exec -it $sdewan_cnf_name ip address | grep $wan_interface | awk '/inet/{print $2}' | cut -f1 -d "/" | grep 192.168) +app_pod=$(kubectl get po | grep $app_pod_name | cut -f1 -d " ") +echo "Logging into the dummy pod $app_pod in edgeA..." + + +echo "Determine the ip address of remote host" +if [ "$sdewan_virtual_ip" == "192.168.1.5" ] +then + remote_ip="192.168.1.6" +else + remote_ip="192.168.1.5" +fi +echo "The remote ip is ${remote_ip}" + +echo "Sending request to the remote httpbin..." +kubectl exec -it $app_pod -- curl -X GET "http://$remote_ip/ip" -H "accept: application/json" >> response.json +cat response.json + +echo "Confirming the testing result..." +sudo apt install -y jq +rs=$(jq -r '.origin' response.json) +if [ "$rs" == "$sdewan_virtual_ip" ] +then + echo "End-to-end test passed" +else + echo "End-to-end test failed. Please check the logs for more details" +fi diff --git a/platform/test/e2e-test/edge-a/scripts/variables b/platform/test/e2e-test/edge-a/scripts/variables new file mode 100644 index 0000000..d01d850 --- /dev/null +++ b/platform/test/e2e-test/edge-a/scripts/variables @@ -0,0 +1,18 @@ +providerSubnet=10.10.10.1/24 +providerGateway=10.10.10.1/24 +providerExcludeIps=10.10.10.2..10.10.10.9 +providerNetworkInterface=eth1 +ovnGateway=172.16.30.1/24 +ovnExcludeIps=172.16.30.2..172.16.30.9 +appLan0=172.16.30.15 +cnfWanGateway=10.10.10.1 + +export cnfWan0=10.10.10.15 +export cnfLan0=172.16.30.10 +export hubIp=10.10.10.35 +export ovnSubnet=172.16.30.1/24 +export edgeName=edgeA +export wan_interface=net1 +export sdewan_cnf_name=sdewan-edge-a +export app_pod_name=simple-http-service +export sdwanConfigs=edge-a-sdwan-config diff --git a/platform/test/e2e-test/edge-b/Vagrantfile b/platform/test/e2e-test/edge-b/Vagrantfile new file mode 100644 index 0000000..27b334e --- /dev/null +++ b/platform/test/e2e-test/edge-b/Vagrantfile @@ -0,0 +1,150 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +box = { + :virtualbox => { :name => 'elastic/ubuntu-18.04-x86_64', :version => '20191013.0.0'}, + :libvirt => { :name => 'intergratedcloudnative/ubuntu1804', :version => '1.0.0'} +} + +require 'yaml' +pdf = File.dirname(__FILE__) + '/config/default.yml' +if File.exist?(File.dirname(__FILE__) + '/config/pdf.yml') + pdf = File.dirname(__FILE__) + '/config/pdf.yml' +end +nodes = YAML.load_file(pdf) + +# Inventory file creation +File.open(File.dirname(__FILE__) + "/inventory/hosts.ini", "w") do |inventory_file| + inventory_file.puts("[all]") + nodes.each do |node| + inventory_file.puts("#{node['name']}\tansible_ssh_host=#{node['ip']} ansible_ssh_port=22") + end + ['kube-master', 'kube-node', 'etcd', 'ovn-central', 'ovn-controller', 'virtlet', 'cmk'].each do|group| + inventory_file.puts("\n[#{group}]") + nodes.each do |node| + if node['roles'].include?("#{group}") + inventory_file.puts(node['name']) + end + end + end + inventory_file.puts("\n[k8s-cluster:children]\nkube-node\nkube-master") +end + +provider = (ENV['VAGRANT_DEFAULT_PROVIDER'] || :libvirt).to_sym +puts "[INFO] Provider: #{provider} " + +if ENV['no_proxy'] != nil or ENV['NO_PROXY'] + $no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || "127.0.0.1,localhost" + nodes.each do |node| + $no_proxy += "," + node['ip'] + end + $subnet = "192.168.121" + if provider == :virtualbox + $subnet = "10.0.2" + end + # NOTE: This range is based on vagrant-libvirt network definition CIDR 192.168.121.0/27 + (1..31).each do |i| + $no_proxy += ",#{$subnet}.#{i}" + end +end + +Vagrant.configure("2") do |config| + config.vm.box = box[provider][:name] + config.vm.box_version = box[provider][:version] + config.ssh.insert_key = false + + if ENV['http_proxy'] != nil and ENV['https_proxy'] != nil + if Vagrant.has_plugin?('vagrant-proxyconf') + config.proxy.http = ENV['http_proxy'] || ENV['HTTP_PROXY'] || "" + config.proxy.https = ENV['https_proxy'] || ENV['HTTPS_PROXY'] || "" + config.proxy.no_proxy = $no_proxy + config.proxy.enabled = { docker: false } + end + end + config.vm.provider 'libvirt' do |v| + v.nested = true + v.cpu_mode = 'host-passthrough' + v.management_network_address = "192.168.121.0/27" + v.random_hostname = true + end + + sync_type = "virtualbox" + if provider == :libvirt + sync_type = "nfs" + end + + nodes.each do |node| + config.vm.define node['name'] do |nodeconfig| + if node['roles'].include?("kube-master") + nodeconfig.vm.synced_folder '../', '/home/vagrant/multicloud-k8s/', type: sync_type + end + if node['roles'].include?("kube-node") + nodeconfig.vm.provision 'shell', privileged: false do |sh| + sh.inline = <<-SHELL + sudo sed -i 's:GRUB_CMDLINE_LINUX=.*:GRUB_CMDLINE_LINUX="isolcpus=0-7":' /etc/default/grub + sudo update-grub + SHELL + end + end + nodeconfig.vm.hostname = node['name'] + nodeconfig.vm.network :private_network, :ip => node['ip'], :type => :static + nodeconfig.vm.provider 'virtualbox' do |v| + v.customize ["modifyvm", :id, "--memory", node['memory']] + v.customize ["modifyvm", :id, "--cpus", node['cpus']] + if node.has_key? "volumes" + node['volumes'].each do |volume| + $volume_file = "#{node['name']}-#{volume['name']}.vdi" + unless File.exist?($volume_file) + v.customize ['createmedium', 'disk', '--filename', $volume_file, '--size', volume['size']] + end + v.customize ['storageattach', :id, '--storagectl', 'IDE Controller', '--port', 1, '--device', 0, '--type', 'hdd', '--medium', $volume_file] + end + end + end + nodeconfig.vm.provider 'libvirt' do |v| + v.memory = node['memory'] + v.cpus = node['cpus'] + nodeconfig.vm.provision 'file', source: './scripts', destination: '$HOME/' + nodeconfig.vm.provision 'file', source: '../edge-scripts/setup-cnf.sh', destination: '$HOME/scripts/' + nodeconfig.vm.provision 'file', source: '../edge-scripts/sdwan_verifier.sh', destination: '$HOME/scripts/' + nodeconfig.vm.provision 'shell' do |sh| + sh.path = "node.sh" + if node.has_key? "volumes" + $volume_mounts_dict = '' + node['volumes'].each do |volume| + $volume_mounts_dict += "#{volume['name']}=#{volume['mount']}," + $volume_file = "./#{node['name']}-#{volume['name']}.qcow2" + v.storage :file, :bus => 'sata', :device => volume['name'], :size => volume['size'] + end + sh.args = ['-v', $volume_mounts_dict[0...-1]] + end + end + end + end + end + + config.vm.define :installer, primary: true, autostart: false do |installer| + installer.vm.hostname = "multicloud" + installer.vm.network :private_network, :ip => "10.10.10.2", :type => :static + installer.vm.synced_folder '../', '/home/vagrant/multicloud-k8s/', type: sync_type + installer.vm.provision 'shell', privileged: false do |sh| + sh.env = {'KUD_PLUGIN_ENABLED': 'false', 'OVN_CENTRAL_INTERFACE': 'eth1'} + sh.inline = <<-SHELL + cp /home/vagrant/multicloud-k8s/edge-b/insecure_keys/key.pub /home/vagrant/.ssh/id_rsa.pub + cp /home/vagrant/multicloud-k8s/edge-b/insecure_keys/key /home/vagrant/.ssh/id_rsa + chown vagrant /home/vagrant/.ssh/id_rsa + chmod 400 /home/vagrant/.ssh/id_rsa + sudo apt install jq -y + cd /home/vagrant/multicloud-k8s/edge-b/ && ./installer.sh | tee kud_installer.log + SHELL + end + end +end diff --git a/platform/test/e2e-test/edge-b/config/default.yml b/platform/test/e2e-test/edge-b/config/default.yml new file mode 100644 index 0000000..6a1f9bc --- /dev/null +++ b/platform/test/e2e-test/edge-b/config/default.yml @@ -0,0 +1,24 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- name: "edge-b" + ip: "10.10.10.4" + memory: 18432 + cpus: 16 + volumes: + - name: sdb + size: 50 + mount: /var/lib/docker/ + roles: + - kube-master + - etcd + - ovn-central + - kube-node + - ovn-controller diff --git a/platform/test/e2e-test/edge-b/insecure_keys/key b/platform/test/e2e-test/edge-b/insecure_keys/key new file mode 100644 index 0000000..7d6a083 --- /dev/null +++ b/platform/test/e2e-test/edge-b/insecure_keys/key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI +w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP +kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2 +hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO +Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW +yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd +ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1 +Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf +TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK +iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A +sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf +4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP +cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk +EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN +CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX +3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG +YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj +3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+ +dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz +6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC +P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF +llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ +kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH ++vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ +NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s= +-----END RSA PRIVATE KEY----- diff --git a/platform/test/e2e-test/edge-b/insecure_keys/key.pub b/platform/test/e2e-test/edge-b/insecure_keys/key.pub new file mode 100644 index 0000000..18a9c00 --- /dev/null +++ b/platform/test/e2e-test/edge-b/insecure_keys/key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key diff --git a/platform/test/e2e-test/edge-b/installer.sh b/platform/test/e2e-test/edge-b/installer.sh new file mode 100755 index 0000000..3ef2762 --- /dev/null +++ b/platform/test/e2e-test/edge-b/installer.sh @@ -0,0 +1,233 @@ +#!/bin/bash +#SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o errexit +set -o nounset +set -o pipefail + +INSTALLER_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" + +source ${INSTALLER_DIR}/../k8s/kud/tests/_functions.sh + +# _install_go() - Install GoLang package +function _install_go { + version=$(grep "go_version" ${kud_playbooks}/kud-vars.yml | awk -F "'" '{print $2}') + local tarball=go$version.linux-amd64.tar.gz + + #gcc is required for go apps compilation + if ! which gcc; then + sudo apt-get install -y gcc + fi + + if $(go version &>/dev/null); then + return + fi + + wget https://dl.google.com/go/$tarball + sudo tar -C /usr/local -xzf $tarball + rm $tarball + + export PATH=$PATH:/usr/local/go/bin + sudo sed -i "s|^PATH=.*|PATH=\"$PATH\"|" /etc/environment +} + +# _install_pip() - Install Python Package Manager +function _install_pip { + if $(pip --version &>/dev/null); then + sudo -E pip install --upgrade pip + else + sudo apt-get install -y python-dev + curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python + fi +} + +# _install_ansible() - Install and Configure Ansible program +function _install_ansible { + if $(ansible --version &>/dev/null); then + sudo pip uninstall -y ansible + fi + _install_pip + local version=$(grep "ansible_version" ${kud_playbooks}/kud-vars.yml | awk -F ': ' '{print $2}') + sudo mkdir -p /etc/ansible/ + sudo -E pip install ansible==$version +} + +# _install_docker() - Download and install docker-engine +function _install_docker { + local max_concurrent_downloads=${1:-3} + + if $(docker version &>/dev/null); then + return + fi + sudo apt-get install -y apt-transport-https ca-certificates curl + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + sudo apt-get update + sudo apt-get install -y docker-ce + + sudo mkdir -p /etc/systemd/system/docker.service.d + if [ ${http_proxy:-} ]; then + echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/http-proxy.conf + echo "Environment=\"HTTP_PROXY=$http_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/http-proxy.conf + fi + if [ ${https_proxy:-} ]; then + echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/https-proxy.conf + echo "Environment=\"HTTPS_PROXY=$https_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/https-proxy.conf + fi + if [ ${no_proxy:-} ]; then + echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/no-proxy.conf + echo "Environment=\"NO_PROXY=$no_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/no-proxy.conf + fi + sudo systemctl daemon-reload + echo "DOCKER_OPTS=\"-H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock --max-concurrent-downloads $max_concurrent_downloads \"" | sudo tee --append /etc/default/docker + if [[ -z $(groups | grep docker) ]]; then + sudo usermod -aG docker $USER + fi + + sudo systemctl restart docker + sleep 10 +} + +function _set_environment_file { + # By default ovn central interface is the first active network interface on localhost. If other wanted, need to export this variable in aio.sh or Vagrant file. + OVN_CENTRAL_INTERFACE="${OVN_CENTRAL_INTERFACE:-$(ip addr show | awk '/inet.*brd/{print $NF; exit}')}" + echo "export OVN_CENTRAL_INTERFACE=${OVN_CENTRAL_INTERFACE}" | sudo tee --append /etc/environment + echo "export OVN_CENTRAL_ADDRESS=$(get_ovn_central_address)" | sudo tee --append /etc/environment + echo "export KUBE_CONFIG_DIR=/opt/kubeconfig" | sudo tee --append /etc/environment + echo "export CSAR_DIR=/opt/csar" | sudo tee --append /etc/environment +} + +# install_k8s() - Install Kubernetes using kubespray tool +function install_k8s { + echo "Deploying kubernetes" + local dest_folder=/opt + version=$(grep "kubespray_version" ${kud_playbooks}/kud-vars.yml | awk -F ': ' '{print $2}') + local_release_dir=$(grep "local_release_dir" $kud_inventory_folder/group_vars/k8s-cluster.yml | awk -F "\"" '{print $2}') + local tarball=v$version.tar.gz + sudo apt-get install -y sshpass make unzip # install make to run mitogen target and unzip is mitogen playbook dependency + sudo apt-get install -y gnupg2 software-properties-common + _install_docker + _install_ansible + wget https://github.com/kubernetes-incubator/kubespray/archive/$tarball + sudo tar -C $dest_folder -xzf $tarball + sudo mv $dest_folder/kubespray-$version/ansible.cfg /etc/ansible/ansible.cfg + sudo chown -R $USER $dest_folder/kubespray-$version + sudo mkdir -p ${local_release_dir}/containers + rm $tarball + + pushd $dest_folder/kubespray-$version/ + sudo -E pip install -r ./requirements.txt + make mitogen + popd + rm -f $kud_inventory_folder/group_vars/all.yml 2> /dev/null + if [[ -n "${verbose:-}" ]]; then + echo "kube_log_level: 5" | tee $kud_inventory_folder/group_vars/all.yml + else + echo "kube_log_level: 2" | tee $kud_inventory_folder/group_vars/all.yml + fi + echo "kubeadm_enabled: true" | tee --append $kud_inventory_folder/group_vars/all.yml + if [[ -n "${http_proxy:-}" ]]; then + echo "http_proxy: \"$http_proxy\"" | tee --append $kud_inventory_folder/group_vars/all.yml + fi + if [[ -n "${https_proxy:-}" ]]; then + echo "https_proxy: \"$https_proxy\"" | tee --append $kud_inventory_folder/group_vars/all.yml + fi + ansible-playbook $verbose -i $kud_inventory $dest_folder/kubespray-$version/cluster.yml --become --become-user=root | sudo tee $log_folder/setup-kubernetes.log + + # Configure environment + mkdir -p $HOME/.kube + cp $kud_inventory_folder/artifacts/admin.conf $HOME/.kube/config + # Copy Kubespray kubectl to be usable in host running Ansible. Requires kubectl_localhost: true in inventory/group_vars/k8s-cluster.yml + sudo cp $kud_inventory_folder/artifacts/kubectl /usr/local/bin/ +} + +# install_addons() - Install Kubenertes AddOns +function install_addons { + source /etc/environment + echo "Installing Kubernetes AddOns" + _install_ansible + sudo ansible-galaxy install $verbose -r $kud_infra_folder/galaxy-requirements.yml --ignore-errors + ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-kud.yml | sudo tee $log_folder/setup-kud.log + for addon in ${KUD_ADDONS:-ovn4nfv}; do + echo "Deploying $addon using configure-$addon.yml playbook.." + ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-${addon}.yml | sudo tee $log_folder/setup-${addon}.log + done + echo "Run the test cases if testing_enabled is set to true." + if [[ "${testing_enabled}" == "true" ]]; then + for addon in ${KUD_ADDONS:-ovn4nfv}; do + pushd $kud_tests + bash ${addon}.sh + popd + done + fi + echo "Add-ons deployment complete..." +} + +# _print_kubernetes_info() - Prints the login Kubernetes information +function _print_kubernetes_info { + if ! $(kubectl version &>/dev/null); then + return + fi + # Expose Dashboard using NodePort + node_port=30080 + KUBE_EDITOR="sed -i \"s|type\: ClusterIP|type\: NodePort|g\"" kubectl -n kube-system edit service kubernetes-dashboard + KUBE_EDITOR="sed -i \"s|nodePort\: .*|nodePort\: $node_port|g\"" kubectl -n kube-system edit service kubernetes-dashboard + + master_ip=$(kubectl cluster-info | grep "Kubernetes master" | awk -F ":" '{print $2}') + + printf "Kubernetes Info\n===============\n" > $k8s_info_file + echo "Dashboard URL: https:$master_ip:$node_port" >> $k8s_info_file + echo "Admin user: kube" >> $k8s_info_file + echo "Admin password: secret" >> $k8s_info_file +} + +sudo -k # forgot sudo password +if ! sudo -n "true"; then + echo "" + echo "passwordless sudo is needed for '$(id -nu)' user." + echo "Please fix your /etc/sudoers file. You likely want an" + echo "entry like the following one..." + echo "" + echo "$(id -nu) ALL=(ALL) NOPASSWD: ALL" + exit 1 +fi + +verbose="" +if [[ -n "${KUD_DEBUG:-}" ]]; then + set -o xtrace + verbose="-vvv" +fi + +# Configuration values +log_folder=/var/log/kud +kud_folder=${INSTALLER_DIR}/../k8s/kud/hosting_providers/vagrant +kud_infra_folder=$kud_folder/../../deployment_infra +export kud_inventory_folder=$kud_folder/../../../../edge-b/inventory +kud_inventory=$kud_inventory_folder/hosts.ini +kud_playbooks=$kud_infra_folder/playbooks +kud_tests=$kud_folder/../../tests +k8s_info_file=$kud_folder/k8s_info.log +testing_enabled=${KUD_ENABLE_TESTS:-false} +sudo mkdir -p $log_folder +sudo mkdir -p /opt/csar +sudo chown -R $USER /opt/csar +# Install dependencies +# Setup proxy variables +if [ -f $kud_folder/sources.list ]; then + sudo mv /etc/apt/sources.list /etc/apt/sources.list.backup + sudo cp $kud_folder/sources.list /etc/apt/sources.list +fi +echo "Removing ppa for jonathonf/python-3.6" +sudo ls /etc/apt/sources.list.d/ || true +sudo find /etc/apt/sources.list.d -maxdepth 1 -name '*jonathonf*' -delete || true +sudo apt-get update +install_k8s +_set_environment_file +install_addons diff --git a/platform/test/e2e-test/edge-b/inventory/group_vars/all.yml b/platform/test/e2e-test/edge-b/inventory/group_vars/all.yml new file mode 100644 index 0000000..528430c --- /dev/null +++ b/platform/test/e2e-test/edge-b/inventory/group_vars/all.yml @@ -0,0 +1,2 @@ +kube_log_level: 2 +kubeadm_enabled: true diff --git a/platform/test/e2e-test/edge-b/inventory/group_vars/k8s-cluster.yml b/platform/test/e2e-test/edge-b/inventory/group_vars/k8s-cluster.yml new file mode 100644 index 0000000..ab91498 --- /dev/null +++ b/platform/test/e2e-test/edge-b/inventory/group_vars/k8s-cluster.yml @@ -0,0 +1,82 @@ +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Kubernetes configuration dirs and system namespace. +# Those are where all the additional config stuff goes +# kubernetes normally puts in /srv/kubernetes. +# This puts them in a sane location and namespace. +# Editing those values will almost surely break something. +system_namespace: kube-system + +# Logging directory (sysvinit systems) +kube_log_dir: "/var/log/kubernetes" + +kube_api_anonymous_auth: true + +# Users to create for basic auth in Kubernetes API via HTTP +# Optionally add groups for user +kube_api_pwd: "secret" +kube_users: + kube: + pass: "{{kube_api_pwd}}" + role: admin + groups: + - system:masters + +## It is possible to activate / deactivate selected authentication methods (basic auth, static token auth) +#kube_oidc_auth: false +kube_basic_auth: true +kube_token_auth: true + +# Choose network plugin (calico, contiv, weave or flannel) +# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing +kube_network_plugin: flannel + +# Make a copy of kubeconfig (admin.conf) on the host that runs Ansible to inventory/artifacts +kubeconfig_localhost: true +# Copy kubectl binary on the host that runs Ansible to inventory/artifacts +kubectl_localhost: true +# Disable nodelocal dns cache +enable_nodelocaldns: false +# Enable MountPropagation gate feature +local_volumes_enabled: true +local_volume_provisioner_enabled: true + +## Change this to use another Kubernetes version, e.g. a current beta release +kube_version: v1.14.3 + +# Helm deployment +helm_enabled: true + +# Kube-proxy proxyMode configuration. +# NOTE: Ipvs is based on netfilter hook function, but uses hash table as the underlying data structure and +# works in the kernel space +# https://kubernetes.io/docs/concepts/services-networking/service/#proxy-mode-ipvs +#kube_proxy_mode: ipvs + +# Download container images only once then push to cluster nodes in batches +download_run_once: true + +# Where the binaries will be downloaded. +# Note: ensure that you've enough disk space (about 1G) +local_release_dir: "/tmp/releases" + +# Makes the installer node a delegate for pushing images while running +# the deployment with ansible. This maybe the case if cluster nodes +# cannot access each over via ssh or you want to use local docker +# images as a cache for multiple clusters. +download_localhost: true + +# Subnet for cluster IPs +kube_service_addresses: 10.244.0.0/18 +# Subnet for Pod IPs +kube_pods_subnet: 10.244.64.0/18 + +# Add specific docker version +docker_version: 'latest' diff --git a/platform/test/e2e-test/edge-b/node.sh b/platform/test/e2e-test/edge-b/node.sh new file mode 100755 index 0000000..a51be19 --- /dev/null +++ b/platform/test/e2e-test/edge-b/node.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o nounset +set -o pipefail + +# usage() - Prints the usage of the program +function usage { + cat <> /etc/fstab +} + +while getopts "h?v:" opt; do + case $opt in + v) + dict_volumes="$OPTARG" + ;; + h|\?) + usage + exit + ;; + esac +done + +swapoff -a +if [[ -n "${dict_volumes+x}" ]]; then + for kv in ${dict_volumes//,/ } ;do + mount_external_partition ${kv%=*} ${kv#*=} + done +fi + +vendor_id=$(lscpu|grep "Vendor ID") +if [[ $vendor_id == *GenuineIntel* ]]; then + kvm_ok=$(cat /sys/module/kvm_intel/parameters/nested) + if [[ $kvm_ok == 'N' ]]; then + echo "Enable Intel Nested-Virtualization" + rmmod kvm-intel + echo 'options kvm-intel nested=y' >> /etc/modprobe.d/dist.conf + modprobe kvm-intel + echo kvm-intel >> /etc/modules + fi +else + kvm_ok=$(cat /sys/module/kvm_amd/parameters/nested) + if [[ $kvm_ok == '0' ]]; then + echo "Enable AMD Nested-Virtualization" + rmmod kvm-amd + sh -c "echo 'options kvm-amd nested=1' >> /etc/modprobe.d/dist.conf" + modprobe kvm-amd + echo kvm-amd >> /etc/modules + fi +fi +modprobe vhost_net +echo vhost_net >> /etc/modules +source /etc/os-release || source /usr/lib/os-release +case ${ID,,} in + *suse) + ;; + ubuntu|debian) + apt-get install -y cpu-checker + kvm-ok + ;; + rhel|centos|fedora) + ;; +esac diff --git a/platform/test/e2e-test/edge-b/scripts/edge-b-sdwan-config b/platform/test/e2e-test/edge-b/scripts/edge-b-sdwan-config new file mode 100644 index 0000000..e41d386 --- /dev/null +++ b/platform/test/e2e-test/edge-b/scripts/edge-b-sdwan-config @@ -0,0 +1,14 @@ +internal_subnet=192.168.1.0/24 +redirect=" +{ + \"name\":\"DNAT\", + \"src\":\"wan0\", + \"src_ip\":\"$internal_subnet\", + \"src_dip\":\"$sdwan_pod_virtual_ip\", + \"src_dport\":\"80\", + \"dest_ip\":\"$appLan0\", + \"proto\":\"tcp\", + \"target\":\"DNAT\" +} +" + diff --git a/platform/test/e2e-test/edge-b/scripts/variables b/platform/test/e2e-test/edge-b/scripts/variables new file mode 100644 index 0000000..1497aff --- /dev/null +++ b/platform/test/e2e-test/edge-b/scripts/variables @@ -0,0 +1,18 @@ +providerSubnet=10.10.10.1/24 +providerGateway=10.10.10.1/24 +providerExcludeIps=10.10.10.2..10.10.10.9 +providerNetworkInterface=eth1 +ovnSubnet=172.16.31.1/24 +ovnGateway=172.16.31.1/24 +ovnExcludeIps=172.16.31.2..172.16.31.9 +cnfWanGateway=10.10.10.1 + +export cnfWan0=10.10.10.25 +export cnfLan0=172.16.31.10 +export appLan0=172.16.31.15 +export hubIp=10.10.10.35 +export edgeName=edgeB +export wan_interface=net1 +export sdewan_cnf_name=sdewan-edge-b +export app_pod_name=simple-http-service +export sdwanConfigs=edge-b-sdwan-config diff --git a/platform/test/e2e-test/edge-scripts/sdwan_verifier.sh b/platform/test/e2e-test/edge-scripts/sdwan_verifier.sh new file mode 100644 index 0000000..cab72f0 --- /dev/null +++ b/platform/test/e2e-test/edge-scripts/sdwan_verifier.sh @@ -0,0 +1,171 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o errexit +set -o nounset +set -o pipefail + + +function login { + login_url=http://$1/cgi-bin/luci/ + echo $(wget -S --spider --post-data "luci_username=root&luci_password=" $login_url 2>&1 | grep sysauth= | sed -r 's/.*sysauth=([^;]+);.*/\1/') +} + +function call_get { + url=$1 + token=$2 + echo $(curl -s -X "GET" -H "Cookie:sysauth=$token" -H "Content-Type: application/json" $url) +} + +function call_post { + url=$1 + token=$2 + payload=$3 + echo $(curl -s -X "POST" -H "Cookie:sysauth=$token" -H "Content-Type: application/json" -d "$payload" $url) +} + +function call_put { + url=$1 + token=$2 + payload=$3 + echo $(curl -s -X "PUT" -H "Cookie:sysauth=$token" -H "Content-Type: application/json" -d "$payload" $url) +} + +function call_delete { + url=$1 + token=$2 + echo $(curl -s -X "DELETE" -H "Cookie:sysauth=$token" -H "Content-Type: application/json" $url) +} + +function wait_for_pod { + status_phase="" + while [[ "$status_phase" != "Running" ]]; do + new_phase="$(kubectl get pods -o wide | grep ^$1 | awk '{print $3}')" + if [[ "$new_phase" != "$status_phase" ]]; then + status_phase="$new_phase" + fi + if [[ "$new_phase" == "Err"* ]]; then + exit 1 + fi + done +} + +sdwan_pod_ip=$(kubectl get pods -o wide | grep $sdewan_cnf_name | awk '{print $6}') + +echo "SDWAN pod ip:"$sdwan_pod_ip + +echo "Login to sdwan ..." +security_token="" +while [[ "$security_token" == "" ]]; do + echo "Get Security Token ..." + security_token=$(login $sdwan_pod_ip) + sleep 2 +done +echo "* Security Token: "$security_token + +zone0=" +{ + \"name\":\"lan0\", + \"network\": [\"lan0\"], + \"input\":\"ACCEPT\", + \"output\":\"ACCEPT\", + \"forward\":\"ACCEPT\" +} +" + +zone1=" +{ + \"name\":\"wan0\", + \"network\": [\"wan0\"], + \"input\":\"ACCEPT\", + \"output\":\"ACCEPT\", + \"forward\":\"REJECT\", + \"masq\":\"0\", + \"mtu_fix\":\"1\" +} +" + +echo $(call_post http://$sdwan_pod_ip/cgi-bin/luci/sdewan/firewall/v1/zones $security_token "$zone0") +sleep 3 +echo $(call_post http://$sdwan_pod_ip/cgi-bin/luci/sdewan/firewall/v1/zones $security_token "$zone1") +sleep 3 +echo $(call_get http://$sdwan_pod_ip/cgi-bin/luci/sdewan/firewall/v1/zones $security_token) + + +echo "Set ipsec rules ..." +proposal=" +{ + \"name\":\"my_test_proposal1\", + \"encryption_algorithm\":\"aes128\", + \"hash_algorithm\":\"sha256\", + \"dh_group\":\"modp3072\" +} +" +echo $(call_post http://$sdwan_pod_ip/cgi-bin/luci/sdewan/ipsec/v1/proposals $security_token "$proposal") +sleep 3 +echo "Proposals inserted..." +echo $(call_get http://$sdwan_pod_ip/cgi-bin/luci/sdewan/ipsec/v1/proposals $security_token) + +rule=" +{ + \"name\":\"$edgeName\", + \"gateway\":\"$hubIp\", + \"pre_shared_key\":\"test_key\", + \"authentication_method\":\"psk\", + \"local_identifier\":\"$cnfWan0\", + \"crypto_proposal\":[\"my_test_proposal1\"], + \"force_crypto_proposal\":\"0\", + \"connections\":[{ + \"name\":\"connectionA\", + \"type\":\"tunnel\", + \"mode\":\"start\", + \"local_sourceip\":\"%config\", + \"remote_subnet\":\"192.168.1.1/24,$hubIp/32\", + \"crypto_proposal\":[\"my_test_proposal1\"] + }] +} +" +echo $(call_post http://$sdwan_pod_ip/cgi-bin/luci/sdewan/ipsec/v1/sites $security_token "$rule") +sleep 3 +echo "IPSec configs inserted..." +echo $(call_get http://$sdwan_pod_ip/cgi-bin/luci/sdewan/ipsec/v1/sites $security_token) + +echo "Restarting IPSec service..." +operation=" +{ + \"action\":\"restart\" +} +" +echo $(call_put http://$sdwan_pod_ip/cgi-bin/luci/sdewan/v1/service/ipsec $security_token "$operation") +sleep 5 +sdwan_pod_virtual_ip=$(kubectl exec -it $sdewan_cnf_name ip address | grep $wan_interface | awk '/inet/{print $2}' | cut -f1 -d "/" | grep 192.168) + +echo "Applying sdwan redirect configs" +test -f ./$sdwanConfigs && . $sdwanConfigs +redirect=${redirect} + +echo $(call_post http://$sdwan_pod_ip/cgi-bin/luci/sdewan/firewall/v1/redirects $security_token "$redirect") +sleep 3 +echo $(call_get http://$sdwan_pod_ip/cgi-bin/luci/sdewan/firewall/v1/redirects $security_token) + +echo "Restarting Firewall service..." +operation=" +{ + \"action\":\"restart\" +} +" +echo $(call_put http://$sdwan_pod_ip/cgi-bin/luci/sdewan/v1/service/firewall $security_token "$operation") + + +echo "Add route to bring the app traffic to sdewan cnf" +app_pod=$(kubectl get po | grep $app_pod_name | cut -f1 -d " ") +kubectl exec -it $app_pod ip route add $internal_subnet via $cnfLan0 dev net0 + +echo "Test Completed!" diff --git a/platform/test/e2e-test/edge-scripts/setup-cnf.sh b/platform/test/e2e-test/edge-scripts/setup-cnf.sh new file mode 100755 index 0000000..42057e4 --- /dev/null +++ b/platform/test/e2e-test/edge-scripts/setup-cnf.sh @@ -0,0 +1,254 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o errexit +set -o nounset +set -o pipefail + +base=$(pwd) + +test -f $base/variables +. $base/variables +providerSubnet=${providerSubnet} +providerGateway=${providerGateway} +providerExcludeIps=${providerExcludeIps} +providerNetworkInterface=${providerNetworkInterface} +ovnSubnet=${ovnSubnet} +ovnGateway=${ovnGateway} +ovnExcludeIps=${ovnExcludeIps} +cnfLan0=${cnfLan0} +appLan0=${appLan0} +cnfWanGateway=${cnfWanGateway} +app_pod_name=${app_pod_name} + +clean() +{ +echo "Cleaning ..." +kubectl delete -f httpbin-svc.yaml +kubectl delete -f sdewan-cnf.yaml +kubectl delete -f network-prepare.yaml +} + +error_detect() +{ + echo "Error on line $1" + clean +} + +trap "error_detect $LINENO" ERR + +echo "Setup CNF for ${edgeName}" +echo "Creating ovn networks..." +cat > network-prepare.yaml << EOF +--- +apiVersion: k8s.plugin.opnfv.org/v1alpha1 +kind: ProviderNetwork +metadata: + name: pnetwork +spec: + cniType: ovn4nfv + ipv4Subnets: + - subnet: $providerSubnet + name: subnet + gateway: $providerGateway + excludeIps: $providerExcludeIps + providerNetType: DIRECT + direct: + providerInterfaceName: $providerNetworkInterface + directNodeSelector: all + +--- +apiVersion: k8s.plugin.opnfv.org/v1alpha1 +kind: Network +metadata: + name: ovn-network +spec: + # Add fields here + cniType: ovn4nfv + ipv4Subnets: + - subnet: $ovnSubnet + name: subnet1 + gateway: $ovnGateway + excludeIps: $ovnExcludeIps + +EOF + +kubectl create -f network-prepare.yaml +sleep 2 + +ovnNet=$(kubectl get network | sed -n 2p | awk '{print $1}') +ovnProviderNet=$(kubectl get providernetwork | sed -n 2p | awk '{print $1}') +if [ -n "${ovnNet}" ] && [ -n "${ovnProviderNet}" ] +then + echo "Networks created successfully" +else + echo "Networks creation failed" + exit 1 +fi + + +echo "Creating sdwan-cnf ..." +cat > sdewan-cnf.yaml << EOF +apiVersion: v1 +kind: ConfigMap +metadata: + name: sdwan-config +data: + entrypoint.sh: | + #!/bin/bash + # Always exit on errors. + set -e + + cat > /etc/config/network << EOF + config interface 'lan0' + option ifname 'net0' + option proto 'static' + option ipaddr '$cnfLan0' + option netmask '255.255.255.0' + + config interface 'wan0' + option ifname 'net1' + option proto 'static' + option ipaddr '$cnfWan0' + option netmask '255.255.255.0' + EOF + + iptables -t nat -L + /sbin/procd & + /sbin/ubusd & + sleep 1 + /etc/init.d/rpcd start + /etc/init.d/dnsmasq start + /etc/init.d/network start + /etc/init.d/odhcpd start + /etc/init.d/uhttpd start + /etc/init.d/log start + /etc/init.d/dropbear start + /etc/init.d/mwan3 restart + + echo "Entering sleep... (success)" + + # Sleep forever. + while true; do sleep 100; done + +--- +apiVersion: v1 +kind: Pod +metadata: + name: $sdewan_cnf_name + annotations: + k8s.v1.cni.cncf.io/networks: '[{ "name": "ovn-networkobj"}]' + k8s.plugin.opnfv.org/nfn-network: '{ "type": "ovn4nfv", "interface": [{"name": "ovn-network", "interface": "net0", "ipAddress": "$cnfLan0"}, {"name": "pnetwork", "interface": "net1", "ipAddress": "$cnfWan0", "gateway": "$cnfWanGateway"}]}' +spec: + containers: + - name: sdwan-pod-net + image: integratedcloudnative/openwrt:test + ports: + - containerPort: 22 + - containerPort: 80 + command: + - /bin/sh + - /init/entrypoint.sh + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + volumeMounts: + - name: entrypoint-sh + mountPath: /init + volumes: + - name: entrypoint-sh + configMap: + name: sdwan-config + items: + - key: entrypoint.sh + path: entrypoint.sh +EOF + +kubectl create -f sdewan-cnf.yaml +sleep 20 + +sdwan_status=$(kubectl get po | grep $sdewan_cnf_name | awk '{print $3}') +if [ "$sdwan_status" == "Running" ] +then + echo "Sdewan cnf $sdewan_cnf_name created successfully" +else + sleep 40 + sdwan_status=$(kubectl get po | grep $sdewan_cnf_name | awk '{print $3}') + if [ "$sdwan_status" != "Running" ] + then + echo "Sdewan cnf creation failed" + exit 2 + fi +fi + +kubectl exec -it $sdewan_cnf_name /etc/init.d/network restart + + +cat > httpbin-svc.yaml << EOF +--- +apiVersion: v1 +kind: Service +metadata: + name: my-http-service +spec: + selector: + app: MyApp + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: $app_pod_name +spec: + replicas: 1 + selector: + matchLabels: + name: simple-http-service + template: + metadata: + labels: + name: simple-http-service + app: MyApp + annotations: + k8s.v1.cni.cncf.io/networks: '[{ "name": "ovn-networkobj"}]' + k8s.plugin.opnfv.org/nfn-network: '{ "type": "ovn4nfv", "interface": [{"name": "ovn-network", "interface": "net0", "ipAddress": "$appLan0"}]}' + spec: + containers: + - name: simple-http-service + image: integratedcloudnative/httpbin:test + ports: + - containerPort: 80 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true +EOF +kubectl create -f httpbin-svc.yaml +sleep 20 + +appStatus=$(kubectl get po | grep simple-http-service | awk '{print $3}') +if [ "$appStatus" == "Running" ] +then + echo "Application $app_pod_name installation success" +else + sleep 20 + appStatus=$(kubectl get po | grep simple-http-service | awk '{print $3}') + if [ "$appStatus" != "Running" ] + then + echo "Application creation failed" + exit 1 + fi +fi + +echo "Setup openwrt configurations" +bash sdwan_verifier.sh diff --git a/platform/test/e2e-test/sdewan-hub/Vagrantfile b/platform/test/e2e-test/sdewan-hub/Vagrantfile new file mode 100644 index 0000000..ef5ee09 --- /dev/null +++ b/platform/test/e2e-test/sdewan-hub/Vagrantfile @@ -0,0 +1,148 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +box = { + :virtualbox => { :name => 'elastic/ubuntu-18.04-x86_64', :version => '20191013.0.0'}, + :libvirt => { :name => 'intergratedcloudnative/ubuntu1804', :version => '1.0.0'} +} + +require 'yaml' +pdf = File.dirname(__FILE__) + '/config/default.yml' +if File.exist?(File.dirname(__FILE__) + '/config/pdf.yml') + pdf = File.dirname(__FILE__) + '/config/pdf.yml' +end +nodes = YAML.load_file(pdf) + +# Inventory file creation +File.open(File.dirname(__FILE__) + "/inventory/hosts.ini", "w") do |inventory_file| + inventory_file.puts("[all]") + nodes.each do |node| + inventory_file.puts("#{node['name']}\tansible_ssh_host=#{node['ip']} ansible_ssh_port=22") + end + ['kube-master', 'kube-node', 'etcd', 'ovn-central', 'ovn-controller', 'virtlet', 'cmk'].each do|group| + inventory_file.puts("\n[#{group}]") + nodes.each do |node| + if node['roles'].include?("#{group}") + inventory_file.puts(node['name']) + end + end + end + inventory_file.puts("\n[k8s-cluster:children]\nkube-node\nkube-master") +end + +provider = (ENV['VAGRANT_DEFAULT_PROVIDER'] || :libvirt).to_sym +puts "[INFO] Provider: #{provider} " + +if ENV['no_proxy'] != nil or ENV['NO_PROXY'] + $no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || "127.0.0.1,localhost" + nodes.each do |node| + $no_proxy += "," + node['ip'] + end + $subnet = "192.168.121" + if provider == :virtualbox + $subnet = "10.0.2" + end + # NOTE: This range is based on vagrant-libvirt network definition CIDR 192.168.121.0/27 + (1..31).each do |i| + $no_proxy += ",#{$subnet}.#{i}" + end +end + +Vagrant.configure("2") do |config| + config.vm.box = box[provider][:name] + config.vm.box_version = box[provider][:version] + config.ssh.insert_key = false + + if ENV['http_proxy'] != nil and ENV['https_proxy'] != nil + if Vagrant.has_plugin?('vagrant-proxyconf') + config.proxy.http = ENV['http_proxy'] || ENV['HTTP_PROXY'] || "" + config.proxy.https = ENV['https_proxy'] || ENV['HTTPS_PROXY'] || "" + config.proxy.no_proxy = $no_proxy + config.proxy.enabled = { docker: false } + end + end + config.vm.provider 'libvirt' do |v| + v.nested = true + v.cpu_mode = 'host-passthrough' + v.management_network_address = "192.168.121.0/27" + v.random_hostname = true + end + + sync_type = "virtualbox" + if provider == :libvirt + sync_type = "nfs" + end + + nodes.each do |node| + config.vm.define node['name'] do |nodeconfig| + if node['roles'].include?("kube-master") + nodeconfig.vm.synced_folder '../', '/home/vagrant/multicloud-k8s/', type: sync_type + end + if node['roles'].include?("kube-node") + nodeconfig.vm.provision 'shell', privileged: false do |sh| + sh.inline = <<-SHELL + sudo sed -i 's:GRUB_CMDLINE_LINUX=.*:GRUB_CMDLINE_LINUX="isolcpus=0-7":' /etc/default/grub + sudo update-grub + SHELL + end + end + nodeconfig.vm.hostname = node['name'] + nodeconfig.vm.network :private_network, :ip => node['ip'], :type => :static + nodeconfig.vm.provider 'virtualbox' do |v| + v.customize ["modifyvm", :id, "--memory", node['memory']] + v.customize ["modifyvm", :id, "--cpus", node['cpus']] + if node.has_key? "volumes" + node['volumes'].each do |volume| + $volume_file = "#{node['name']}-#{volume['name']}.vdi" + unless File.exist?($volume_file) + v.customize ['createmedium', 'disk', '--filename', $volume_file, '--size', volume['size']] + end + v.customize ['storageattach', :id, '--storagectl', 'IDE Controller', '--port', 1, '--device', 0, '--type', 'hdd', '--medium', $volume_file] + end + end + end + nodeconfig.vm.provider 'libvirt' do |v| + v.memory = node['memory'] + v.cpus = node['cpus'] + nodeconfig.vm.provision 'file', source: './scripts', destination: '$HOME/' + nodeconfig.vm.provision 'shell' do |sh| + sh.path = "node.sh" + if node.has_key? "volumes" + $volume_mounts_dict = '' + node['volumes'].each do |volume| + $volume_mounts_dict += "#{volume['name']}=#{volume['mount']}," + $volume_file = "./#{node['name']}-#{volume['name']}.qcow2" + v.storage :file, :bus => 'sata', :device => volume['name'], :size => volume['size'] + end + sh.args = ['-v', $volume_mounts_dict[0...-1]] + end + end + end + end + end + + config.vm.define :installer, primary: true, autostart: false do |installer| + installer.vm.hostname = "multicloud" + installer.vm.network :private_network, :ip => "10.10.10.2", :type => :static + installer.vm.synced_folder '../', '/home/vagrant/multicloud-k8s/', type: sync_type + installer.vm.provision 'shell', privileged: false do |sh| + sh.env = {'KUD_PLUGIN_ENABLED': 'false', 'OVN_CENTRAL_INTERFACE': 'eth1'} + sh.inline = <<-SHELL + cp /home/vagrant/multicloud-k8s/sdewan-hub/insecure_keys/key.pub /home/vagrant/.ssh/id_rsa.pub + cp /home/vagrant/multicloud-k8s/sdewan-hub/insecure_keys/key /home/vagrant/.ssh/id_rsa + chown vagrant /home/vagrant/.ssh/id_rsa + chmod 400 /home/vagrant/.ssh/id_rsa + sudo apt install jq -y + cd /home/vagrant/multicloud-k8s/sdewan-hub/ && ./installer.sh | tee kud_installer.log + SHELL + end + end +end diff --git a/platform/test/e2e-test/sdewan-hub/config/default.yml b/platform/test/e2e-test/sdewan-hub/config/default.yml new file mode 100644 index 0000000..c63d548 --- /dev/null +++ b/platform/test/e2e-test/sdewan-hub/config/default.yml @@ -0,0 +1,25 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- name: "sdewan-hub" + ip: "10.10.10.5" + memory: 18432 + cpus: 16 + volumes: + - name: sdb + size: 50 + mount: /var/lib/docker/ + roles: + - kube-master + - etcd + - ovn-central + - kube-node + - ovn-controller + diff --git a/platform/test/e2e-test/sdewan-hub/insecure_keys/key b/platform/test/e2e-test/sdewan-hub/insecure_keys/key new file mode 100644 index 0000000..7d6a083 --- /dev/null +++ b/platform/test/e2e-test/sdewan-hub/insecure_keys/key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI +w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP +kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2 +hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO +Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW +yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd +ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1 +Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf +TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK +iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A +sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf +4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP +cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk +EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN +CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX +3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG +YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj +3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+ +dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz +6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC +P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF +llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ +kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH ++vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ +NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s= +-----END RSA PRIVATE KEY----- diff --git a/platform/test/e2e-test/sdewan-hub/insecure_keys/key.pub b/platform/test/e2e-test/sdewan-hub/insecure_keys/key.pub new file mode 100644 index 0000000..18a9c00 --- /dev/null +++ b/platform/test/e2e-test/sdewan-hub/insecure_keys/key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key diff --git a/platform/test/e2e-test/sdewan-hub/installer.sh b/platform/test/e2e-test/sdewan-hub/installer.sh new file mode 100755 index 0000000..5f509a7 --- /dev/null +++ b/platform/test/e2e-test/sdewan-hub/installer.sh @@ -0,0 +1,233 @@ +#!/bin/bash +#SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o errexit +set -o nounset +set -o pipefail + +INSTALLER_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" + +source ${INSTALLER_DIR}/../k8s/kud/tests/_functions.sh + +# _install_go() - Install GoLang package +function _install_go { + version=$(grep "go_version" ${kud_playbooks}/kud-vars.yml | awk -F "'" '{print $2}') + local tarball=go$version.linux-amd64.tar.gz + + #gcc is required for go apps compilation + if ! which gcc; then + sudo apt-get install -y gcc + fi + + if $(go version &>/dev/null); then + return + fi + + wget https://dl.google.com/go/$tarball + sudo tar -C /usr/local -xzf $tarball + rm $tarball + + export PATH=$PATH:/usr/local/go/bin + sudo sed -i "s|^PATH=.*|PATH=\"$PATH\"|" /etc/environment +} + +# _install_pip() - Install Python Package Manager +function _install_pip { + if $(pip --version &>/dev/null); then + sudo -E pip install --upgrade pip + else + sudo apt-get install -y python-dev + curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python + fi +} + +# _install_ansible() - Install and Configure Ansible program +function _install_ansible { + if $(ansible --version &>/dev/null); then + sudo pip uninstall -y ansible + fi + _install_pip + local version=$(grep "ansible_version" ${kud_playbooks}/kud-vars.yml | awk -F ': ' '{print $2}') + sudo mkdir -p /etc/ansible/ + sudo -E pip install ansible==$version +} + +# _install_docker() - Download and install docker-engine +function _install_docker { + local max_concurrent_downloads=${1:-3} + + if $(docker version &>/dev/null); then + return + fi + sudo apt-get install -y apt-transport-https ca-certificates curl + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + sudo apt-get update + sudo apt-get install -y docker-ce + + sudo mkdir -p /etc/systemd/system/docker.service.d + if [ ${http_proxy:-} ]; then + echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/http-proxy.conf + echo "Environment=\"HTTP_PROXY=$http_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/http-proxy.conf + fi + if [ ${https_proxy:-} ]; then + echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/https-proxy.conf + echo "Environment=\"HTTPS_PROXY=$https_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/https-proxy.conf + fi + if [ ${no_proxy:-} ]; then + echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/no-proxy.conf + echo "Environment=\"NO_PROXY=$no_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/no-proxy.conf + fi + sudo systemctl daemon-reload + echo "DOCKER_OPTS=\"-H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock --max-concurrent-downloads $max_concurrent_downloads \"" | sudo tee --append /etc/default/docker + if [[ -z $(groups | grep docker) ]]; then + sudo usermod -aG docker $USER + fi + + sudo systemctl restart docker + sleep 10 +} + +function _set_environment_file { + # By default ovn central interface is the first active network interface on localhost. If other wanted, need to export this variable in aio.sh or Vagrant file. + OVN_CENTRAL_INTERFACE="${OVN_CENTRAL_INTERFACE:-$(ip addr show | awk '/inet.*brd/{print $NF; exit}')}" + echo "export OVN_CENTRAL_INTERFACE=${OVN_CENTRAL_INTERFACE}" | sudo tee --append /etc/environment + echo "export OVN_CENTRAL_ADDRESS=$(get_ovn_central_address)" | sudo tee --append /etc/environment + echo "export KUBE_CONFIG_DIR=/opt/kubeconfig" | sudo tee --append /etc/environment + echo "export CSAR_DIR=/opt/csar" | sudo tee --append /etc/environment +} + +# install_k8s() - Install Kubernetes using kubespray tool +function install_k8s { + echo "Deploying kubernetes" + local dest_folder=/opt + version=$(grep "kubespray_version" ${kud_playbooks}/kud-vars.yml | awk -F ': ' '{print $2}') + local_release_dir=$(grep "local_release_dir" $kud_inventory_folder/group_vars/k8s-cluster.yml | awk -F "\"" '{print $2}') + local tarball=v$version.tar.gz + sudo apt-get install -y sshpass make unzip # install make to run mitogen target and unzip is mitogen playbook dependency + sudo apt-get install -y gnupg2 software-properties-common + _install_docker + _install_ansible + wget https://github.com/kubernetes-incubator/kubespray/archive/$tarball + sudo tar -C $dest_folder -xzf $tarball + sudo mv $dest_folder/kubespray-$version/ansible.cfg /etc/ansible/ansible.cfg + sudo chown -R $USER $dest_folder/kubespray-$version + sudo mkdir -p ${local_release_dir}/containers + rm $tarball + + pushd $dest_folder/kubespray-$version/ + sudo -E pip install -r ./requirements.txt + make mitogen + popd + rm -f $kud_inventory_folder/group_vars/all.yml 2> /dev/null + if [[ -n "${verbose:-}" ]]; then + echo "kube_log_level: 5" | tee $kud_inventory_folder/group_vars/all.yml + else + echo "kube_log_level: 2" | tee $kud_inventory_folder/group_vars/all.yml + fi + echo "kubeadm_enabled: true" | tee --append $kud_inventory_folder/group_vars/all.yml + if [[ -n "${http_proxy:-}" ]]; then + echo "http_proxy: \"$http_proxy\"" | tee --append $kud_inventory_folder/group_vars/all.yml + fi + if [[ -n "${https_proxy:-}" ]]; then + echo "https_proxy: \"$https_proxy\"" | tee --append $kud_inventory_folder/group_vars/all.yml + fi + ansible-playbook $verbose -i $kud_inventory $dest_folder/kubespray-$version/cluster.yml --become --become-user=root | sudo tee $log_folder/setup-kubernetes.log + + # Configure environment + mkdir -p $HOME/.kube + cp $kud_inventory_folder/artifacts/admin.conf $HOME/.kube/config + # Copy Kubespray kubectl to be usable in host running Ansible. Requires kubectl_localhost: true in inventory/group_vars/k8s-cluster.yml + sudo cp $kud_inventory_folder/artifacts/kubectl /usr/local/bin/ +} + +# install_addons() - Install Kubenertes AddOns +function install_addons { + source /etc/environment + echo "Installing Kubernetes AddOns" + _install_ansible + sudo ansible-galaxy install $verbose -r $kud_infra_folder/galaxy-requirements.yml --ignore-errors + ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-kud.yml | sudo tee $log_folder/setup-kud.log + for addon in ${KUD_ADDONS:-ovn4nfv}; do + echo "Deploying $addon using configure-$addon.yml playbook.." + ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-${addon}.yml | sudo tee $log_folder/setup-${addon}.log + done + echo "Run the test cases if testing_enabled is set to true." + if [[ "${testing_enabled}" == "true" ]]; then + for addon in ${KUD_ADDONS:-ovn4nfv}; do + pushd $kud_tests + bash ${addon}.sh + popd + done + fi + echo "Add-ons deployment complete..." +} + +# _print_kubernetes_info() - Prints the login Kubernetes information +function _print_kubernetes_info { + if ! $(kubectl version &>/dev/null); then + return + fi + # Expose Dashboard using NodePort + node_port=30080 + KUBE_EDITOR="sed -i \"s|type\: ClusterIP|type\: NodePort|g\"" kubectl -n kube-system edit service kubernetes-dashboard + KUBE_EDITOR="sed -i \"s|nodePort\: .*|nodePort\: $node_port|g\"" kubectl -n kube-system edit service kubernetes-dashboard + + master_ip=$(kubectl cluster-info | grep "Kubernetes master" | awk -F ":" '{print $2}') + + printf "Kubernetes Info\n===============\n" > $k8s_info_file + echo "Dashboard URL: https:$master_ip:$node_port" >> $k8s_info_file + echo "Admin user: kube" >> $k8s_info_file + echo "Admin password: secret" >> $k8s_info_file +} + +sudo -k # forgot sudo password +if ! sudo -n "true"; then + echo "" + echo "passwordless sudo is needed for '$(id -nu)' user." + echo "Please fix your /etc/sudoers file. You likely want an" + echo "entry like the following one..." + echo "" + echo "$(id -nu) ALL=(ALL) NOPASSWD: ALL" + exit 1 +fi + +verbose="" +if [[ -n "${KUD_DEBUG:-}" ]]; then + set -o xtrace + verbose="-vvv" +fi + +# Configuration values +log_folder=/var/log/kud +kud_folder=${INSTALLER_DIR}/../k8s/kud/hosting_providers/vagrant +kud_infra_folder=$kud_folder/../../deployment_infra +export kud_inventory_folder=$kud_folder/../../../../sdewan-hub/inventory +kud_inventory=$kud_inventory_folder/hosts.ini +kud_playbooks=$kud_infra_folder/playbooks +kud_tests=$kud_folder/../../tests +k8s_info_file=$kud_folder/k8s_info.log +testing_enabled=${KUD_ENABLE_TESTS:-false} +sudo mkdir -p $log_folder +sudo mkdir -p /opt/csar +sudo chown -R $USER /opt/csar +# Install dependencies +# Setup proxy variables +if [ -f $kud_folder/sources.list ]; then + sudo mv /etc/apt/sources.list /etc/apt/sources.list.backup + sudo cp $kud_folder/sources.list /etc/apt/sources.list +fi +echo "Removing ppa for jonathonf/python-3.6" +sudo ls /etc/apt/sources.list.d/ || true +sudo find /etc/apt/sources.list.d -maxdepth 1 -name '*jonathonf*' -delete || true +sudo apt-get update +install_k8s +_set_environment_file +install_addons diff --git a/platform/test/e2e-test/sdewan-hub/inventory/group_vars/all.yml b/platform/test/e2e-test/sdewan-hub/inventory/group_vars/all.yml new file mode 100644 index 0000000..528430c --- /dev/null +++ b/platform/test/e2e-test/sdewan-hub/inventory/group_vars/all.yml @@ -0,0 +1,2 @@ +kube_log_level: 2 +kubeadm_enabled: true diff --git a/platform/test/e2e-test/sdewan-hub/inventory/group_vars/k8s-cluster.yml b/platform/test/e2e-test/sdewan-hub/inventory/group_vars/k8s-cluster.yml new file mode 100644 index 0000000..ab91498 --- /dev/null +++ b/platform/test/e2e-test/sdewan-hub/inventory/group_vars/k8s-cluster.yml @@ -0,0 +1,82 @@ +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Kubernetes configuration dirs and system namespace. +# Those are where all the additional config stuff goes +# kubernetes normally puts in /srv/kubernetes. +# This puts them in a sane location and namespace. +# Editing those values will almost surely break something. +system_namespace: kube-system + +# Logging directory (sysvinit systems) +kube_log_dir: "/var/log/kubernetes" + +kube_api_anonymous_auth: true + +# Users to create for basic auth in Kubernetes API via HTTP +# Optionally add groups for user +kube_api_pwd: "secret" +kube_users: + kube: + pass: "{{kube_api_pwd}}" + role: admin + groups: + - system:masters + +## It is possible to activate / deactivate selected authentication methods (basic auth, static token auth) +#kube_oidc_auth: false +kube_basic_auth: true +kube_token_auth: true + +# Choose network plugin (calico, contiv, weave or flannel) +# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing +kube_network_plugin: flannel + +# Make a copy of kubeconfig (admin.conf) on the host that runs Ansible to inventory/artifacts +kubeconfig_localhost: true +# Copy kubectl binary on the host that runs Ansible to inventory/artifacts +kubectl_localhost: true +# Disable nodelocal dns cache +enable_nodelocaldns: false +# Enable MountPropagation gate feature +local_volumes_enabled: true +local_volume_provisioner_enabled: true + +## Change this to use another Kubernetes version, e.g. a current beta release +kube_version: v1.14.3 + +# Helm deployment +helm_enabled: true + +# Kube-proxy proxyMode configuration. +# NOTE: Ipvs is based on netfilter hook function, but uses hash table as the underlying data structure and +# works in the kernel space +# https://kubernetes.io/docs/concepts/services-networking/service/#proxy-mode-ipvs +#kube_proxy_mode: ipvs + +# Download container images only once then push to cluster nodes in batches +download_run_once: true + +# Where the binaries will be downloaded. +# Note: ensure that you've enough disk space (about 1G) +local_release_dir: "/tmp/releases" + +# Makes the installer node a delegate for pushing images while running +# the deployment with ansible. This maybe the case if cluster nodes +# cannot access each over via ssh or you want to use local docker +# images as a cache for multiple clusters. +download_localhost: true + +# Subnet for cluster IPs +kube_service_addresses: 10.244.0.0/18 +# Subnet for Pod IPs +kube_pods_subnet: 10.244.64.0/18 + +# Add specific docker version +docker_version: 'latest' diff --git a/platform/test/e2e-test/sdewan-hub/node.sh b/platform/test/e2e-test/sdewan-hub/node.sh new file mode 100755 index 0000000..a51be19 --- /dev/null +++ b/platform/test/e2e-test/sdewan-hub/node.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o nounset +set -o pipefail + +# usage() - Prints the usage of the program +function usage { + cat <> /etc/fstab +} + +while getopts "h?v:" opt; do + case $opt in + v) + dict_volumes="$OPTARG" + ;; + h|\?) + usage + exit + ;; + esac +done + +swapoff -a +if [[ -n "${dict_volumes+x}" ]]; then + for kv in ${dict_volumes//,/ } ;do + mount_external_partition ${kv%=*} ${kv#*=} + done +fi + +vendor_id=$(lscpu|grep "Vendor ID") +if [[ $vendor_id == *GenuineIntel* ]]; then + kvm_ok=$(cat /sys/module/kvm_intel/parameters/nested) + if [[ $kvm_ok == 'N' ]]; then + echo "Enable Intel Nested-Virtualization" + rmmod kvm-intel + echo 'options kvm-intel nested=y' >> /etc/modprobe.d/dist.conf + modprobe kvm-intel + echo kvm-intel >> /etc/modules + fi +else + kvm_ok=$(cat /sys/module/kvm_amd/parameters/nested) + if [[ $kvm_ok == '0' ]]; then + echo "Enable AMD Nested-Virtualization" + rmmod kvm-amd + sh -c "echo 'options kvm-amd nested=1' >> /etc/modprobe.d/dist.conf" + modprobe kvm-amd + echo kvm-amd >> /etc/modules + fi +fi +modprobe vhost_net +echo vhost_net >> /etc/modules +source /etc/os-release || source /usr/lib/os-release +case ${ID,,} in + *suse) + ;; + ubuntu|debian) + apt-get install -y cpu-checker + kvm-ok + ;; + rhel|centos|fedora) + ;; +esac diff --git a/platform/test/e2e-test/sdewan-hub/scripts/sdwan_verifier.sh b/platform/test/e2e-test/sdewan-hub/scripts/sdwan_verifier.sh new file mode 100755 index 0000000..c52a8b4 --- /dev/null +++ b/platform/test/e2e-test/sdewan-hub/scripts/sdwan_verifier.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o errexit +set -o nounset +set -o pipefail + + +function login { + login_url=http://$1/cgi-bin/luci/ + echo $(wget -S --spider --post-data "luci_username=root&luci_password=" $login_url 2>&1 | grep sysauth= | sed -r 's/.*sysauth=([^;]+);.*/\1/') +} + +function call_get { + url=$1 + token=$2 + echo $(curl -s -X "GET" -H "Cookie:sysauth=$token" -H "Content-Type: application/json" $url) +} + +function call_post { + url=$1 + token=$2 + payload=$3 + echo $(curl -s -X "POST" -H "Cookie:sysauth=$token" -H "Content-Type: application/json" -d "$payload" $url) +} + +function call_put { + url=$1 + token=$2 + payload=$3 + echo $(curl -s -X "PUT" -H "Cookie:sysauth=$token" -H "Content-Type: application/json" -d "$payload" $url) +} + +function call_delete { + url=$1 + token=$2 + echo $(curl -s -X "DELETE" -H "Cookie:sysauth=$token" -H "Content-Type: application/json" $url) +} + +function wait_for_pod { + status_phase="" + while [[ "$status_phase" != "Running" ]]; do + new_phase="$(kubectl get pods -o wide | grep ^$1 | awk '{print $3}')" + if [[ "$new_phase" != "$status_phase" ]]; then + status_phase="$new_phase" + fi + if [[ "$new_phase" == "Err"* ]]; then + exit 1 + fi + done +} + +sdwan_pod_ip=$(kubectl get pods -o wide | grep $sdewan_cnf_name | awk '{print $6}') + +echo "SDWAN pod ip:"$sdwan_pod_ip + +echo "Login to sdwan ..." +security_token="" +while [[ "$security_token" == "" ]]; do + echo "Get Security Token ..." + security_token=$(login $sdwan_pod_ip) + sleep 2 +done +echo "* Security Token: "$security_token + +echo "Set ipsec proposals ..." +proposal=" +{ + \"name\":\"my_test_proposal1\", + \"encryption_algorithm\":\"aes128\", + \"hash_algorithm\":\"sha256\", + \"dh_group\":\"modp3072\" +} +" +echo $(call_post http://$sdwan_pod_ip/cgi-bin/luci/sdewan/ipsec/v1/proposals $security_token "$proposal") +sleep 3 +echo "Proposals inserted..." +echo $(call_get http://$sdwan_pod_ip/cgi-bin/luci/sdewan/ipsec/v1/proposals $security_token) + +rule=" +{ + \"name\":\"Hub\", + \"gateway\":\"%any\", + \"pre_shared_key\":\"test_key\", + \"authentication_method\":\"psk\", + \"local_identifier\":\"$hubIp\", + \"crypto_proposal\":[\"my_test_proposal1\"], + \"force_crypto_proposal\":\"0\", + \"connections\":[{ + \"name\":\"connectionA\", + \"type\":\"tunnel\", + \"mode\":\"start\", + \"remote_sourceip\":\"192.168.1.5-192.168.1.6\", + \"local_subnet\":\"192.168.1.1/24,$hubIp/32\", + \"crypto_proposal\":[\"my_test_proposal1\"] + }] +} +" +echo $(call_post http://$sdwan_pod_ip/cgi-bin/luci/sdewan/ipsec/v1/sites $security_token "$rule") +sleep 3 +echo "IPSec configs inserted..." +echo $(call_get http://$sdwan_pod_ip/cgi-bin/luci/sdewan/ipsec/v1/sites $security_token) + +echo "Restarting IPSec service..." +operation=" +{ + \"action\":\"restart\" +} +" +echo $(call_put http://$sdwan_pod_ip/cgi-bin/luci/sdewan/v1/service/ipsec $security_token "$operation") +sleep 2 + +echo "Test Completed!" diff --git a/platform/test/e2e-test/sdewan-hub/scripts/setup-cnf.sh b/platform/test/e2e-test/sdewan-hub/scripts/setup-cnf.sh new file mode 100755 index 0000000..96a66e5 --- /dev/null +++ b/platform/test/e2e-test/sdewan-hub/scripts/setup-cnf.sh @@ -0,0 +1,170 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +set -o errexit +set -o nounset +set -o pipefail + +echo "Setup CNF for sdewan hub" +test -f ./variables +. variables +providerSubnet=${providerSubnet} +providerGateway=${providerGateway} +providerExcludeIps=${providerExcludeIps} +providerNetworkInterface=${providerNetworkInterface} +cnfWanGateway=${cnfWanGateway} + +clean() +{ +echo "Cleaning ..." +kubectl delete -f sdewan-cnf.yaml +kubectl delete -f network-prepare.yaml +} + +error_detect() +{ + echo "Error on line $1" + clean +} + +trap 'error_detect $LINENO' ERR + +echo "Creating ovn networks..." +cat > network-prepare.yaml << EOF +--- +apiVersion: k8s.plugin.opnfv.org/v1alpha1 +kind: ProviderNetwork +metadata: + name: pnetwork +spec: + cniType: ovn4nfv + ipv4Subnets: + - subnet: $providerSubnet + name: subnet + gateway: $providerGateway + excludeIps: $providerExcludeIps + providerNetType: DIRECT + direct: + providerInterfaceName: $providerNetworkInterface + directNodeSelector: all + +EOF + +kubectl create -f network-prepare.yaml +sleep 2 + +ovnProviderNet=$(kubectl get providernetwork | sed -n 2p | awk '{print $1}') +if [ -n "${ovnProviderNet}" ] +then + echo "Network created successfully" +else + echo "Network creation failed" + exit 1 +fi + + +echo "Creating sdwan-cnf ..." +cat > sdewan-cnf.yaml << EOF +apiVersion: v1 +kind: ConfigMap +metadata: + name: sdwan-config +data: + entrypoint.sh: | + #!/bin/bash + # Always exit on errors. + set -e + + cat > /etc/config/network << EOF + config interface 'wan0' + option ifname 'net1' + option proto 'static' + option ipaddr '$cnfWan0' + option netmask '255.255.255.0' + EOF + + iptables -t nat -L + /sbin/procd & + /sbin/ubusd & + sleep 1 + /etc/init.d/rpcd start + /etc/init.d/dnsmasq start + /etc/init.d/network start + /etc/init.d/odhcpd start + /etc/init.d/uhttpd start + /etc/init.d/log start + /etc/init.d/dropbear start + /etc/init.d/mwan3 restart + + echo "Entering sleep... (success)" + + # Sleep forever. + while true; do sleep 100; done + +--- +apiVersion: v1 +kind: Pod +metadata: + name: $sdewan_cnf_name + annotations: + k8s.v1.cni.cncf.io/networks: '[{ "name": "ovn-networkobj"}]' + k8s.plugin.opnfv.org/nfn-network: '{ "type": "ovn4nfv", "interface": [{"name": "pnetwork", "interface": "net1", "ipAddress": "$cnfWan0", "gateway": "$cnfWanGateway"}]}' +spec: + containers: + - name: sdwan-pod-net + image: integratedcloudnative/openwrt:test + ports: + - containerPort: 22 + - containerPort: 80 + command: + - /bin/sh + - /init/entrypoint.sh + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + volumeMounts: + - name: entrypoint-sh + mountPath: /init + volumes: + - name: entrypoint-sh + configMap: + name: sdwan-config + items: + - key: entrypoint.sh + path: entrypoint.sh +EOF + +kubectl create -f sdewan-cnf.yaml +sleep 10 + +sdwan_status=$(kubectl get po | grep $sdewan_cnf_name | awk '{print $3}') +if [ "$sdwan_status" == "Running" ] +then + echo "Sdewan cnf $sdewan_cnf_name created successfully" +else + sleep 50 + sdwan_status=$(kubectl get po | grep $sdewan_cnf_name | awk '{print $3}') + if [ "$sdwan_status" != "Running" ] + then + echo "Sdewan cnf $sdewan_cnf_name creation failed" + exit 2 + fi +fi + +kubectl exec -it $sdewan_cnf_name /etc/init.d/network restart + + +echo "Setup openwrt configurations" +bash sdwan_verifier.sh + + +#clean + diff --git a/platform/test/e2e-test/sdewan-hub/scripts/variables b/platform/test/e2e-test/sdewan-hub/scripts/variables new file mode 100644 index 0000000..a143317 --- /dev/null +++ b/platform/test/e2e-test/sdewan-hub/scripts/variables @@ -0,0 +1,9 @@ +providerSubnet=10.10.10.1/24 +providerGateway=10.10.10.1/24 +providerExcludeIps=10.10.10.2..10.10.10.9 +providerNetworkInterface=eth1 +cnfWanGateway=10.10.10.1 +cnfWan0=10.10.10.35 + +export hubIp=10.10.10.35 +export sdewan_cnf_name=sdewan-cnf-hub diff --git a/platform/test/e2e-test/setup.sh b/platform/test/e2e-test/setup.sh new file mode 100755 index 0000000..9c65ccd --- /dev/null +++ b/platform/test/e2e-test/setup.sh @@ -0,0 +1,201 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o nounset +set -o pipefail + +vagrant_version=2.2.4 +if ! vagrant version &>/dev/null; then + enable_vagrant_install=true +else + if [[ "$vagrant_version" != "$(vagrant version | awk 'NR==1{print $3}')" ]]; then + enable_vagrant_install=true + fi +fi + +function usage { + cat < +Installation of vagrant and its dependencies in Linux OS + +Argument: + -p Vagrant provider +EOF +} + +while getopts ":p:" OPTION; do + case $OPTION in + p) + provider=$OPTARG + ;; + \?) + usage + exit 1 + ;; + esac +done +if [[ -z "${provider+x}" ]]; then + usage + exit 1 +fi + +case $provider in + "virtualbox" | "libvirt" ) + export VAGRANT_DEFAULT_PROVIDER=${provider} + ;; + * ) + usage + exit 1 +esac +source /etc/os-release || source /usr/lib/os-release + +libvirt_group="libvirt" +packages=() +case ${ID,,} in + *suse) + INSTALLER_CMD="sudo -H -E zypper -q install -y --no-recommends" + packages+=(python-devel) + + # Vagrant installation + if [[ "${enable_vagrant_install+x}" ]]; then + vagrant_pgp="pgp_keys.asc" + wget -q https://keybase.io/hashicorp/$vagrant_pgp + wget -q https://releases.hashicorp.com/vagrant/$vagrant_version/vagrant_${vagrant_version}_x86_64.rpm + gpg --quiet --with-fingerprint $vagrant_pgp + sudo rpm --import $vagrant_pgp + sudo rpm --checksig vagrant_${vagrant_version}_x86_64.rpm + sudo rpm --install vagrant_${vagrant_version}_x86_64.rpm + rm vagrant_${vagrant_version}_x86_64.rpm + rm $vagrant_pgp + fi + + case $VAGRANT_DEFAULT_PROVIDER in + virtualbox) + wget -q "http://download.virtualbox.org/virtualbox/rpm/opensuse/$VERSION/virtualbox.repo" -P /etc/zypp/repos.d/ + $INSTALLER_CMD --enablerepo=epel dkms + wget -q https://www.virtualbox.org/download/oracle_vbox.asc -O- | rpm --import - + packages+=(VirtualBox-5.1) + ;; + libvirt) + # vagrant-libvirt dependencies + packages+=(qemu libvirt libvirt-devel ruby-devel gcc qemu-kvm zlib-devel libxml2-devel libxslt-devel make) + # NFS + packages+=(nfs-kernel-server) + ;; + esac + sudo zypper -n ref + ;; + + ubuntu|debian) + libvirt_group="libvirtd" + INSTALLER_CMD="sudo -H -E apt-get -y -q=3 install" + packages+=(python-dev) + + # Vagrant installation + if [[ "${enable_vagrant_install+x}" ]]; then + wget -q https://releases.hashicorp.com/vagrant/$vagrant_version/vagrant_${vagrant_version}_x86_64.deb + sudo dpkg -i vagrant_${vagrant_version}_x86_64.deb + rm vagrant_${vagrant_version}_x86_64.deb + fi + + case $VAGRANT_DEFAULT_PROVIDER in + virtualbox) + echo "deb http://download.virtualbox.org/virtualbox/debian trusty contrib" >> /etc/apt/sources.list + wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add - + wget -q https://www.virtualbox.org/download/oracle_vbox.asc -O- | sudo apt-key add - + packages+=(virtualbox-5.1 dkms) + ;; + libvirt) + # vagrant-libvirt dependencies + packages+=(qemu libvirt-bin ebtables dnsmasq libxslt-dev libxml2-dev libvirt-dev zlib1g-dev ruby-dev cpu-checker) + # NFS + packages+=(nfs-kernel-server) + ;; + esac + sudo apt-get update + ;; + + rhel|centos|fedora) + PKG_MANAGER=$(which dnf || which yum) + sudo "$PKG_MANAGER" updateinfo + INSTALLER_CMD="sudo -H -E ${PKG_MANAGER} -q -y install" + packages+=(python-devel) + + # Vagrant installation + if [[ "${enable_vagrant_install+x}" ]]; then + wget -q https://releases.hashicorp.com/vagrant/$vagrant_version/vagrant_${vagrant_version}_x86_64.rpm + $INSTALLER_CMD vagrant_${vagrant_version}_x86_64.rpm + rm vagrant_${vagrant_version}_x86_64.rpm + fi + + case $VAGRANT_DEFAULT_PROVIDER in + virtualbox) + wget -q http://download.virtualbox.org/virtualbox/rpm/rhel/virtualbox.repo -P /etc/yum.repos.d + $INSTALLER_CMD --enablerepo=epel dkms + wget -q https://www.virtualbox.org/download/oracle_vbox.asc -O- | rpm --import - + packages+=(VirtualBox-5.1) + ;; + libvirt) + # vagrant-libvirt dependencies + packages+=(qemu libvirt libvirt-devel ruby-devel gcc qemu-kvm) + # NFS + packages+=(nfs-utils nfs-utils-lib) + ;; + esac + ;; + +esac + +# Enable Nested-Virtualization +vendor_id=$(lscpu|grep "Vendor ID") +if [[ $vendor_id == *GenuineIntel* ]]; then + kvm_ok=$(cat /sys/module/kvm_intel/parameters/nested) + if [[ $kvm_ok == 'N' ]]; then + echo "Enable Intel Nested-Virtualization" + sudo rmmod kvm-intel + echo 'options kvm-intel nested=y' | sudo tee --append /etc/modprobe.d/dist.conf + sudo modprobe kvm-intel + fi +else + kvm_ok=$(cat /sys/module/kvm_amd/parameters/nested) + if [[ $kvm_ok == '0' ]]; then + echo "Enable AMD Nested-Virtualization" + sudo rmmod kvm-amd + echo 'options kvm-amd nested=1' | sudo tee --append /etc/modprobe.d/dist.conf + sudo modprobe kvm-amd + fi +fi +sudo modprobe vhost_net + +${INSTALLER_CMD} "${packages[@]}" +if ! which pip; then + curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python +else + sudo -H -E pip install --upgrade pip +fi +sudo -H -E pip install tox +if [[ ${http_proxy+x} ]]; then + vagrant plugin install vagrant-proxyconf +fi +if [ "$VAGRANT_DEFAULT_PROVIDER" == libvirt ]; then + vagrant plugin install vagrant-libvirt + sudo usermod -a -G $libvirt_group "$USER" # This might require to reload user's group assigments + sudo systemctl restart libvirtd + + # Start statd service to prevent NFS lock errors + sudo systemctl enable rpc-statd + sudo systemctl start rpc-statd + + case ${ID,,} in + ubuntu|debian) + kvm-ok + ;; + esac +fi diff --git a/platform/test/e2e-test/test.sh b/platform/test/e2e-test/test.sh new file mode 100755 index 0000000..788f1a9 --- /dev/null +++ b/platform/test/e2e-test/test.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +set -o errexit +set -o nounset +set -o pipefail + + +BASE=$(pwd) + + +clean() +{ +# Cleaning the env +echo "Cleaning the environment..." +echo "Deleting the vms..." +for name in edge-a edge-b sdewan-hub +do + cd $BASE/${name} + vagrant destroy -f & + sleep 10 +done +echo "Cleaning completed" +} + +error_report() +{ + echo "Error on line $1" + echo "End-to-end test failed" +} + +trap 'error_report $LINENO' ERR +trap 'clean' EXIT + +# Install dependencies +echo "Installing dependencies..." +sudo ./setup.sh -p libvirt + +# Bring up 3 vms for edge-a, edge-b and sdewan-hub +cd $BASE +git clone http://gerrit.onap.org/r/multicloud/k8s && cd k8s +echo "Bringing up virtual machines for three clusters..." +for name in edge-a edge-b sdewan-hub +do + cd $BASE/${name} + echo "Start up cluster for ${name}..." + vagrant up && vagrant up installer + sleep 40 +done + + +# Checking vm status... +for name in edge-a edge-b sdewan-hub +do + cd $BASE/${name} + vagrant ssh ${name} -- -t 'mkdir -p /home/vagrant/.kube; sudo cp -i /etc/kubernetes/admin.conf /home/vagrant/.kube/config; sudo chown $(id -u):$(id -g) $HOME/.kube/config' + Status=$(vagrant ssh ${name} -- -t 'kubectl get po -n operator | grep 'nfn-agent'' | grep 'nfn-agent' | awk '{print $3}') + if [ $Status != "Running" ] + then + echo "Virtual machine ${name} provision failed" + exit 1 + else + echo "Virtual machine ${name} provision success" + fi +done + + +# Setup ipsec tunnels and applications +echo "Setup configs for the e2e scenario..." +for name in sdewan-hub edge-a edge-b +do + cd $BASE/${name} + vagrant ssh ${name} -- -t 'cd /home/vagrant/scripts; ./setup-cnf.sh' & + sleep 2m +done + + +echo "Testing the connectivity between applications..." +cd $BASE/edge-a +vagrant ssh edge-a -- -t './scripts/test-connection.sh' +sleep 3