From d544b900ae774bf481a7b1afb76c9d56d83f3bd1 Mon Sep 17 00:00:00 2001 From: Ruoyu Date: Sun, 28 Jun 2020 13:22:40 +0800 Subject: [PATCH] Support e2e test thru CRs * Installing the CNF and the controller with helm charts * Applying the firewall and IPsec configs thru CRs * Adding default policies to enable remote access to api server and Istio ingress Issue-ID: ICN-390 Change-Id: I7c5ca03829ad1a7c3c90bc4edb5921ec60d4e530 Signed-off-by: Ruoyu --- platform/cnf/src/Dockerfile_1806_mwan3.tpl | 2 + platform/cnf/src/Dockerfile_1806_mwan3_noproxy.tpl | 2 + platform/cnf/src/default_firewall | 11 + platform/cnf/src/sdewan.user | 25 + platform/test/e2e-test-crd/README.md | 30 + platform/test/e2e-test-crd/edge-a/Vagrantfile | 151 +++ .../test/e2e-test-crd/edge-a/config/default.yml | 24 + .../test/e2e-test-crd/edge-a/insecure_keys/key | 27 + .../test/e2e-test-crd/edge-a/insecure_keys/key.pub | 1 + platform/test/e2e-test-crd/edge-a/installer.sh | 233 +++++ .../edge-a/inventory/group_vars/all.yml | 2 + .../edge-a/inventory/group_vars/k8s-cluster.yml | 82 ++ platform/test/e2e-test-crd/edge-a/node.sh | 88 ++ .../edge-a/scripts/edge-a-sdwan-config | 22 + .../e2e-test-crd/edge-a/scripts/test-connection.sh | 44 + .../test/e2e-test-crd/edge-a/scripts/variables | 19 + platform/test/e2e-test-crd/edge-b/Vagrantfile | 151 +++ .../test/e2e-test-crd/edge-b/config/default.yml | 24 + .../test/e2e-test-crd/edge-b/insecure_keys/key | 27 + .../test/e2e-test-crd/edge-b/insecure_keys/key.pub | 1 + platform/test/e2e-test-crd/edge-b/installer.sh | 233 +++++ .../edge-b/inventory/group_vars/all.yml | 2 + .../edge-b/inventory/group_vars/k8s-cluster.yml | 82 ++ platform/test/e2e-test-crd/edge-b/node.sh | 88 ++ .../edge-b/scripts/edge-b-sdwan-config | 21 + .../test/e2e-test-crd/edge-b/scripts/variables | 19 + .../edge-scripts/helm-tmp/cnf/.helmignore | 23 + .../edge-scripts/helm-tmp/cnf/Chart.yaml | 21 + .../helm-tmp/cnf/templates/_helpers.tpl | 63 ++ .../edge-scripts/helm-tmp/cnf/templates/cm.yaml | 77 ++ .../helm-tmp/cnf/templates/deployment.yaml | 94 ++ .../edge-scripts/helm-tmp/cnf/values.yaml | 37 + .../edge-scripts/helm-tmp/controllers/.helmignore | 23 + .../edge-scripts/helm-tmp/controllers/Chart.yaml | 21 + .../helm-tmp/controllers/templates/_helpers.tpl | 63 ++ .../controllers/templates/certificate.yaml | 29 + .../helm-tmp/controllers/templates/crd.yaml | 1017 ++++++++++++++++++++ .../helm-tmp/controllers/templates/deployment.yaml | 72 ++ .../helm-tmp/controllers/templates/issuer.yaml | 24 + .../helm-tmp/controllers/templates/namespace.yaml | 21 + .../helm-tmp/controllers/templates/role.yaml | 342 +++++++ .../helm-tmp/controllers/templates/service.yaml | 42 + .../helm-tmp/controllers/templates/webhook.yaml | 80 ++ .../edge-scripts/helm-tmp/controllers/values.yaml | 19 + .../test/e2e-test-crd/edge-scripts/setup-cnf.sh | 299 ++++++ platform/test/e2e-test-crd/sdewan-hub/Vagrantfile | 150 +++ .../e2e-test-crd/sdewan-hub/config/default.yml | 25 + .../test/e2e-test-crd/sdewan-hub/insecure_keys/key | 27 + .../e2e-test-crd/sdewan-hub/insecure_keys/key.pub | 1 + platform/test/e2e-test-crd/sdewan-hub/installer.sh | 233 +++++ .../sdewan-hub/inventory/group_vars/all.yml | 2 + .../inventory/group_vars/k8s-cluster.yml | 82 ++ platform/test/e2e-test-crd/sdewan-hub/node.sh | 88 ++ .../sdewan-hub/scripts/cnf/.helmignore | 23 + .../e2e-test-crd/sdewan-hub/scripts/cnf/Chart.yaml | 21 + .../sdewan-hub/scripts/cnf/templates/_helpers.tpl | 63 ++ .../sdewan-hub/scripts/cnf/templates/cm.yaml | 77 ++ .../scripts/cnf/templates/deployment.yaml | 94 ++ .../sdewan-hub/scripts/cnf/values.yaml | 32 + .../e2e-test-crd/sdewan-hub/scripts/setup-cnf.sh | 162 ++++ .../test/e2e-test-crd/sdewan-hub/scripts/variables | 9 + platform/test/e2e-test-crd/setup.sh | 201 ++++ platform/test/e2e-test-crd/test.sh | 87 ++ 63 files changed, 5155 insertions(+) create mode 100644 platform/cnf/src/default_firewall create mode 100644 platform/cnf/src/sdewan.user create mode 100644 platform/test/e2e-test-crd/README.md create mode 100644 platform/test/e2e-test-crd/edge-a/Vagrantfile create mode 100644 platform/test/e2e-test-crd/edge-a/config/default.yml create mode 100644 platform/test/e2e-test-crd/edge-a/insecure_keys/key create mode 100644 platform/test/e2e-test-crd/edge-a/insecure_keys/key.pub create mode 100755 platform/test/e2e-test-crd/edge-a/installer.sh create mode 100644 platform/test/e2e-test-crd/edge-a/inventory/group_vars/all.yml create mode 100644 platform/test/e2e-test-crd/edge-a/inventory/group_vars/k8s-cluster.yml create mode 100755 platform/test/e2e-test-crd/edge-a/node.sh create mode 100644 platform/test/e2e-test-crd/edge-a/scripts/edge-a-sdwan-config create mode 100755 platform/test/e2e-test-crd/edge-a/scripts/test-connection.sh create mode 100644 platform/test/e2e-test-crd/edge-a/scripts/variables create mode 100644 platform/test/e2e-test-crd/edge-b/Vagrantfile create mode 100644 platform/test/e2e-test-crd/edge-b/config/default.yml create mode 100644 platform/test/e2e-test-crd/edge-b/insecure_keys/key create mode 100644 platform/test/e2e-test-crd/edge-b/insecure_keys/key.pub create mode 100755 platform/test/e2e-test-crd/edge-b/installer.sh create mode 100644 platform/test/e2e-test-crd/edge-b/inventory/group_vars/all.yml create mode 100644 platform/test/e2e-test-crd/edge-b/inventory/group_vars/k8s-cluster.yml create mode 100755 platform/test/e2e-test-crd/edge-b/node.sh create mode 100644 platform/test/e2e-test-crd/edge-b/scripts/edge-b-sdwan-config create mode 100644 platform/test/e2e-test-crd/edge-b/scripts/variables create mode 100644 platform/test/e2e-test-crd/edge-scripts/helm-tmp/cnf/.helmignore create mode 100644 platform/test/e2e-test-crd/edge-scripts/helm-tmp/cnf/Chart.yaml create mode 100644 platform/test/e2e-test-crd/edge-scripts/helm-tmp/cnf/templates/_helpers.tpl create mode 100644 platform/test/e2e-test-crd/edge-scripts/helm-tmp/cnf/templates/cm.yaml create mode 100644 platform/test/e2e-test-crd/edge-scripts/helm-tmp/cnf/templates/deployment.yaml create mode 100644 platform/test/e2e-test-crd/edge-scripts/helm-tmp/cnf/values.yaml create mode 100644 platform/test/e2e-test-crd/edge-scripts/helm-tmp/controllers/.helmignore create mode 100644 platform/test/e2e-test-crd/edge-scripts/helm-tmp/controllers/Chart.yaml create mode 100644 platform/test/e2e-test-crd/edge-scripts/helm-tmp/controllers/templates/_helpers.tpl create mode 100644 platform/test/e2e-test-crd/edge-scripts/helm-tmp/controllers/templates/certificate.yaml create mode 100644 platform/test/e2e-test-crd/edge-scripts/helm-tmp/controllers/templates/crd.yaml create mode 100644 platform/test/e2e-test-crd/edge-scripts/helm-tmp/controllers/templates/deployment.yaml create mode 100644 platform/test/e2e-test-crd/edge-scripts/helm-tmp/controllers/templates/issuer.yaml create mode 100644 platform/test/e2e-test-crd/edge-scripts/helm-tmp/controllers/templates/namespace.yaml create mode 100644 platform/test/e2e-test-crd/edge-scripts/helm-tmp/controllers/templates/role.yaml create mode 100644 platform/test/e2e-test-crd/edge-scripts/helm-tmp/controllers/templates/service.yaml create mode 100644 platform/test/e2e-test-crd/edge-scripts/helm-tmp/controllers/templates/webhook.yaml create mode 100644 platform/test/e2e-test-crd/edge-scripts/helm-tmp/controllers/values.yaml create mode 100755 platform/test/e2e-test-crd/edge-scripts/setup-cnf.sh create mode 100644 platform/test/e2e-test-crd/sdewan-hub/Vagrantfile create mode 100644 platform/test/e2e-test-crd/sdewan-hub/config/default.yml create mode 100644 platform/test/e2e-test-crd/sdewan-hub/insecure_keys/key create mode 100644 platform/test/e2e-test-crd/sdewan-hub/insecure_keys/key.pub create mode 100755 platform/test/e2e-test-crd/sdewan-hub/installer.sh create mode 100644 platform/test/e2e-test-crd/sdewan-hub/inventory/group_vars/all.yml create mode 100644 platform/test/e2e-test-crd/sdewan-hub/inventory/group_vars/k8s-cluster.yml create mode 100755 platform/test/e2e-test-crd/sdewan-hub/node.sh create mode 100644 platform/test/e2e-test-crd/sdewan-hub/scripts/cnf/.helmignore create mode 100644 platform/test/e2e-test-crd/sdewan-hub/scripts/cnf/Chart.yaml create mode 100644 platform/test/e2e-test-crd/sdewan-hub/scripts/cnf/templates/_helpers.tpl create mode 100644 platform/test/e2e-test-crd/sdewan-hub/scripts/cnf/templates/cm.yaml create mode 100644 platform/test/e2e-test-crd/sdewan-hub/scripts/cnf/templates/deployment.yaml create mode 100644 platform/test/e2e-test-crd/sdewan-hub/scripts/cnf/values.yaml create mode 100755 platform/test/e2e-test-crd/sdewan-hub/scripts/setup-cnf.sh create mode 100644 platform/test/e2e-test-crd/sdewan-hub/scripts/variables create mode 100755 platform/test/e2e-test-crd/setup.sh create mode 100755 platform/test/e2e-test-crd/test.sh diff --git a/platform/cnf/src/Dockerfile_1806_mwan3.tpl b/platform/cnf/src/Dockerfile_1806_mwan3.tpl index 658f463..50bfe50 100644 --- a/platform/cnf/src/Dockerfile_1806_mwan3.tpl +++ b/platform/cnf/src/Dockerfile_1806_mwan3.tpl @@ -17,6 +17,8 @@ RUN mkdir /var/lock && \ COPY system /etc/config/system COPY ipsec /etc/config/ipsec COPY ipsec_exec /etc/init.d/ipsec +COPY sdewan.user /etc/sdewan.user +COPY default_firewall /etc/config/firewall COPY rest_v1 /usr/lib/lua/luci/controller/rest_v1 ENV http_proxy= diff --git a/platform/cnf/src/Dockerfile_1806_mwan3_noproxy.tpl b/platform/cnf/src/Dockerfile_1806_mwan3_noproxy.tpl index 0fe098b..c1ef291 100644 --- a/platform/cnf/src/Dockerfile_1806_mwan3_noproxy.tpl +++ b/platform/cnf/src/Dockerfile_1806_mwan3_noproxy.tpl @@ -14,6 +14,8 @@ RUN mkdir /var/lock && \ COPY system /etc/config/system COPY ipsec /etc/config/ipsec COPY ipsec_exec /etc/init.d/ipsec +COPY sdewan.user /etc/sdewan.user +COPY default_firewall /etc/config/firewall COPY rest_v1 /usr/lib/lua/luci/controller/rest_v1 USER root diff --git a/platform/cnf/src/default_firewall b/platform/cnf/src/default_firewall new file mode 100644 index 0000000..403caed --- /dev/null +++ b/platform/cnf/src/default_firewall @@ -0,0 +1,11 @@ +config defaults + option syn_flood '1' + option input 'ACCEPT' + option output 'ACCEPT' + option forward 'ACCEPT' + +config include + option path '/etc/firewall.user' + +config include + option path '/etc/sdewan.user' diff --git a/platform/cnf/src/sdewan.user b/platform/cnf/src/sdewan.user new file mode 100644 index 0000000..82642e7 --- /dev/null +++ b/platform/cnf/src/sdewan.user @@ -0,0 +1,25 @@ +set -o pipefail + +api_server=$(nslookup kubernetes.default.svc.cluster.local | tail -n2 | awk -F':' '{print $2}' | head -1) +istio_ingress=$(nslookup istio-ingressgateway.istio-system.svc.cluster.local | tail -n2 | awk -F':' '{print $2}' | head -1) + +SNAT_RULE='POSTROUTING -o eth0 -j MASQUERADE' + +iptables -t nat -C $SNAT_RULE || if_exist=$? + +if [ $if_exist -ne 0 ]; +then + echo "Insert Default rules" +else + iptables -t nat -D $SNAT_RULE + iptables -t nat -D PREROUTING -p tcp --dport 6443 -j DNAT --to-destination $api_server + iptables -t nat -D PREROUTING -i net0 -j DNAT --to-destination $istio_ingress +fi + +iptables -t nat -A $SNAT_RULE +iptables -t nat -A PREROUTING -p tcp --dport 6443 -j DNAT --to-destination $api_server +if [[ $istio_ingress != "NXDOMAIN" ]]; +then + iptables -t nat -A PREROUTING -i net0 -j DNAT --to-destination $istio_ingress +fi + diff --git a/platform/test/e2e-test-crd/README.md b/platform/test/e2e-test-crd/README.md new file mode 100644 index 0000000..57f17b8 --- /dev/null +++ b/platform/test/e2e-test-crd/README.md @@ -0,0 +1,30 @@ +#SD-eWAN test scenario +[Overview] +In this test scenario, three clusters are created for edge-a, edge-b and sdewan-hub. +Two tunnels are established between the edge and the hub, and also two applications +are installed in edga-a and edge-b. Tunnels are verified thru the connection test between +the two applications. + +[Test guide] +Run the test.sh under sdwan/platform/test/e2e-test/ to invoke the vm creation and configurations. + $ ./test.sh + +Scripts description: +1. The Vagrantfile will be used to setup the base environment. +2. The installer.sh script contains the minimal Ubuntu instructions required for bringing up ICN. +3. The setup-cnf.sh script creates ovn networks, sdewan cnfs and application pods if needed. +4. The sdwan_verifier.sh script inserts configs into the sdewan cnf, including firewall and ipsec. +5. The test-connection.sh script under edge-a tests the connection between the applications +reside in edge-a and edge-b. + + +[License] + +Apache-2.0 + +[1]: https://gerrit.akraino.org/r/icn/sdwan + +[2]: https://git.onap.org/multicloud/k8s + +[3]: https://www.vagrantup.com/ + diff --git a/platform/test/e2e-test-crd/edge-a/Vagrantfile b/platform/test/e2e-test-crd/edge-a/Vagrantfile new file mode 100644 index 0000000..2e34893 --- /dev/null +++ b/platform/test/e2e-test-crd/edge-a/Vagrantfile @@ -0,0 +1,151 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +box = { + :virtualbox => { :name => 'elastic/ubuntu-18.04-x86_64', :version => '20191013.0.0'}, + :libvirt => { :name => 'intergratedcloudnative/ubuntu1804', :version => '1.0.0'} +} + +require 'yaml' +pdf = File.dirname(__FILE__) + '/config/default.yml' +if File.exist?(File.dirname(__FILE__) + '/config/pdf.yml') + pdf = File.dirname(__FILE__) + '/config/pdf.yml' +end +nodes = YAML.load_file(pdf) + +# Inventory file creation +File.open(File.dirname(__FILE__) + "/inventory/hosts.ini", "w") do |inventory_file| + inventory_file.puts("[all]") + nodes.each do |node| + inventory_file.puts("#{node['name']}\tansible_ssh_host=#{node['ip']} ansible_ssh_port=22") + end + ['kube-master', 'kube-node', 'etcd', 'ovn-central', 'ovn-controller', 'virtlet', 'cmk'].each do|group| + inventory_file.puts("\n[#{group}]") + nodes.each do |node| + if node['roles'].include?("#{group}") + inventory_file.puts(node['name']) + end + end + end + inventory_file.puts("\n[k8s-cluster:children]\nkube-node\nkube-master") +end + +provider = (ENV['VAGRANT_DEFAULT_PROVIDER'] || :libvirt).to_sym +puts "[INFO] Provider: #{provider} " + +if ENV['no_proxy'] != nil or ENV['NO_PROXY'] + $no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || "127.0.0.1,localhost" + nodes.each do |node| + $no_proxy += "," + node['ip'] + end + $subnet = "192.168.121" + if provider == :virtualbox + $subnet = "10.0.2" + end + # NOTE: This range is based on vagrant-libvirt network definition CIDR 192.168.121.0/27 + (1..31).each do |i| + $no_proxy += ",#{$subnet}.#{i}" + end +end + +Vagrant.configure("2") do |config| + config.vm.box = box[provider][:name] + config.vm.box_version = box[provider][:version] + config.ssh.insert_key = false + + if ENV['http_proxy'] != nil and ENV['https_proxy'] != nil + if Vagrant.has_plugin?('vagrant-proxyconf') + config.proxy.http = ENV['http_proxy'] || ENV['HTTP_PROXY'] || "" + config.proxy.https = ENV['https_proxy'] || ENV['HTTPS_PROXY'] || "" + config.proxy.no_proxy = $no_proxy + config.proxy.enabled = { docker: false } + end + end + config.vm.provider 'libvirt' do |v| + v.nested = true + v.cpu_mode = 'host-passthrough' + v.management_network_address = "192.168.121.0/27" + v.random_hostname = true + end + + sync_type = "virtualbox" + if provider == :libvirt + sync_type = "nfs" + end + + nodes.each do |node| + config.vm.define node['name'] do |nodeconfig| + if node['roles'].include?("kube-master") + nodeconfig.vm.synced_folder '../', '/home/vagrant/multicloud-k8s/', type: sync_type + end + if node['roles'].include?("kube-node") + nodeconfig.vm.provision 'shell', privileged: false do |sh| + sh.inline = <<-SHELL + sudo sed -i 's:GRUB_CMDLINE_LINUX=.*:GRUB_CMDLINE_LINUX="isolcpus=0-7":' /etc/default/grub + sudo update-grub + SHELL + end + end + nodeconfig.vm.hostname = node['name'] + nodeconfig.vm.network :private_network, :ip => node['ip'], :type => :static + nodeconfig.vm.provider 'virtualbox' do |v| + v.customize ["modifyvm", :id, "--memory", node['memory']] + v.customize ["modifyvm", :id, "--cpus", node['cpus']] + if node.has_key? "volumes" + node['volumes'].each do |volume| + $volume_file = "#{node['name']}-#{volume['name']}.vdi" + unless File.exist?($volume_file) + v.customize ['createmedium', 'disk', '--filename', $volume_file, '--size', volume['size']] + end + v.customize ['storageattach', :id, '--storagectl', 'IDE Controller', '--port', 1, '--device', 0, '--type', 'hdd', '--medium', $volume_file] + end + end + end + nodeconfig.vm.provider 'libvirt' do |v| + v.memory = node['memory'] + v.cpus = node['cpus'] + nodeconfig.vm.provision 'file', source: './scripts', destination: '$HOME/' + nodeconfig.vm.provision 'file', source: '../edge-scripts/setup-cnf.sh', destination: '$HOME/scripts/' + nodeconfig.vm.provision 'file', source: '../edge-scripts/helm-tmp/cnf', destination: '$HOME/scripts/cnf' + nodeconfig.vm.provision 'file', source: '../edge-scripts/helm-tmp/controllers', destination: '$HOME/scripts/controllers' + nodeconfig.vm.provision 'shell' do |sh| + sh.path = "node.sh" + if node.has_key? "volumes" + $volume_mounts_dict = '' + node['volumes'].each do |volume| + $volume_mounts_dict += "#{volume['name']}=#{volume['mount']}," + $volume_file = "./#{node['name']}-#{volume['name']}.qcow2" + v.storage :file, :bus => 'sata', :device => volume['name'], :size => volume['size'] + end + sh.args = ['-v', $volume_mounts_dict[0...-1]] + end + end + end + end + end + + config.vm.define :installer, primary: true, autostart: false do |installer| + installer.vm.hostname = "multicloud" + installer.vm.network :private_network, :ip => "10.10.10.2", :type => :static + installer.vm.synced_folder '../', '/home/vagrant/multicloud-k8s/', type: sync_type + installer.vm.provision 'shell', privileged: false do |sh| + sh.env = {'KUD_PLUGIN_ENABLED': 'false', 'OVN_CENTRAL_INTERFACE': 'eth1'} + sh.inline = <<-SHELL + cp /home/vagrant/multicloud-k8s/edge-a/insecure_keys/key.pub /home/vagrant/.ssh/id_rsa.pub + cp /home/vagrant/multicloud-k8s/edge-a/insecure_keys/key /home/vagrant/.ssh/id_rsa + chown vagrant /home/vagrant/.ssh/id_rsa + chmod 400 /home/vagrant/.ssh/id_rsa + sudo apt install jq -y + cd /home/vagrant/multicloud-k8s/edge-a/ && ./installer.sh | tee kud_installer.log + SHELL + end + end +end diff --git a/platform/test/e2e-test-crd/edge-a/config/default.yml b/platform/test/e2e-test-crd/edge-a/config/default.yml new file mode 100644 index 0000000..6f29c68 --- /dev/null +++ b/platform/test/e2e-test-crd/edge-a/config/default.yml @@ -0,0 +1,24 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- name: "edge-a" + ip: "10.10.10.3" + memory: 18432 + cpus: 16 + volumes: + - name: sdb + size: 50 + mount: /var/lib/docker/ + roles: + - kube-master + - etcd + - ovn-central + - kube-node + - ovn-controller diff --git a/platform/test/e2e-test-crd/edge-a/insecure_keys/key b/platform/test/e2e-test-crd/edge-a/insecure_keys/key new file mode 100644 index 0000000..7d6a083 --- /dev/null +++ b/platform/test/e2e-test-crd/edge-a/insecure_keys/key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI +w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP +kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2 +hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO +Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW +yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd +ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1 +Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf +TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK +iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A +sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf +4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP +cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk +EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN +CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX +3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG +YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj +3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+ +dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz +6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC +P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF +llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ +kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH ++vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ +NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s= +-----END RSA PRIVATE KEY----- diff --git a/platform/test/e2e-test-crd/edge-a/insecure_keys/key.pub b/platform/test/e2e-test-crd/edge-a/insecure_keys/key.pub new file mode 100644 index 0000000..18a9c00 --- /dev/null +++ b/platform/test/e2e-test-crd/edge-a/insecure_keys/key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key diff --git a/platform/test/e2e-test-crd/edge-a/installer.sh b/platform/test/e2e-test-crd/edge-a/installer.sh new file mode 100755 index 0000000..fd6349f --- /dev/null +++ b/platform/test/e2e-test-crd/edge-a/installer.sh @@ -0,0 +1,233 @@ +#!/bin/bash +#SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o errexit +set -o nounset +set -o pipefail + +INSTALLER_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" + +source ${INSTALLER_DIR}/../k8s/kud/tests/_functions.sh + +# _install_go() - Install GoLang package +function _install_go { + version=$(grep "go_version" ${kud_playbooks}/kud-vars.yml | awk -F "'" '{print $2}') + local tarball=go$version.linux-amd64.tar.gz + + #gcc is required for go apps compilation + if ! which gcc; then + sudo apt-get install -y gcc + fi + + if $(go version &>/dev/null); then + return + fi + + wget https://dl.google.com/go/$tarball + sudo tar -C /usr/local -xzf $tarball + rm $tarball + + export PATH=$PATH:/usr/local/go/bin + sudo sed -i "s|^PATH=.*|PATH=\"$PATH\"|" /etc/environment +} + +# _install_pip() - Install Python Package Manager +function _install_pip { + if $(pip --version &>/dev/null); then + sudo -E pip install --upgrade pip + else + sudo apt-get install -y python-dev + curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python + fi +} + +# _install_ansible() - Install and Configure Ansible program +function _install_ansible { + if $(ansible --version &>/dev/null); then + sudo pip uninstall -y ansible + fi + _install_pip + local version=$(grep "ansible_version" ${kud_playbooks}/kud-vars.yml | awk -F ': ' '{print $2}') + sudo mkdir -p /etc/ansible/ + sudo -E pip install ansible==$version +} + +# _install_docker() - Download and install docker-engine +function _install_docker { + local max_concurrent_downloads=${1:-3} + + if $(docker version &>/dev/null); then + return + fi + sudo apt-get install -y apt-transport-https ca-certificates curl + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + sudo apt-get update + sudo apt-get install -y docker-ce + + sudo mkdir -p /etc/systemd/system/docker.service.d + if [ ${http_proxy:-} ]; then + echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/http-proxy.conf + echo "Environment=\"HTTP_PROXY=$http_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/http-proxy.conf + fi + if [ ${https_proxy:-} ]; then + echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/https-proxy.conf + echo "Environment=\"HTTPS_PROXY=$https_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/https-proxy.conf + fi + if [ ${no_proxy:-} ]; then + echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/no-proxy.conf + echo "Environment=\"NO_PROXY=$no_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/no-proxy.conf + fi + sudo systemctl daemon-reload + echo "DOCKER_OPTS=\"-H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock --max-concurrent-downloads $max_concurrent_downloads \"" | sudo tee --append /etc/default/docker + if [[ -z $(groups | grep docker) ]]; then + sudo usermod -aG docker $USER + fi + + sudo systemctl restart docker + sleep 10 +} + +function _set_environment_file { + # By default ovn central interface is the first active network interface on localhost. If other wanted, need to export this variable in aio.sh or Vagrant file. + OVN_CENTRAL_INTERFACE="${OVN_CENTRAL_INTERFACE:-$(ip addr show | awk '/inet.*brd/{print $NF; exit}')}" + echo "export OVN_CENTRAL_INTERFACE=${OVN_CENTRAL_INTERFACE}" | sudo tee --append /etc/environment + echo "export OVN_CENTRAL_ADDRESS=$(get_ovn_central_address)" | sudo tee --append /etc/environment + echo "export KUBE_CONFIG_DIR=/opt/kubeconfig" | sudo tee --append /etc/environment + echo "export CSAR_DIR=/opt/csar" | sudo tee --append /etc/environment +} + +# install_k8s() - Install Kubernetes using kubespray tool +function install_k8s { + echo "Deploying kubernetes" + local dest_folder=/opt + version=$(grep "kubespray_version" ${kud_playbooks}/kud-vars.yml | awk -F ': ' '{print $2}') + local_release_dir=$(grep "local_release_dir" $kud_inventory_folder/group_vars/k8s-cluster.yml | awk -F "\"" '{print $2}') + local tarball=v$version.tar.gz + sudo apt-get install -y sshpass make unzip # install make to run mitogen target and unzip is mitogen playbook dependency + sudo apt-get install -y gnupg2 software-properties-common + _install_docker + _install_ansible + wget https://github.com/kubernetes-incubator/kubespray/archive/$tarball + sudo tar -C $dest_folder -xzf $tarball + sudo mv $dest_folder/kubespray-$version/ansible.cfg /etc/ansible/ansible.cfg + sudo chown -R $USER $dest_folder/kubespray-$version + sudo mkdir -p ${local_release_dir}/containers + rm $tarball + + pushd $dest_folder/kubespray-$version/ + sudo -E pip install -r ./requirements.txt + make mitogen + popd + rm -f $kud_inventory_folder/group_vars/all.yml 2> /dev/null + if [[ -n "${verbose:-}" ]]; then + echo "kube_log_level: 5" | tee $kud_inventory_folder/group_vars/all.yml + else + echo "kube_log_level: 2" | tee $kud_inventory_folder/group_vars/all.yml + fi + echo "kubeadm_enabled: true" | tee --append $kud_inventory_folder/group_vars/all.yml + if [[ -n "${http_proxy:-}" ]]; then + echo "http_proxy: \"$http_proxy\"" | tee --append $kud_inventory_folder/group_vars/all.yml + fi + if [[ -n "${https_proxy:-}" ]]; then + echo "https_proxy: \"$https_proxy\"" | tee --append $kud_inventory_folder/group_vars/all.yml + fi + ansible-playbook $verbose -i $kud_inventory $dest_folder/kubespray-$version/cluster.yml --become --become-user=root | sudo tee $log_folder/setup-kubernetes.log + + # Configure environment + mkdir -p $HOME/.kube + cp $kud_inventory_folder/artifacts/admin.conf $HOME/.kube/config + # Copy Kubespray kubectl to be usable in host running Ansible. Requires kubectl_localhost: true in inventory/group_vars/k8s-cluster.yml + sudo cp $kud_inventory_folder/artifacts/kubectl /usr/local/bin/ +} + +# install_addons() - Install Kubenertes AddOns +function install_addons { + source /etc/environment + echo "Installing Kubernetes AddOns" + _install_ansible + sudo ansible-galaxy install $verbose -r $kud_infra_folder/galaxy-requirements.yml --ignore-errors + ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-kud.yml | sudo tee $log_folder/setup-kud.log + for addon in ${KUD_ADDONS:-ovn4nfv}; do + echo "Deploying $addon using configure-$addon.yml playbook.." + ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-${addon}.yml | sudo tee $log_folder/setup-${addon}.log + done + echo "Run the test cases if testing_enabled is set to true." + if [[ "${testing_enabled}" == "true" ]]; then + for addon in ${KUD_ADDONS:-ovn4nfv}; do + pushd $kud_tests + bash ${addon}.sh + popd + done + fi + echo "Add-ons deployment complete..." +} + +# _print_kubernetes_info() - Prints the login Kubernetes information +function _print_kubernetes_info { + if ! $(kubectl version &>/dev/null); then + return + fi + # Expose Dashboard using NodePort + node_port=30080 + KUBE_EDITOR="sed -i \"s|type\: ClusterIP|type\: NodePort|g\"" kubectl -n kube-system edit service kubernetes-dashboard + KUBE_EDITOR="sed -i \"s|nodePort\: .*|nodePort\: $node_port|g\"" kubectl -n kube-system edit service kubernetes-dashboard + + master_ip=$(kubectl cluster-info | grep "Kubernetes master" | awk -F ":" '{print $2}') + + printf "Kubernetes Info\n===============\n" > $k8s_info_file + echo "Dashboard URL: https:$master_ip:$node_port" >> $k8s_info_file + echo "Admin user: kube" >> $k8s_info_file + echo "Admin password: secret" >> $k8s_info_file +} + +sudo -k # forgot sudo password +if ! sudo -n "true"; then + echo "" + echo "passwordless sudo is needed for '$(id -nu)' user." + echo "Please fix your /etc/sudoers file. You likely want an" + echo "entry like the following one..." + echo "" + echo "$(id -nu) ALL=(ALL) NOPASSWD: ALL" + exit 1 +fi + +verbose="" +if [[ -n "${KUD_DEBUG:-}" ]]; then + set -o xtrace + verbose="-vvv" +fi + +# Configuration values +log_folder=/var/log/kud +kud_folder=${INSTALLER_DIR}/../k8s/kud/hosting_providers/vagrant +kud_infra_folder=$kud_folder/../../deployment_infra +export kud_inventory_folder=$kud_folder/../../../../edge-a/inventory +kud_inventory=$kud_inventory_folder/hosts.ini +kud_playbooks=$kud_infra_folder/playbooks +kud_tests=$kud_folder/../../tests +k8s_info_file=$kud_folder/k8s_info.log +testing_enabled=${KUD_ENABLE_TESTS:-false} +sudo mkdir -p $log_folder +sudo mkdir -p /opt/csar +sudo chown -R $USER /opt/csar +# Install dependencies +# Setup proxy variables +if [ -f $kud_folder/sources.list ]; then + sudo mv /etc/apt/sources.list /etc/apt/sources.list.backup + sudo cp $kud_folder/sources.list /etc/apt/sources.list +fi +echo "Removing ppa for jonathonf/python-3.6" +sudo ls /etc/apt/sources.list.d/ || true +sudo find /etc/apt/sources.list.d -maxdepth 1 -name '*jonathonf*' -delete || true +sudo apt-get update +install_k8s +_set_environment_file +install_addons diff --git a/platform/test/e2e-test-crd/edge-a/inventory/group_vars/all.yml b/platform/test/e2e-test-crd/edge-a/inventory/group_vars/all.yml new file mode 100644 index 0000000..528430c --- /dev/null +++ b/platform/test/e2e-test-crd/edge-a/inventory/group_vars/all.yml @@ -0,0 +1,2 @@ +kube_log_level: 2 +kubeadm_enabled: true diff --git a/platform/test/e2e-test-crd/edge-a/inventory/group_vars/k8s-cluster.yml b/platform/test/e2e-test-crd/edge-a/inventory/group_vars/k8s-cluster.yml new file mode 100644 index 0000000..6f1c40b --- /dev/null +++ b/platform/test/e2e-test-crd/edge-a/inventory/group_vars/k8s-cluster.yml @@ -0,0 +1,82 @@ +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Kubernetes configuration dirs and system namespace. +# Those are where all the additional config stuff goes +# kubernetes normally puts in /srv/kubernetes. +# This puts them in a sane location and namespace. +# Editing those values will almost surely break something. +system_namespace: kube-system + +# Logging directory (sysvinit systems) +kube_log_dir: "/var/log/kubernetes" + +kube_api_anonymous_auth: true + +# Users to create for basic auth in Kubernetes API via HTTP +# Optionally add groups for user +kube_api_pwd: "secret" +kube_users: + kube: + pass: "{{kube_api_pwd}}" + role: admin + groups: + - system:masters + +## It is possible to activate / deactivate selected authentication methods (basic auth, static token auth) +#kube_oidc_auth: false +kube_basic_auth: true +kube_token_auth: true + +# Choose network plugin (calico, contiv, weave or flannel) +# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing +kube_network_plugin: flannel + +# Make a copy of kubeconfig (admin.conf) on the host that runs Ansible to inventory/artifacts +kubeconfig_localhost: true +# Copy kubectl binary on the host that runs Ansible to inventory/artifacts +kubectl_localhost: true +# Disable nodelocal dns cache +enable_nodelocaldns: false +# Enable MountPropagation gate feature +local_volumes_enabled: true +local_volume_provisioner_enabled: true + +## Change this to use another Kubernetes version, e.g. a current beta release +kube_version: v1.16.9 + +# Helm deployment +helm_enabled: true + +# Kube-proxy proxyMode configuration. +# NOTE: Ipvs is based on netfilter hook function, but uses hash table as the underlying data structure and +# works in the kernel space +# https://kubernetes.io/docs/concepts/services-networking/service/#proxy-mode-ipvs +#kube_proxy_mode: ipvs + +# Download container images only once then push to cluster nodes in batches +download_run_once: true + +# Where the binaries will be downloaded. +# Note: ensure that you've enough disk space (about 1G) +local_release_dir: "/tmp/releases" + +# Makes the installer node a delegate for pushing images while running +# the deployment with ansible. This maybe the case if cluster nodes +# cannot access each over via ssh or you want to use local docker +# images as a cache for multiple clusters. +download_localhost: true + +# Subnet for cluster IPs +kube_service_addresses: 10.244.0.0/18 +# Subnet for Pod IPs +kube_pods_subnet: 10.244.64.0/18 + +# Add specific docker version +docker_version: 'latest' diff --git a/platform/test/e2e-test-crd/edge-a/node.sh b/platform/test/e2e-test-crd/edge-a/node.sh new file mode 100755 index 0000000..a51be19 --- /dev/null +++ b/platform/test/e2e-test-crd/edge-a/node.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o nounset +set -o pipefail + +# usage() - Prints the usage of the program +function usage { + cat <> /etc/fstab +} + +while getopts "h?v:" opt; do + case $opt in + v) + dict_volumes="$OPTARG" + ;; + h|\?) + usage + exit + ;; + esac +done + +swapoff -a +if [[ -n "${dict_volumes+x}" ]]; then + for kv in ${dict_volumes//,/ } ;do + mount_external_partition ${kv%=*} ${kv#*=} + done +fi + +vendor_id=$(lscpu|grep "Vendor ID") +if [[ $vendor_id == *GenuineIntel* ]]; then + kvm_ok=$(cat /sys/module/kvm_intel/parameters/nested) + if [[ $kvm_ok == 'N' ]]; then + echo "Enable Intel Nested-Virtualization" + rmmod kvm-intel + echo 'options kvm-intel nested=y' >> /etc/modprobe.d/dist.conf + modprobe kvm-intel + echo kvm-intel >> /etc/modules + fi +else + kvm_ok=$(cat /sys/module/kvm_amd/parameters/nested) + if [[ $kvm_ok == '0' ]]; then + echo "Enable AMD Nested-Virtualization" + rmmod kvm-amd + sh -c "echo 'options kvm-amd nested=1' >> /etc/modprobe.d/dist.conf" + modprobe kvm-amd + echo kvm-amd >> /etc/modules + fi +fi +modprobe vhost_net +echo vhost_net >> /etc/modules +source /etc/os-release || source /usr/lib/os-release +case ${ID,,} in + *suse) + ;; + ubuntu|debian) + apt-get install -y cpu-checker + kvm-ok + ;; + rhel|centos|fedora) + ;; +esac diff --git a/platform/test/e2e-test-crd/edge-a/scripts/edge-a-sdwan-config b/platform/test/e2e-test-crd/edge-a/scripts/edge-a-sdwan-config new file mode 100644 index 0000000..268b227 --- /dev/null +++ b/platform/test/e2e-test-crd/edge-a/scripts/edge-a-sdwan-config @@ -0,0 +1,22 @@ +#!/bin/bash + +#Adding SNAT rules +cat > $sdwanConfigs.yaml << EOF +apiVersion: batch.sdewan.akraino.org/v1alpha1 +kind: FirewallSNAT +metadata: + name: firewallsnat + namespace: default + labels: + sdewanPurpose: $sdewan_cnf_name +spec: + src_ip: $ovnSubnet + src_dip: $sdwan_pod_virtual_ip + dest: pnetwork + dest_ip: $internal_subnet + proto: tcp + target: SNAT + +EOF + + diff --git a/platform/test/e2e-test-crd/edge-a/scripts/test-connection.sh b/platform/test/e2e-test-crd/edge-a/scripts/test-connection.sh new file mode 100755 index 0000000..e5f97c0 --- /dev/null +++ b/platform/test/e2e-test-crd/edge-a/scripts/test-connection.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o errexit +set -o nounset +set -o pipefail + +test -f /home/vagrant/scripts/variables && . /home/vagrant/scripts/variables + +sdewan_cnf=$(kubectl get po | grep $sdewan_cnf_name | awk '{print $1}' | head -1) +sdewan_virtual_ip=$(kubectl exec -it $sdewan_cnf ip address | grep $wan_interface | awk '/inet/{print $2}' | cut -f1 -d "/" | grep 192.168) +app_pod=$(kubectl get po | grep $app_pod_name | cut -f1 -d " ") +echo "Logging into the dummy pod $app_pod in edgeA..." + + +echo "Determine the ip address of remote host" +if [ "$sdewan_virtual_ip" == "192.168.1.5" ] +then + remote_ip="192.168.1.6" +else + remote_ip="192.168.1.5" +fi +echo "The remote ip is ${remote_ip}" + +echo "Sending request to the remote httpbin. If the connection is established, it shall return the ip of the caller." +kubectl exec -it $app_pod -- curl -X GET "http://$remote_ip/ip" -H "accept: application/json" >> response.json +cat response.json + +echo "Confirming the testing result..." +sudo apt install -y jq +rs=$(jq -r '.origin' response.json) +if [ "$rs" == "$sdewan_virtual_ip" ] +then + echo "Ip matched. End-to-end test passed" +else + echo "End-to-end test failed. Please check the logs for more details" +fi diff --git a/platform/test/e2e-test-crd/edge-a/scripts/variables b/platform/test/e2e-test-crd/edge-a/scripts/variables new file mode 100644 index 0000000..e34c0a5 --- /dev/null +++ b/platform/test/e2e-test-crd/edge-a/scripts/variables @@ -0,0 +1,19 @@ +providerSubnet=10.10.10.1/24 +providerGateway=10.10.10.1/24 +providerExcludeIps=10.10.10.2..10.10.10.9 +providerNetworkInterface=eth1 +ovnGateway=172.16.30.1/24 +ovnExcludeIps=172.16.30.2..172.16.30.9 +appLan0=172.16.30.15 +cnfWanGateway=10.10.10.1 + +export cnfWan0=10.10.10.15 +export cnfLan0=172.16.30.10 +export hubIp=10.10.10.35 +export ovnSubnet=172.16.30.1/24 +export edgeName=edgeA +export wan_interface=net1 +export sdewan_cnf_name=sdewan-edge-a +export app_pod_name=simple-http-service +export sdwanConfigs=edge-a-sdwan-config +export internal_subnet=192.168.1.1/24 diff --git a/platform/test/e2e-test-crd/edge-b/Vagrantfile b/platform/test/e2e-test-crd/edge-b/Vagrantfile new file mode 100644 index 0000000..1e7e8ce --- /dev/null +++ b/platform/test/e2e-test-crd/edge-b/Vagrantfile @@ -0,0 +1,151 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +box = { + :virtualbox => { :name => 'elastic/ubuntu-18.04-x86_64', :version => '20191013.0.0'}, + :libvirt => { :name => 'intergratedcloudnative/ubuntu1804', :version => '1.0.0'} +} + +require 'yaml' +pdf = File.dirname(__FILE__) + '/config/default.yml' +if File.exist?(File.dirname(__FILE__) + '/config/pdf.yml') + pdf = File.dirname(__FILE__) + '/config/pdf.yml' +end +nodes = YAML.load_file(pdf) + +# Inventory file creation +File.open(File.dirname(__FILE__) + "/inventory/hosts.ini", "w") do |inventory_file| + inventory_file.puts("[all]") + nodes.each do |node| + inventory_file.puts("#{node['name']}\tansible_ssh_host=#{node['ip']} ansible_ssh_port=22") + end + ['kube-master', 'kube-node', 'etcd', 'ovn-central', 'ovn-controller', 'virtlet', 'cmk'].each do|group| + inventory_file.puts("\n[#{group}]") + nodes.each do |node| + if node['roles'].include?("#{group}") + inventory_file.puts(node['name']) + end + end + end + inventory_file.puts("\n[k8s-cluster:children]\nkube-node\nkube-master") +end + +provider = (ENV['VAGRANT_DEFAULT_PROVIDER'] || :libvirt).to_sym +puts "[INFO] Provider: #{provider} " + +if ENV['no_proxy'] != nil or ENV['NO_PROXY'] + $no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || "127.0.0.1,localhost" + nodes.each do |node| + $no_proxy += "," + node['ip'] + end + $subnet = "192.168.121" + if provider == :virtualbox + $subnet = "10.0.2" + end + # NOTE: This range is based on vagrant-libvirt network definition CIDR 192.168.121.0/27 + (1..31).each do |i| + $no_proxy += ",#{$subnet}.#{i}" + end +end + +Vagrant.configure("2") do |config| + config.vm.box = box[provider][:name] + config.vm.box_version = box[provider][:version] + config.ssh.insert_key = false + + if ENV['http_proxy'] != nil and ENV['https_proxy'] != nil + if Vagrant.has_plugin?('vagrant-proxyconf') + config.proxy.http = ENV['http_proxy'] || ENV['HTTP_PROXY'] || "" + config.proxy.https = ENV['https_proxy'] || ENV['HTTPS_PROXY'] || "" + config.proxy.no_proxy = $no_proxy + config.proxy.enabled = { docker: false } + end + end + config.vm.provider 'libvirt' do |v| + v.nested = true + v.cpu_mode = 'host-passthrough' + v.management_network_address = "192.168.121.0/27" + v.random_hostname = true + end + + sync_type = "virtualbox" + if provider == :libvirt + sync_type = "nfs" + end + + nodes.each do |node| + config.vm.define node['name'] do |nodeconfig| + if node['roles'].include?("kube-master") + nodeconfig.vm.synced_folder '../', '/home/vagrant/multicloud-k8s/', type: sync_type + end + if node['roles'].include?("kube-node") + nodeconfig.vm.provision 'shell', privileged: false do |sh| + sh.inline = <<-SHELL + sudo sed -i 's:GRUB_CMDLINE_LINUX=.*:GRUB_CMDLINE_LINUX="isolcpus=0-7":' /etc/default/grub + sudo update-grub + SHELL + end + end + nodeconfig.vm.hostname = node['name'] + nodeconfig.vm.network :private_network, :ip => node['ip'], :type => :static + nodeconfig.vm.provider 'virtualbox' do |v| + v.customize ["modifyvm", :id, "--memory", node['memory']] + v.customize ["modifyvm", :id, "--cpus", node['cpus']] + if node.has_key? "volumes" + node['volumes'].each do |volume| + $volume_file = "#{node['name']}-#{volume['name']}.vdi" + unless File.exist?($volume_file) + v.customize ['createmedium', 'disk', '--filename', $volume_file, '--size', volume['size']] + end + v.customize ['storageattach', :id, '--storagectl', 'IDE Controller', '--port', 1, '--device', 0, '--type', 'hdd', '--medium', $volume_file] + end + end + end + nodeconfig.vm.provider 'libvirt' do |v| + v.memory = node['memory'] + v.cpus = node['cpus'] + nodeconfig.vm.provision 'file', source: './scripts', destination: '$HOME/' + nodeconfig.vm.provision 'file', source: '../edge-scripts/setup-cnf.sh', destination: '$HOME/scripts/' + nodeconfig.vm.provision 'file', source: '../edge-scripts/helm-tmp/cnf', destination: '$HOME/scripts/cnf' + nodeconfig.vm.provision 'file', source: '../edge-scripts/helm-tmp/controllers', destination: '$HOME/scripts/controllers' + nodeconfig.vm.provision 'shell' do |sh| + sh.path = "node.sh" + if node.has_key? "volumes" + $volume_mounts_dict = '' + node['volumes'].each do |volume| + $volume_mounts_dict += "#{volume['name']}=#{volume['mount']}," + $volume_file = "./#{node['name']}-#{volume['name']}.qcow2" + v.storage :file, :bus => 'sata', :device => volume['name'], :size => volume['size'] + end + sh.args = ['-v', $volume_mounts_dict[0...-1]] + end + end + end + end + end + + config.vm.define :installer, primary: true, autostart: false do |installer| + installer.vm.hostname = "multicloud" + installer.vm.network :private_network, :ip => "10.10.10.2", :type => :static + installer.vm.synced_folder '../', '/home/vagrant/multicloud-k8s/', type: sync_type + installer.vm.provision 'shell', privileged: false do |sh| + sh.env = {'KUD_PLUGIN_ENABLED': 'false', 'OVN_CENTRAL_INTERFACE': 'eth1'} + sh.inline = <<-SHELL + cp /home/vagrant/multicloud-k8s/edge-b/insecure_keys/key.pub /home/vagrant/.ssh/id_rsa.pub + cp /home/vagrant/multicloud-k8s/edge-b/insecure_keys/key /home/vagrant/.ssh/id_rsa + chown vagrant /home/vagrant/.ssh/id_rsa + chmod 400 /home/vagrant/.ssh/id_rsa + sudo apt install jq -y + cd /home/vagrant/multicloud-k8s/edge-b/ && ./installer.sh | tee kud_installer.log + SHELL + end + end +end diff --git a/platform/test/e2e-test-crd/edge-b/config/default.yml b/platform/test/e2e-test-crd/edge-b/config/default.yml new file mode 100644 index 0000000..6a1f9bc --- /dev/null +++ b/platform/test/e2e-test-crd/edge-b/config/default.yml @@ -0,0 +1,24 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- name: "edge-b" + ip: "10.10.10.4" + memory: 18432 + cpus: 16 + volumes: + - name: sdb + size: 50 + mount: /var/lib/docker/ + roles: + - kube-master + - etcd + - ovn-central + - kube-node + - ovn-controller diff --git a/platform/test/e2e-test-crd/edge-b/insecure_keys/key b/platform/test/e2e-test-crd/edge-b/insecure_keys/key new file mode 100644 index 0000000..7d6a083 --- /dev/null +++ b/platform/test/e2e-test-crd/edge-b/insecure_keys/key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI +w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP +kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2 +hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO +Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW +yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd +ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1 +Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf +TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK +iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A +sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf +4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP +cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk +EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN +CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX +3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG +YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj +3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+ +dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz +6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC +P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF +llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ +kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH ++vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ +NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s= +-----END RSA PRIVATE KEY----- diff --git a/platform/test/e2e-test-crd/edge-b/insecure_keys/key.pub b/platform/test/e2e-test-crd/edge-b/insecure_keys/key.pub new file mode 100644 index 0000000..18a9c00 --- /dev/null +++ b/platform/test/e2e-test-crd/edge-b/insecure_keys/key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key diff --git a/platform/test/e2e-test-crd/edge-b/installer.sh b/platform/test/e2e-test-crd/edge-b/installer.sh new file mode 100755 index 0000000..3ef2762 --- /dev/null +++ b/platform/test/e2e-test-crd/edge-b/installer.sh @@ -0,0 +1,233 @@ +#!/bin/bash +#SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o errexit +set -o nounset +set -o pipefail + +INSTALLER_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" + +source ${INSTALLER_DIR}/../k8s/kud/tests/_functions.sh + +# _install_go() - Install GoLang package +function _install_go { + version=$(grep "go_version" ${kud_playbooks}/kud-vars.yml | awk -F "'" '{print $2}') + local tarball=go$version.linux-amd64.tar.gz + + #gcc is required for go apps compilation + if ! which gcc; then + sudo apt-get install -y gcc + fi + + if $(go version &>/dev/null); then + return + fi + + wget https://dl.google.com/go/$tarball + sudo tar -C /usr/local -xzf $tarball + rm $tarball + + export PATH=$PATH:/usr/local/go/bin + sudo sed -i "s|^PATH=.*|PATH=\"$PATH\"|" /etc/environment +} + +# _install_pip() - Install Python Package Manager +function _install_pip { + if $(pip --version &>/dev/null); then + sudo -E pip install --upgrade pip + else + sudo apt-get install -y python-dev + curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python + fi +} + +# _install_ansible() - Install and Configure Ansible program +function _install_ansible { + if $(ansible --version &>/dev/null); then + sudo pip uninstall -y ansible + fi + _install_pip + local version=$(grep "ansible_version" ${kud_playbooks}/kud-vars.yml | awk -F ': ' '{print $2}') + sudo mkdir -p /etc/ansible/ + sudo -E pip install ansible==$version +} + +# _install_docker() - Download and install docker-engine +function _install_docker { + local max_concurrent_downloads=${1:-3} + + if $(docker version &>/dev/null); then + return + fi + sudo apt-get install -y apt-transport-https ca-certificates curl + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + sudo apt-get update + sudo apt-get install -y docker-ce + + sudo mkdir -p /etc/systemd/system/docker.service.d + if [ ${http_proxy:-} ]; then + echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/http-proxy.conf + echo "Environment=\"HTTP_PROXY=$http_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/http-proxy.conf + fi + if [ ${https_proxy:-} ]; then + echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/https-proxy.conf + echo "Environment=\"HTTPS_PROXY=$https_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/https-proxy.conf + fi + if [ ${no_proxy:-} ]; then + echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/no-proxy.conf + echo "Environment=\"NO_PROXY=$no_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/no-proxy.conf + fi + sudo systemctl daemon-reload + echo "DOCKER_OPTS=\"-H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock --max-concurrent-downloads $max_concurrent_downloads \"" | sudo tee --append /etc/default/docker + if [[ -z $(groups | grep docker) ]]; then + sudo usermod -aG docker $USER + fi + + sudo systemctl restart docker + sleep 10 +} + +function _set_environment_file { + # By default ovn central interface is the first active network interface on localhost. If other wanted, need to export this variable in aio.sh or Vagrant file. + OVN_CENTRAL_INTERFACE="${OVN_CENTRAL_INTERFACE:-$(ip addr show | awk '/inet.*brd/{print $NF; exit}')}" + echo "export OVN_CENTRAL_INTERFACE=${OVN_CENTRAL_INTERFACE}" | sudo tee --append /etc/environment + echo "export OVN_CENTRAL_ADDRESS=$(get_ovn_central_address)" | sudo tee --append /etc/environment + echo "export KUBE_CONFIG_DIR=/opt/kubeconfig" | sudo tee --append /etc/environment + echo "export CSAR_DIR=/opt/csar" | sudo tee --append /etc/environment +} + +# install_k8s() - Install Kubernetes using kubespray tool +function install_k8s { + echo "Deploying kubernetes" + local dest_folder=/opt + version=$(grep "kubespray_version" ${kud_playbooks}/kud-vars.yml | awk -F ': ' '{print $2}') + local_release_dir=$(grep "local_release_dir" $kud_inventory_folder/group_vars/k8s-cluster.yml | awk -F "\"" '{print $2}') + local tarball=v$version.tar.gz + sudo apt-get install -y sshpass make unzip # install make to run mitogen target and unzip is mitogen playbook dependency + sudo apt-get install -y gnupg2 software-properties-common + _install_docker + _install_ansible + wget https://github.com/kubernetes-incubator/kubespray/archive/$tarball + sudo tar -C $dest_folder -xzf $tarball + sudo mv $dest_folder/kubespray-$version/ansible.cfg /etc/ansible/ansible.cfg + sudo chown -R $USER $dest_folder/kubespray-$version + sudo mkdir -p ${local_release_dir}/containers + rm $tarball + + pushd $dest_folder/kubespray-$version/ + sudo -E pip install -r ./requirements.txt + make mitogen + popd + rm -f $kud_inventory_folder/group_vars/all.yml 2> /dev/null + if [[ -n "${verbose:-}" ]]; then + echo "kube_log_level: 5" | tee $kud_inventory_folder/group_vars/all.yml + else + echo "kube_log_level: 2" | tee $kud_inventory_folder/group_vars/all.yml + fi + echo "kubeadm_enabled: true" | tee --append $kud_inventory_folder/group_vars/all.yml + if [[ -n "${http_proxy:-}" ]]; then + echo "http_proxy: \"$http_proxy\"" | tee --append $kud_inventory_folder/group_vars/all.yml + fi + if [[ -n "${https_proxy:-}" ]]; then + echo "https_proxy: \"$https_proxy\"" | tee --append $kud_inventory_folder/group_vars/all.yml + fi + ansible-playbook $verbose -i $kud_inventory $dest_folder/kubespray-$version/cluster.yml --become --become-user=root | sudo tee $log_folder/setup-kubernetes.log + + # Configure environment + mkdir -p $HOME/.kube + cp $kud_inventory_folder/artifacts/admin.conf $HOME/.kube/config + # Copy Kubespray kubectl to be usable in host running Ansible. Requires kubectl_localhost: true in inventory/group_vars/k8s-cluster.yml + sudo cp $kud_inventory_folder/artifacts/kubectl /usr/local/bin/ +} + +# install_addons() - Install Kubenertes AddOns +function install_addons { + source /etc/environment + echo "Installing Kubernetes AddOns" + _install_ansible + sudo ansible-galaxy install $verbose -r $kud_infra_folder/galaxy-requirements.yml --ignore-errors + ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-kud.yml | sudo tee $log_folder/setup-kud.log + for addon in ${KUD_ADDONS:-ovn4nfv}; do + echo "Deploying $addon using configure-$addon.yml playbook.." + ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-${addon}.yml | sudo tee $log_folder/setup-${addon}.log + done + echo "Run the test cases if testing_enabled is set to true." + if [[ "${testing_enabled}" == "true" ]]; then + for addon in ${KUD_ADDONS:-ovn4nfv}; do + pushd $kud_tests + bash ${addon}.sh + popd + done + fi + echo "Add-ons deployment complete..." +} + +# _print_kubernetes_info() - Prints the login Kubernetes information +function _print_kubernetes_info { + if ! $(kubectl version &>/dev/null); then + return + fi + # Expose Dashboard using NodePort + node_port=30080 + KUBE_EDITOR="sed -i \"s|type\: ClusterIP|type\: NodePort|g\"" kubectl -n kube-system edit service kubernetes-dashboard + KUBE_EDITOR="sed -i \"s|nodePort\: .*|nodePort\: $node_port|g\"" kubectl -n kube-system edit service kubernetes-dashboard + + master_ip=$(kubectl cluster-info | grep "Kubernetes master" | awk -F ":" '{print $2}') + + printf "Kubernetes Info\n===============\n" > $k8s_info_file + echo "Dashboard URL: https:$master_ip:$node_port" >> $k8s_info_file + echo "Admin user: kube" >> $k8s_info_file + echo "Admin password: secret" >> $k8s_info_file +} + +sudo -k # forgot sudo password +if ! sudo -n "true"; then + echo "" + echo "passwordless sudo is needed for '$(id -nu)' user." + echo "Please fix your /etc/sudoers file. You likely want an" + echo "entry like the following one..." + echo "" + echo "$(id -nu) ALL=(ALL) NOPASSWD: ALL" + exit 1 +fi + +verbose="" +if [[ -n "${KUD_DEBUG:-}" ]]; then + set -o xtrace + verbose="-vvv" +fi + +# Configuration values +log_folder=/var/log/kud +kud_folder=${INSTALLER_DIR}/../k8s/kud/hosting_providers/vagrant +kud_infra_folder=$kud_folder/../../deployment_infra +export kud_inventory_folder=$kud_folder/../../../../edge-b/inventory +kud_inventory=$kud_inventory_folder/hosts.ini +kud_playbooks=$kud_infra_folder/playbooks +kud_tests=$kud_folder/../../tests +k8s_info_file=$kud_folder/k8s_info.log +testing_enabled=${KUD_ENABLE_TESTS:-false} +sudo mkdir -p $log_folder +sudo mkdir -p /opt/csar +sudo chown -R $USER /opt/csar +# Install dependencies +# Setup proxy variables +if [ -f $kud_folder/sources.list ]; then + sudo mv /etc/apt/sources.list /etc/apt/sources.list.backup + sudo cp $kud_folder/sources.list /etc/apt/sources.list +fi +echo "Removing ppa for jonathonf/python-3.6" +sudo ls /etc/apt/sources.list.d/ || true +sudo find /etc/apt/sources.list.d -maxdepth 1 -name '*jonathonf*' -delete || true +sudo apt-get update +install_k8s +_set_environment_file +install_addons diff --git a/platform/test/e2e-test-crd/edge-b/inventory/group_vars/all.yml b/platform/test/e2e-test-crd/edge-b/inventory/group_vars/all.yml new file mode 100644 index 0000000..528430c --- /dev/null +++ b/platform/test/e2e-test-crd/edge-b/inventory/group_vars/all.yml @@ -0,0 +1,2 @@ +kube_log_level: 2 +kubeadm_enabled: true diff --git a/platform/test/e2e-test-crd/edge-b/inventory/group_vars/k8s-cluster.yml b/platform/test/e2e-test-crd/edge-b/inventory/group_vars/k8s-cluster.yml new file mode 100644 index 0000000..6f1c40b --- /dev/null +++ b/platform/test/e2e-test-crd/edge-b/inventory/group_vars/k8s-cluster.yml @@ -0,0 +1,82 @@ +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Kubernetes configuration dirs and system namespace. +# Those are where all the additional config stuff goes +# kubernetes normally puts in /srv/kubernetes. +# This puts them in a sane location and namespace. +# Editing those values will almost surely break something. +system_namespace: kube-system + +# Logging directory (sysvinit systems) +kube_log_dir: "/var/log/kubernetes" + +kube_api_anonymous_auth: true + +# Users to create for basic auth in Kubernetes API via HTTP +# Optionally add groups for user +kube_api_pwd: "secret" +kube_users: + kube: + pass: "{{kube_api_pwd}}" + role: admin + groups: + - system:masters + +## It is possible to activate / deactivate selected authentication methods (basic auth, static token auth) +#kube_oidc_auth: false +kube_basic_auth: true +kube_token_auth: true + +# Choose network plugin (calico, contiv, weave or flannel) +# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing +kube_network_plugin: flannel + +# Make a copy of kubeconfig (admin.conf) on the host that runs Ansible to inventory/artifacts +kubeconfig_localhost: true +# Copy kubectl binary on the host that runs Ansible to inventory/artifacts +kubectl_localhost: true +# Disable nodelocal dns cache +enable_nodelocaldns: false +# Enable MountPropagation gate feature +local_volumes_enabled: true +local_volume_provisioner_enabled: true + +## Change this to use another Kubernetes version, e.g. a current beta release +kube_version: v1.16.9 + +# Helm deployment +helm_enabled: true + +# Kube-proxy proxyMode configuration. +# NOTE: Ipvs is based on netfilter hook function, but uses hash table as the underlying data structure and +# works in the kernel space +# https://kubernetes.io/docs/concepts/services-networking/service/#proxy-mode-ipvs +#kube_proxy_mode: ipvs + +# Download container images only once then push to cluster nodes in batches +download_run_once: true + +# Where the binaries will be downloaded. +# Note: ensure that you've enough disk space (about 1G) +local_release_dir: "/tmp/releases" + +# Makes the installer node a delegate for pushing images while running +# the deployment with ansible. This maybe the case if cluster nodes +# cannot access each over via ssh or you want to use local docker +# images as a cache for multiple clusters. +download_localhost: true + +# Subnet for cluster IPs +kube_service_addresses: 10.244.0.0/18 +# Subnet for Pod IPs +kube_pods_subnet: 10.244.64.0/18 + +# Add specific docker version +docker_version: 'latest' diff --git a/platform/test/e2e-test-crd/edge-b/node.sh b/platform/test/e2e-test-crd/edge-b/node.sh new file mode 100755 index 0000000..a51be19 --- /dev/null +++ b/platform/test/e2e-test-crd/edge-b/node.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o nounset +set -o pipefail + +# usage() - Prints the usage of the program +function usage { + cat <> /etc/fstab +} + +while getopts "h?v:" opt; do + case $opt in + v) + dict_volumes="$OPTARG" + ;; + h|\?) + usage + exit + ;; + esac +done + +swapoff -a +if [[ -n "${dict_volumes+x}" ]]; then + for kv in ${dict_volumes//,/ } ;do + mount_external_partition ${kv%=*} ${kv#*=} + done +fi + +vendor_id=$(lscpu|grep "Vendor ID") +if [[ $vendor_id == *GenuineIntel* ]]; then + kvm_ok=$(cat /sys/module/kvm_intel/parameters/nested) + if [[ $kvm_ok == 'N' ]]; then + echo "Enable Intel Nested-Virtualization" + rmmod kvm-intel + echo 'options kvm-intel nested=y' >> /etc/modprobe.d/dist.conf + modprobe kvm-intel + echo kvm-intel >> /etc/modules + fi +else + kvm_ok=$(cat /sys/module/kvm_amd/parameters/nested) + if [[ $kvm_ok == '0' ]]; then + echo "Enable AMD Nested-Virtualization" + rmmod kvm-amd + sh -c "echo 'options kvm-amd nested=1' >> /etc/modprobe.d/dist.conf" + modprobe kvm-amd + echo kvm-amd >> /etc/modules + fi +fi +modprobe vhost_net +echo vhost_net >> /etc/modules +source /etc/os-release || source /usr/lib/os-release +case ${ID,,} in + *suse) + ;; + ubuntu|debian) + apt-get install -y cpu-checker + kvm-ok + ;; + rhel|centos|fedora) + ;; +esac diff --git a/platform/test/e2e-test-crd/edge-b/scripts/edge-b-sdwan-config b/platform/test/e2e-test-crd/edge-b/scripts/edge-b-sdwan-config new file mode 100644 index 0000000..0879b7a --- /dev/null +++ b/platform/test/e2e-test-crd/edge-b/scripts/edge-b-sdwan-config @@ -0,0 +1,21 @@ +#Adding DNAT... +cat > $sdwanConfigs.yaml << EOF +apiVersion: batch.sdewan.akraino.org/v1alpha1 +kind: FirewallDNAT +metadata: + name: firewalldnat + namespace: default + labels: + sdewanPurpose: $sdewan_cnf_name +spec: + src: pnetwork + src_ip: $internal_subnet + src_dip: $sdwan_pod_virtual_ip + src_dport: "80" + dest_ip: $appLan0 + proto: tcp + target: DNAT + +EOF + + diff --git a/platform/test/e2e-test-crd/edge-b/scripts/variables b/platform/test/e2e-test-crd/edge-b/scripts/variables new file mode 100644 index 0000000..73e523d --- /dev/null +++ b/platform/test/e2e-test-crd/edge-b/scripts/variables @@ -0,0 +1,19 @@ +providerSubnet=10.10.10.1/24 +providerGateway=10.10.10.1/24 +providerExcludeIps=10.10.10.2..10.10.10.9 +providerNetworkInterface=eth1 +ovnGateway=172.16.31.1/24 +ovnExcludeIps=172.16.31.2..172.16.31.9 +cnfWanGateway=10.10.10.1 + +export cnfWan0=10.10.10.25 +export cnfLan0=172.16.31.10 +export appLan0=172.16.31.15 +export hubIp=10.10.10.35 +export ovnSubnet=172.16.31.1/24 +export edgeName=edgeB +export wan_interface=net1 +export sdewan_cnf_name=sdewan-edge-b +export app_pod_name=simple-http-service +export sdwanConfigs=edge-b-sdwan-config +export internal_subnet=192.168.1.1/24 diff --git a/platform/test/e2e-test-crd/edge-scripts/helm-tmp/cnf/.helmignore b/platform/test/e2e-test-crd/edge-scripts/helm-tmp/cnf/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/platform/test/e2e-test-crd/edge-scripts/helm-tmp/cnf/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/platform/test/e2e-test-crd/edge-scripts/helm-tmp/cnf/Chart.yaml b/platform/test/e2e-test-crd/edge-scripts/helm-tmp/cnf/Chart.yaml new file mode 100644 index 0000000..13085f2 --- /dev/null +++ b/platform/test/e2e-test-crd/edge-scripts/helm-tmp/cnf/Chart.yaml @@ -0,0 +1,21 @@ +#/* +# * Copyright 2019 Intel Corporation, Inc +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +apiVersion: v1 +name: cnf +description: A Helm chart for Kubernetes +version: 0.1.0 +appVersion: "1.0" diff --git a/platform/test/e2e-test-crd/edge-scripts/helm-tmp/cnf/templates/_helpers.tpl b/platform/test/e2e-test-crd/edge-scripts/helm-tmp/cnf/templates/_helpers.tpl new file mode 100644 index 0000000..e83bd22 --- /dev/null +++ b/platform/test/e2e-test-crd/edge-scripts/helm-tmp/cnf/templates/_helpers.tpl @@ -0,0 +1,63 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "cnf.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "cnf.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cnf.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "cnf.labels" -}} +helm.sh/chart: {{ include "cnf.chart" . }} +{{ include "cnf.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "cnf.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cnf.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "cnf.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "cnf.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/platform/test/e2e-test-crd/edge-scripts/helm-tmp/cnf/templates/cm.yaml b/platform/test/e2e-test-crd/edge-scripts/helm-tmp/cnf/templates/cm.yaml new file mode 100644 index 0000000..e15dd5c --- /dev/null +++ b/platform/test/e2e-test-crd/edge-scripts/helm-tmp/cnf/templates/cm.yaml @@ -0,0 +1,77 @@ +#/* Copyright 2020 Intel Corporation, Inc +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +apiVersion: v1 +data: + entrypoint.sh: |- + #!/bin/bash + # Always exit on errors. + set -ex + echo "" > /etc/config/network + cat > /etc/config/mwan3 <> /etc/config/network <> /etc/config/mwan3 < network-prepare.yaml << EOF +--- +apiVersion: k8s.plugin.opnfv.org/v1alpha1 +kind: ProviderNetwork +metadata: + name: pnetwork +spec: + cniType: ovn4nfv + ipv4Subnets: + - subnet: $providerSubnet + name: subnet + gateway: $providerGateway + excludeIps: $providerExcludeIps + providerNetType: DIRECT + direct: + providerInterfaceName: $providerNetworkInterface + directNodeSelector: all + +--- +apiVersion: k8s.plugin.opnfv.org/v1alpha1 +kind: Network +metadata: + name: ovn-network +spec: + # Add fields here + cniType: ovn4nfv + ipv4Subnets: + - subnet: $ovnSubnet + name: subnet1 + gateway: $ovnGateway + excludeIps: $ovnExcludeIps + +EOF + +kubectl apply -f network-prepare.yaml +sleep 2 + +ovnNet=$(kubectl get network | sed -n 2p | awk '{print $1}') +ovnProviderNet=$(kubectl get providernetwork | sed -n 2p | awk '{print $1}') +if [ -n "${ovnNet}" ] && [ -n "${ovnProviderNet}" ] +then + echo "Networks created successfully" +else + echo "Networks creation failed" + exit 1 +fi + +echo "--------------------- Installing sdewan cnf with helm ---------------------" +curl https://helm.baltorepo.com/organization/signing.asc | sudo apt-key add - +sudo apt-get install apt-transport-https --yes +echo "deb https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list +sudo apt-get update +sudo apt-get install helm + +envsubst < ./cnf/values.yaml >> ./cnf/value.yaml +mv ./cnf/value.yaml ./cnf/values.yaml +helm init +helm package ./cnf +helm install ./cnf-0.1.0.tgz +sleep 20 + +sdwan_status=$(kubectl get po | grep $sdewan_cnf_name | awk '{print $3}' | head -1) +if [ "$sdwan_status" == "Running" ] +then + echo "Sdewan cnf $sdewan_cnf_name created successfully" +else + sleep 40 + sdwan_status=$(kubectl get po | grep $sdewan_cnf_name | awk '{print $3}' | head -1) + if [ "$sdwan_status" != "Running" ] + then + echo "Sdewan cnf creation failed" + exit 2 + fi +fi + +echo "--------------------- Setup Application ---------------------" + +cat > httpbin-svc.yaml << EOF +--- +apiVersion: v1 +kind: Service +metadata: + name: my-http-service +spec: + selector: + app: MyApp + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: $app_pod_name +spec: + replicas: 1 + selector: + matchLabels: + name: simple-http-service + template: + metadata: + labels: + name: simple-http-service + app: MyApp + annotations: + k8s.v1.cni.cncf.io/networks: '[{ "name": "ovn-networkobj"}]' + k8s.plugin.opnfv.org/nfn-network: '{ "type": "ovn4nfv", "interface": [{"name": "ovn-network", "interface": "net0", "ipAddress": "$appLan0"}]}' + spec: + containers: + - name: simple-http-service + image: integratedcloudnative/httpbin:test + ports: + - containerPort: 80 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true +EOF +kubectl apply -f httpbin-svc.yaml +sleep 20 + +appStatus=$(kubectl get po | grep simple-http-service | awk '{print $3}') +if [ "$appStatus" == "Running" ] +then + echo "Application $app_pod_name installation success" +else + sleep 40 + appStatus=$(kubectl get po | grep simple-http-service | awk '{print $3}') + if [ "$appStatus" != "Running" ] + then + echo "Application creation failed" + exit 1 + fi +fi + +echo "--------------------- Setup sdewan controller with helm ---------------------" +helm package ./controllers +helm install controllers-0.1.0.tgz +sleep 1m + +echo "--------------------- Applying IPSec CRDs ---------------------" +cat > ipsec_proposal.yaml << EOF +--- +apiVersion: batch.sdewan.akraino.org/v1alpha1 +kind: IpsecProposal +metadata: + name: ipsecproposal + namespace: default + labels: + sdewanPurpose: $sdewan_cnf_name +spec: + dh_group: modp3072 + encryption_algorithm: aes128 + hash_algorithm: sha256 + +EOF + +kubectl apply -f ipsec_proposal.yaml + +cat > ipsec_config.yaml << EOF +--- +apiVersion: batch.sdewan.akraino.org/v1alpha1 +kind: IpsecHost +metadata: + name: ipsechost + namespace: default + labels: + sdewanPurpose: $sdewan_cnf_name +spec: + name: $edgeName + remote: $hubIp + pre_shared_key: test_key + authentication_method: psk + local_identifier: $cnfWan0 + crypto_proposal: + - ipsecproposal + force_crypto_proposal: "0" + connections: + - name: connA + conn_type: tunnel + mode: start + local_sourceip: "%config" + remote_subnet: 192.168.1.1/24,$hubIp/32 + crypto_proposal: + - ipsecproposal + +EOF + +kubectl apply -f ipsec_config.yaml +sleep 10 + +export sdewan_cnf=$(kubectl get po | grep $sdewan_cnf_name | awk '{print $1}' | head -1) +export sdwan_pod_virtual_ip=$(kubectl exec -it $sdewan_cnf ip address | grep $wan_interface | awk '/inet/{print $2}' | cut -f1 -d "/" | grep 192.168) +echo "IPSec tunnel established. $sdewan_cnf_name gets the virtual ip $sdwan_pod_virtual_ip" + +echo "--------------------- Adding firewall rules ---------------------" +cat > firewall_zone.yaml << EOF +apiVersion: batch.sdewan.akraino.org/v1alpha1 +kind: FirewallZone +metadata: + name: ovnnetwork + namespace: default + labels: + sdewanPurpose: $sdewan_cnf_name +spec: + network: + - ovn-network + input: ACCEPT + output: ACCEPT + forward: ACCEPT +EOF + +kubectl apply -f firewall_zone.yaml + +cat > firewall_zone_1.yaml << EOF +apiVersion: batch.sdewan.akraino.org/v1alpha1 +kind: FirewallZone +metadata: + name: pnetwork + namespace: default + labels: + sdewanPurpose: $sdewan_cnf_name +spec: + network: + - pnetwork + input: ACCEPT + output: ACCEPT + forward: REJECT + masq: "0" + mtu_fix: "1" +EOF + +kubectl apply -f firewall_zone_1.yaml + +echo "--------------------- Adding NAT rules ---------------------" +test -f $sdwanConfigs && . $sdwanConfigs +kubectl apply -f $sdwanConfigs.yaml + +app_pod=$(kubectl get po | grep $app_pod_name | cut -f1 -d " ") +kubectl exec -it $app_pod ip route add 192.168.1.0/24 via $cnfLan0 dev net0 + +echo "Configuration finished" diff --git a/platform/test/e2e-test-crd/sdewan-hub/Vagrantfile b/platform/test/e2e-test-crd/sdewan-hub/Vagrantfile new file mode 100644 index 0000000..21aabb1 --- /dev/null +++ b/platform/test/e2e-test-crd/sdewan-hub/Vagrantfile @@ -0,0 +1,150 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +box = { + :virtualbox => { :name => 'elastic/ubuntu-18.04-x86_64', :version => '20191013.0.0'}, + :libvirt => { :name => 'intergratedcloudnative/ubuntu1804', :version => '1.0.0'} +} + +require 'yaml' +pdf = File.dirname(__FILE__) + '/config/default.yml' +if File.exist?(File.dirname(__FILE__) + '/config/pdf.yml') + pdf = File.dirname(__FILE__) + '/config/pdf.yml' +end +nodes = YAML.load_file(pdf) + +# Inventory file creation +File.open(File.dirname(__FILE__) + "/inventory/hosts.ini", "w") do |inventory_file| + inventory_file.puts("[all]") + nodes.each do |node| + inventory_file.puts("#{node['name']}\tansible_ssh_host=#{node['ip']} ansible_ssh_port=22") + end + ['kube-master', 'kube-node', 'etcd', 'ovn-central', 'ovn-controller', 'virtlet', 'cmk'].each do|group| + inventory_file.puts("\n[#{group}]") + nodes.each do |node| + if node['roles'].include?("#{group}") + inventory_file.puts(node['name']) + end + end + end + inventory_file.puts("\n[k8s-cluster:children]\nkube-node\nkube-master") +end + +provider = (ENV['VAGRANT_DEFAULT_PROVIDER'] || :libvirt).to_sym +puts "[INFO] Provider: #{provider} " + +if ENV['no_proxy'] != nil or ENV['NO_PROXY'] + $no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || "127.0.0.1,localhost" + nodes.each do |node| + $no_proxy += "," + node['ip'] + end + $subnet = "192.168.121" + if provider == :virtualbox + $subnet = "10.0.2" + end + # NOTE: This range is based on vagrant-libvirt network definition CIDR 192.168.121.0/27 + (1..31).each do |i| + $no_proxy += ",#{$subnet}.#{i}" + end +end + +Vagrant.configure("2") do |config| + config.vm.box = box[provider][:name] + config.vm.box_version = box[provider][:version] + config.ssh.insert_key = false + + if ENV['http_proxy'] != nil and ENV['https_proxy'] != nil + if Vagrant.has_plugin?('vagrant-proxyconf') + config.proxy.http = ENV['http_proxy'] || ENV['HTTP_PROXY'] || "" + config.proxy.https = ENV['https_proxy'] || ENV['HTTPS_PROXY'] || "" + config.proxy.no_proxy = $no_proxy + config.proxy.enabled = { docker: false } + end + end + config.vm.provider 'libvirt' do |v| + v.nested = true + v.cpu_mode = 'host-passthrough' + v.management_network_address = "192.168.121.0/27" + v.random_hostname = true + end + + sync_type = "virtualbox" + if provider == :libvirt + sync_type = "nfs" + end + + nodes.each do |node| + config.vm.define node['name'] do |nodeconfig| + if node['roles'].include?("kube-master") + nodeconfig.vm.synced_folder '../', '/home/vagrant/multicloud-k8s/', type: sync_type + end + if node['roles'].include?("kube-node") + nodeconfig.vm.provision 'shell', privileged: false do |sh| + sh.inline = <<-SHELL + sudo sed -i 's:GRUB_CMDLINE_LINUX=.*:GRUB_CMDLINE_LINUX="isolcpus=0-7":' /etc/default/grub + sudo update-grub + SHELL + end + end + nodeconfig.vm.hostname = node['name'] + nodeconfig.vm.network :private_network, :ip => node['ip'], :type => :static + nodeconfig.vm.provider 'virtualbox' do |v| + v.customize ["modifyvm", :id, "--memory", node['memory']] + v.customize ["modifyvm", :id, "--cpus", node['cpus']] + if node.has_key? "volumes" + node['volumes'].each do |volume| + $volume_file = "#{node['name']}-#{volume['name']}.vdi" + unless File.exist?($volume_file) + v.customize ['createmedium', 'disk', '--filename', $volume_file, '--size', volume['size']] + end + v.customize ['storageattach', :id, '--storagectl', 'IDE Controller', '--port', 1, '--device', 0, '--type', 'hdd', '--medium', $volume_file] + end + end + end + nodeconfig.vm.provider 'libvirt' do |v| + v.memory = node['memory'] + v.cpus = node['cpus'] + nodeconfig.vm.provision 'file', source: './scripts', destination: '$HOME/' + nodeconfig.vm.provision 'file', source: './scripts/cnf', destination: '$HOME/scripts/cnf' + nodeconfig.vm.provision 'file', source: '../edge-scripts/helm-tmp/controllers', destination: '$HOME/scripts/controllers' + nodeconfig.vm.provision 'shell' do |sh| + sh.path = "node.sh" + if node.has_key? "volumes" + $volume_mounts_dict = '' + node['volumes'].each do |volume| + $volume_mounts_dict += "#{volume['name']}=#{volume['mount']}," + $volume_file = "./#{node['name']}-#{volume['name']}.qcow2" + v.storage :file, :bus => 'sata', :device => volume['name'], :size => volume['size'] + end + sh.args = ['-v', $volume_mounts_dict[0...-1]] + end + end + end + end + end + + config.vm.define :installer, primary: true, autostart: false do |installer| + installer.vm.hostname = "multicloud" + installer.vm.network :private_network, :ip => "10.10.10.2", :type => :static + installer.vm.synced_folder '../', '/home/vagrant/multicloud-k8s/', type: sync_type + installer.vm.provision 'shell', privileged: false do |sh| + sh.env = {'KUD_PLUGIN_ENABLED': 'false', 'OVN_CENTRAL_INTERFACE': 'eth1'} + sh.inline = <<-SHELL + cp /home/vagrant/multicloud-k8s/sdewan-hub/insecure_keys/key.pub /home/vagrant/.ssh/id_rsa.pub + cp /home/vagrant/multicloud-k8s/sdewan-hub/insecure_keys/key /home/vagrant/.ssh/id_rsa + chown vagrant /home/vagrant/.ssh/id_rsa + chmod 400 /home/vagrant/.ssh/id_rsa + sudo apt install jq -y + cd /home/vagrant/multicloud-k8s/sdewan-hub/ && ./installer.sh | tee kud_installer.log + SHELL + end + end +end diff --git a/platform/test/e2e-test-crd/sdewan-hub/config/default.yml b/platform/test/e2e-test-crd/sdewan-hub/config/default.yml new file mode 100644 index 0000000..c63d548 --- /dev/null +++ b/platform/test/e2e-test-crd/sdewan-hub/config/default.yml @@ -0,0 +1,25 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- name: "sdewan-hub" + ip: "10.10.10.5" + memory: 18432 + cpus: 16 + volumes: + - name: sdb + size: 50 + mount: /var/lib/docker/ + roles: + - kube-master + - etcd + - ovn-central + - kube-node + - ovn-controller + diff --git a/platform/test/e2e-test-crd/sdewan-hub/insecure_keys/key b/platform/test/e2e-test-crd/sdewan-hub/insecure_keys/key new file mode 100644 index 0000000..7d6a083 --- /dev/null +++ b/platform/test/e2e-test-crd/sdewan-hub/insecure_keys/key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI +w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP +kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2 +hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO +Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW +yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd +ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1 +Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf +TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK +iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A +sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf +4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP +cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk +EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN +CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX +3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG +YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj +3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+ +dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz +6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC +P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF +llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ +kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH ++vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ +NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s= +-----END RSA PRIVATE KEY----- diff --git a/platform/test/e2e-test-crd/sdewan-hub/insecure_keys/key.pub b/platform/test/e2e-test-crd/sdewan-hub/insecure_keys/key.pub new file mode 100644 index 0000000..18a9c00 --- /dev/null +++ b/platform/test/e2e-test-crd/sdewan-hub/insecure_keys/key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key diff --git a/platform/test/e2e-test-crd/sdewan-hub/installer.sh b/platform/test/e2e-test-crd/sdewan-hub/installer.sh new file mode 100755 index 0000000..5f509a7 --- /dev/null +++ b/platform/test/e2e-test-crd/sdewan-hub/installer.sh @@ -0,0 +1,233 @@ +#!/bin/bash +#SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o errexit +set -o nounset +set -o pipefail + +INSTALLER_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" + +source ${INSTALLER_DIR}/../k8s/kud/tests/_functions.sh + +# _install_go() - Install GoLang package +function _install_go { + version=$(grep "go_version" ${kud_playbooks}/kud-vars.yml | awk -F "'" '{print $2}') + local tarball=go$version.linux-amd64.tar.gz + + #gcc is required for go apps compilation + if ! which gcc; then + sudo apt-get install -y gcc + fi + + if $(go version &>/dev/null); then + return + fi + + wget https://dl.google.com/go/$tarball + sudo tar -C /usr/local -xzf $tarball + rm $tarball + + export PATH=$PATH:/usr/local/go/bin + sudo sed -i "s|^PATH=.*|PATH=\"$PATH\"|" /etc/environment +} + +# _install_pip() - Install Python Package Manager +function _install_pip { + if $(pip --version &>/dev/null); then + sudo -E pip install --upgrade pip + else + sudo apt-get install -y python-dev + curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python + fi +} + +# _install_ansible() - Install and Configure Ansible program +function _install_ansible { + if $(ansible --version &>/dev/null); then + sudo pip uninstall -y ansible + fi + _install_pip + local version=$(grep "ansible_version" ${kud_playbooks}/kud-vars.yml | awk -F ': ' '{print $2}') + sudo mkdir -p /etc/ansible/ + sudo -E pip install ansible==$version +} + +# _install_docker() - Download and install docker-engine +function _install_docker { + local max_concurrent_downloads=${1:-3} + + if $(docker version &>/dev/null); then + return + fi + sudo apt-get install -y apt-transport-https ca-certificates curl + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + sudo apt-get update + sudo apt-get install -y docker-ce + + sudo mkdir -p /etc/systemd/system/docker.service.d + if [ ${http_proxy:-} ]; then + echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/http-proxy.conf + echo "Environment=\"HTTP_PROXY=$http_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/http-proxy.conf + fi + if [ ${https_proxy:-} ]; then + echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/https-proxy.conf + echo "Environment=\"HTTPS_PROXY=$https_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/https-proxy.conf + fi + if [ ${no_proxy:-} ]; then + echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/no-proxy.conf + echo "Environment=\"NO_PROXY=$no_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/no-proxy.conf + fi + sudo systemctl daemon-reload + echo "DOCKER_OPTS=\"-H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock --max-concurrent-downloads $max_concurrent_downloads \"" | sudo tee --append /etc/default/docker + if [[ -z $(groups | grep docker) ]]; then + sudo usermod -aG docker $USER + fi + + sudo systemctl restart docker + sleep 10 +} + +function _set_environment_file { + # By default ovn central interface is the first active network interface on localhost. If other wanted, need to export this variable in aio.sh or Vagrant file. + OVN_CENTRAL_INTERFACE="${OVN_CENTRAL_INTERFACE:-$(ip addr show | awk '/inet.*brd/{print $NF; exit}')}" + echo "export OVN_CENTRAL_INTERFACE=${OVN_CENTRAL_INTERFACE}" | sudo tee --append /etc/environment + echo "export OVN_CENTRAL_ADDRESS=$(get_ovn_central_address)" | sudo tee --append /etc/environment + echo "export KUBE_CONFIG_DIR=/opt/kubeconfig" | sudo tee --append /etc/environment + echo "export CSAR_DIR=/opt/csar" | sudo tee --append /etc/environment +} + +# install_k8s() - Install Kubernetes using kubespray tool +function install_k8s { + echo "Deploying kubernetes" + local dest_folder=/opt + version=$(grep "kubespray_version" ${kud_playbooks}/kud-vars.yml | awk -F ': ' '{print $2}') + local_release_dir=$(grep "local_release_dir" $kud_inventory_folder/group_vars/k8s-cluster.yml | awk -F "\"" '{print $2}') + local tarball=v$version.tar.gz + sudo apt-get install -y sshpass make unzip # install make to run mitogen target and unzip is mitogen playbook dependency + sudo apt-get install -y gnupg2 software-properties-common + _install_docker + _install_ansible + wget https://github.com/kubernetes-incubator/kubespray/archive/$tarball + sudo tar -C $dest_folder -xzf $tarball + sudo mv $dest_folder/kubespray-$version/ansible.cfg /etc/ansible/ansible.cfg + sudo chown -R $USER $dest_folder/kubespray-$version + sudo mkdir -p ${local_release_dir}/containers + rm $tarball + + pushd $dest_folder/kubespray-$version/ + sudo -E pip install -r ./requirements.txt + make mitogen + popd + rm -f $kud_inventory_folder/group_vars/all.yml 2> /dev/null + if [[ -n "${verbose:-}" ]]; then + echo "kube_log_level: 5" | tee $kud_inventory_folder/group_vars/all.yml + else + echo "kube_log_level: 2" | tee $kud_inventory_folder/group_vars/all.yml + fi + echo "kubeadm_enabled: true" | tee --append $kud_inventory_folder/group_vars/all.yml + if [[ -n "${http_proxy:-}" ]]; then + echo "http_proxy: \"$http_proxy\"" | tee --append $kud_inventory_folder/group_vars/all.yml + fi + if [[ -n "${https_proxy:-}" ]]; then + echo "https_proxy: \"$https_proxy\"" | tee --append $kud_inventory_folder/group_vars/all.yml + fi + ansible-playbook $verbose -i $kud_inventory $dest_folder/kubespray-$version/cluster.yml --become --become-user=root | sudo tee $log_folder/setup-kubernetes.log + + # Configure environment + mkdir -p $HOME/.kube + cp $kud_inventory_folder/artifacts/admin.conf $HOME/.kube/config + # Copy Kubespray kubectl to be usable in host running Ansible. Requires kubectl_localhost: true in inventory/group_vars/k8s-cluster.yml + sudo cp $kud_inventory_folder/artifacts/kubectl /usr/local/bin/ +} + +# install_addons() - Install Kubenertes AddOns +function install_addons { + source /etc/environment + echo "Installing Kubernetes AddOns" + _install_ansible + sudo ansible-galaxy install $verbose -r $kud_infra_folder/galaxy-requirements.yml --ignore-errors + ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-kud.yml | sudo tee $log_folder/setup-kud.log + for addon in ${KUD_ADDONS:-ovn4nfv}; do + echo "Deploying $addon using configure-$addon.yml playbook.." + ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-${addon}.yml | sudo tee $log_folder/setup-${addon}.log + done + echo "Run the test cases if testing_enabled is set to true." + if [[ "${testing_enabled}" == "true" ]]; then + for addon in ${KUD_ADDONS:-ovn4nfv}; do + pushd $kud_tests + bash ${addon}.sh + popd + done + fi + echo "Add-ons deployment complete..." +} + +# _print_kubernetes_info() - Prints the login Kubernetes information +function _print_kubernetes_info { + if ! $(kubectl version &>/dev/null); then + return + fi + # Expose Dashboard using NodePort + node_port=30080 + KUBE_EDITOR="sed -i \"s|type\: ClusterIP|type\: NodePort|g\"" kubectl -n kube-system edit service kubernetes-dashboard + KUBE_EDITOR="sed -i \"s|nodePort\: .*|nodePort\: $node_port|g\"" kubectl -n kube-system edit service kubernetes-dashboard + + master_ip=$(kubectl cluster-info | grep "Kubernetes master" | awk -F ":" '{print $2}') + + printf "Kubernetes Info\n===============\n" > $k8s_info_file + echo "Dashboard URL: https:$master_ip:$node_port" >> $k8s_info_file + echo "Admin user: kube" >> $k8s_info_file + echo "Admin password: secret" >> $k8s_info_file +} + +sudo -k # forgot sudo password +if ! sudo -n "true"; then + echo "" + echo "passwordless sudo is needed for '$(id -nu)' user." + echo "Please fix your /etc/sudoers file. You likely want an" + echo "entry like the following one..." + echo "" + echo "$(id -nu) ALL=(ALL) NOPASSWD: ALL" + exit 1 +fi + +verbose="" +if [[ -n "${KUD_DEBUG:-}" ]]; then + set -o xtrace + verbose="-vvv" +fi + +# Configuration values +log_folder=/var/log/kud +kud_folder=${INSTALLER_DIR}/../k8s/kud/hosting_providers/vagrant +kud_infra_folder=$kud_folder/../../deployment_infra +export kud_inventory_folder=$kud_folder/../../../../sdewan-hub/inventory +kud_inventory=$kud_inventory_folder/hosts.ini +kud_playbooks=$kud_infra_folder/playbooks +kud_tests=$kud_folder/../../tests +k8s_info_file=$kud_folder/k8s_info.log +testing_enabled=${KUD_ENABLE_TESTS:-false} +sudo mkdir -p $log_folder +sudo mkdir -p /opt/csar +sudo chown -R $USER /opt/csar +# Install dependencies +# Setup proxy variables +if [ -f $kud_folder/sources.list ]; then + sudo mv /etc/apt/sources.list /etc/apt/sources.list.backup + sudo cp $kud_folder/sources.list /etc/apt/sources.list +fi +echo "Removing ppa for jonathonf/python-3.6" +sudo ls /etc/apt/sources.list.d/ || true +sudo find /etc/apt/sources.list.d -maxdepth 1 -name '*jonathonf*' -delete || true +sudo apt-get update +install_k8s +_set_environment_file +install_addons diff --git a/platform/test/e2e-test-crd/sdewan-hub/inventory/group_vars/all.yml b/platform/test/e2e-test-crd/sdewan-hub/inventory/group_vars/all.yml new file mode 100644 index 0000000..528430c --- /dev/null +++ b/platform/test/e2e-test-crd/sdewan-hub/inventory/group_vars/all.yml @@ -0,0 +1,2 @@ +kube_log_level: 2 +kubeadm_enabled: true diff --git a/platform/test/e2e-test-crd/sdewan-hub/inventory/group_vars/k8s-cluster.yml b/platform/test/e2e-test-crd/sdewan-hub/inventory/group_vars/k8s-cluster.yml new file mode 100644 index 0000000..6f1c40b --- /dev/null +++ b/platform/test/e2e-test-crd/sdewan-hub/inventory/group_vars/k8s-cluster.yml @@ -0,0 +1,82 @@ +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Kubernetes configuration dirs and system namespace. +# Those are where all the additional config stuff goes +# kubernetes normally puts in /srv/kubernetes. +# This puts them in a sane location and namespace. +# Editing those values will almost surely break something. +system_namespace: kube-system + +# Logging directory (sysvinit systems) +kube_log_dir: "/var/log/kubernetes" + +kube_api_anonymous_auth: true + +# Users to create for basic auth in Kubernetes API via HTTP +# Optionally add groups for user +kube_api_pwd: "secret" +kube_users: + kube: + pass: "{{kube_api_pwd}}" + role: admin + groups: + - system:masters + +## It is possible to activate / deactivate selected authentication methods (basic auth, static token auth) +#kube_oidc_auth: false +kube_basic_auth: true +kube_token_auth: true + +# Choose network plugin (calico, contiv, weave or flannel) +# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing +kube_network_plugin: flannel + +# Make a copy of kubeconfig (admin.conf) on the host that runs Ansible to inventory/artifacts +kubeconfig_localhost: true +# Copy kubectl binary on the host that runs Ansible to inventory/artifacts +kubectl_localhost: true +# Disable nodelocal dns cache +enable_nodelocaldns: false +# Enable MountPropagation gate feature +local_volumes_enabled: true +local_volume_provisioner_enabled: true + +## Change this to use another Kubernetes version, e.g. a current beta release +kube_version: v1.16.9 + +# Helm deployment +helm_enabled: true + +# Kube-proxy proxyMode configuration. +# NOTE: Ipvs is based on netfilter hook function, but uses hash table as the underlying data structure and +# works in the kernel space +# https://kubernetes.io/docs/concepts/services-networking/service/#proxy-mode-ipvs +#kube_proxy_mode: ipvs + +# Download container images only once then push to cluster nodes in batches +download_run_once: true + +# Where the binaries will be downloaded. +# Note: ensure that you've enough disk space (about 1G) +local_release_dir: "/tmp/releases" + +# Makes the installer node a delegate for pushing images while running +# the deployment with ansible. This maybe the case if cluster nodes +# cannot access each over via ssh or you want to use local docker +# images as a cache for multiple clusters. +download_localhost: true + +# Subnet for cluster IPs +kube_service_addresses: 10.244.0.0/18 +# Subnet for Pod IPs +kube_pods_subnet: 10.244.64.0/18 + +# Add specific docker version +docker_version: 'latest' diff --git a/platform/test/e2e-test-crd/sdewan-hub/node.sh b/platform/test/e2e-test-crd/sdewan-hub/node.sh new file mode 100755 index 0000000..a51be19 --- /dev/null +++ b/platform/test/e2e-test-crd/sdewan-hub/node.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o nounset +set -o pipefail + +# usage() - Prints the usage of the program +function usage { + cat <> /etc/fstab +} + +while getopts "h?v:" opt; do + case $opt in + v) + dict_volumes="$OPTARG" + ;; + h|\?) + usage + exit + ;; + esac +done + +swapoff -a +if [[ -n "${dict_volumes+x}" ]]; then + for kv in ${dict_volumes//,/ } ;do + mount_external_partition ${kv%=*} ${kv#*=} + done +fi + +vendor_id=$(lscpu|grep "Vendor ID") +if [[ $vendor_id == *GenuineIntel* ]]; then + kvm_ok=$(cat /sys/module/kvm_intel/parameters/nested) + if [[ $kvm_ok == 'N' ]]; then + echo "Enable Intel Nested-Virtualization" + rmmod kvm-intel + echo 'options kvm-intel nested=y' >> /etc/modprobe.d/dist.conf + modprobe kvm-intel + echo kvm-intel >> /etc/modules + fi +else + kvm_ok=$(cat /sys/module/kvm_amd/parameters/nested) + if [[ $kvm_ok == '0' ]]; then + echo "Enable AMD Nested-Virtualization" + rmmod kvm-amd + sh -c "echo 'options kvm-amd nested=1' >> /etc/modprobe.d/dist.conf" + modprobe kvm-amd + echo kvm-amd >> /etc/modules + fi +fi +modprobe vhost_net +echo vhost_net >> /etc/modules +source /etc/os-release || source /usr/lib/os-release +case ${ID,,} in + *suse) + ;; + ubuntu|debian) + apt-get install -y cpu-checker + kvm-ok + ;; + rhel|centos|fedora) + ;; +esac diff --git a/platform/test/e2e-test-crd/sdewan-hub/scripts/cnf/.helmignore b/platform/test/e2e-test-crd/sdewan-hub/scripts/cnf/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/platform/test/e2e-test-crd/sdewan-hub/scripts/cnf/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/platform/test/e2e-test-crd/sdewan-hub/scripts/cnf/Chart.yaml b/platform/test/e2e-test-crd/sdewan-hub/scripts/cnf/Chart.yaml new file mode 100644 index 0000000..13085f2 --- /dev/null +++ b/platform/test/e2e-test-crd/sdewan-hub/scripts/cnf/Chart.yaml @@ -0,0 +1,21 @@ +#/* +# * Copyright 2019 Intel Corporation, Inc +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +apiVersion: v1 +name: cnf +description: A Helm chart for Kubernetes +version: 0.1.0 +appVersion: "1.0" diff --git a/platform/test/e2e-test-crd/sdewan-hub/scripts/cnf/templates/_helpers.tpl b/platform/test/e2e-test-crd/sdewan-hub/scripts/cnf/templates/_helpers.tpl new file mode 100644 index 0000000..e83bd22 --- /dev/null +++ b/platform/test/e2e-test-crd/sdewan-hub/scripts/cnf/templates/_helpers.tpl @@ -0,0 +1,63 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "cnf.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "cnf.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cnf.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "cnf.labels" -}} +helm.sh/chart: {{ include "cnf.chart" . }} +{{ include "cnf.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "cnf.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cnf.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "cnf.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "cnf.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/platform/test/e2e-test-crd/sdewan-hub/scripts/cnf/templates/cm.yaml b/platform/test/e2e-test-crd/sdewan-hub/scripts/cnf/templates/cm.yaml new file mode 100644 index 0000000..e15dd5c --- /dev/null +++ b/platform/test/e2e-test-crd/sdewan-hub/scripts/cnf/templates/cm.yaml @@ -0,0 +1,77 @@ +#/* Copyright 2020 Intel Corporation, Inc +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +apiVersion: v1 +data: + entrypoint.sh: |- + #!/bin/bash + # Always exit on errors. + set -ex + echo "" > /etc/config/network + cat > /etc/config/mwan3 <> /etc/config/network <> /etc/config/mwan3 < network-prepare.yaml << EOF +--- +apiVersion: k8s.plugin.opnfv.org/v1alpha1 +kind: ProviderNetwork +metadata: + name: pnetwork +spec: + cniType: ovn4nfv + ipv4Subnets: + - subnet: $providerSubnet + name: subnet + gateway: $providerGateway + excludeIps: $providerExcludeIps + providerNetType: DIRECT + direct: + providerInterfaceName: $providerNetworkInterface + directNodeSelector: all + +EOF + +kubectl apply -f network-prepare.yaml +sleep 2 + +ovnProviderNet=$(kubectl get providernetwork | sed -n 2p | awk '{print $1}') +if [ -n "${ovnProviderNet}" ] +then + echo "Network created successfully" +else + echo "Network creation failed" + exit 1 +fi + + +echo "--------------------- Creating sdwan-cnf with helm ---------------------" +curl https://helm.baltorepo.com/organization/signing.asc | sudo apt-key add - +sudo apt-get install apt-transport-https --yes +echo "deb https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list +sudo apt-get update +sudo apt-get install helm + +envsubst < ./cnf/values.yaml >> ./cnf/values.yaml +helm package ./cnf +helm install ./cnf-0.1.0.tgz + +sleep 20 + +sdwan_status=$(kubectl get po | grep $sdewan_cnf_name | awk '{print $3}' | head -1) +if [ "$sdwan_status" == "Running" ] +then + echo "Sdewan cnf $sdewan_cnf_name created successfully" +else + sleep 40 + sdwan_status=$(kubectl get po | grep $sdewan_cnf_name | awk '{print $3}' | head -1) + if [ "$sdwan_status" != "Running" ] + then + echo "Sdewan cnf creation failed" + exit 2 + fi +fi + +echo "--------------------- Setup sdewan controller ---------------------" +helm package ./controllers +helm install ./controllers-0.1.0.tgz +sleep 1m + +echo "--------------------- Applying CRDs ---------------------" +cat > ipsec_proposal.yaml << EOF +--- +apiVersion: batch.sdewan.akraino.org/v1alpha1 +kind: IpsecProposal +metadata: + name: ipsecproposal + namespace: default + labels: + sdewanPurpose: $sdewan_cnf_name +spec: + dh_group: modp3072 + encryption_algorithm: aes128 + hash_algorithm: sha256 + +EOF + +kubectl apply -f ipsec_proposal.yaml + +cat > ipsec_config.yaml << EOF +--- +apiVersion: batch.sdewan.akraino.org/v1alpha1 +kind: IpsecSite +metadata: + name: ipsecsite + namespace: default + labels: + sdewanPurpose: $sdewan_cnf_name +spec: + name: sdewan-hub + remote: "%any" + pre_shared_key: test_key + authentication_method: psk + local_identifier: $hubIp + crypto_proposal: + - ipsecproposal + force_crypto_proposal: "0" + connections: + - name: connA + conn_type: tunnel + mode: start + remote_sourceip: "192.168.1.5-192.168.1.6" + local_subnet: 192.168.1.1/24,$hubIp/32 + crypto_proposal: + - ipsecproposal + +EOF + +kubectl apply -f ipsec_config.yaml + +echo "--------------------- Configuration finished ---------------------" diff --git a/platform/test/e2e-test-crd/sdewan-hub/scripts/variables b/platform/test/e2e-test-crd/sdewan-hub/scripts/variables new file mode 100644 index 0000000..a143317 --- /dev/null +++ b/platform/test/e2e-test-crd/sdewan-hub/scripts/variables @@ -0,0 +1,9 @@ +providerSubnet=10.10.10.1/24 +providerGateway=10.10.10.1/24 +providerExcludeIps=10.10.10.2..10.10.10.9 +providerNetworkInterface=eth1 +cnfWanGateway=10.10.10.1 +cnfWan0=10.10.10.35 + +export hubIp=10.10.10.35 +export sdewan_cnf_name=sdewan-cnf-hub diff --git a/platform/test/e2e-test-crd/setup.sh b/platform/test/e2e-test-crd/setup.sh new file mode 100755 index 0000000..9c65ccd --- /dev/null +++ b/platform/test/e2e-test-crd/setup.sh @@ -0,0 +1,201 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o nounset +set -o pipefail + +vagrant_version=2.2.4 +if ! vagrant version &>/dev/null; then + enable_vagrant_install=true +else + if [[ "$vagrant_version" != "$(vagrant version | awk 'NR==1{print $3}')" ]]; then + enable_vagrant_install=true + fi +fi + +function usage { + cat < +Installation of vagrant and its dependencies in Linux OS + +Argument: + -p Vagrant provider +EOF +} + +while getopts ":p:" OPTION; do + case $OPTION in + p) + provider=$OPTARG + ;; + \?) + usage + exit 1 + ;; + esac +done +if [[ -z "${provider+x}" ]]; then + usage + exit 1 +fi + +case $provider in + "virtualbox" | "libvirt" ) + export VAGRANT_DEFAULT_PROVIDER=${provider} + ;; + * ) + usage + exit 1 +esac +source /etc/os-release || source /usr/lib/os-release + +libvirt_group="libvirt" +packages=() +case ${ID,,} in + *suse) + INSTALLER_CMD="sudo -H -E zypper -q install -y --no-recommends" + packages+=(python-devel) + + # Vagrant installation + if [[ "${enable_vagrant_install+x}" ]]; then + vagrant_pgp="pgp_keys.asc" + wget -q https://keybase.io/hashicorp/$vagrant_pgp + wget -q https://releases.hashicorp.com/vagrant/$vagrant_version/vagrant_${vagrant_version}_x86_64.rpm + gpg --quiet --with-fingerprint $vagrant_pgp + sudo rpm --import $vagrant_pgp + sudo rpm --checksig vagrant_${vagrant_version}_x86_64.rpm + sudo rpm --install vagrant_${vagrant_version}_x86_64.rpm + rm vagrant_${vagrant_version}_x86_64.rpm + rm $vagrant_pgp + fi + + case $VAGRANT_DEFAULT_PROVIDER in + virtualbox) + wget -q "http://download.virtualbox.org/virtualbox/rpm/opensuse/$VERSION/virtualbox.repo" -P /etc/zypp/repos.d/ + $INSTALLER_CMD --enablerepo=epel dkms + wget -q https://www.virtualbox.org/download/oracle_vbox.asc -O- | rpm --import - + packages+=(VirtualBox-5.1) + ;; + libvirt) + # vagrant-libvirt dependencies + packages+=(qemu libvirt libvirt-devel ruby-devel gcc qemu-kvm zlib-devel libxml2-devel libxslt-devel make) + # NFS + packages+=(nfs-kernel-server) + ;; + esac + sudo zypper -n ref + ;; + + ubuntu|debian) + libvirt_group="libvirtd" + INSTALLER_CMD="sudo -H -E apt-get -y -q=3 install" + packages+=(python-dev) + + # Vagrant installation + if [[ "${enable_vagrant_install+x}" ]]; then + wget -q https://releases.hashicorp.com/vagrant/$vagrant_version/vagrant_${vagrant_version}_x86_64.deb + sudo dpkg -i vagrant_${vagrant_version}_x86_64.deb + rm vagrant_${vagrant_version}_x86_64.deb + fi + + case $VAGRANT_DEFAULT_PROVIDER in + virtualbox) + echo "deb http://download.virtualbox.org/virtualbox/debian trusty contrib" >> /etc/apt/sources.list + wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add - + wget -q https://www.virtualbox.org/download/oracle_vbox.asc -O- | sudo apt-key add - + packages+=(virtualbox-5.1 dkms) + ;; + libvirt) + # vagrant-libvirt dependencies + packages+=(qemu libvirt-bin ebtables dnsmasq libxslt-dev libxml2-dev libvirt-dev zlib1g-dev ruby-dev cpu-checker) + # NFS + packages+=(nfs-kernel-server) + ;; + esac + sudo apt-get update + ;; + + rhel|centos|fedora) + PKG_MANAGER=$(which dnf || which yum) + sudo "$PKG_MANAGER" updateinfo + INSTALLER_CMD="sudo -H -E ${PKG_MANAGER} -q -y install" + packages+=(python-devel) + + # Vagrant installation + if [[ "${enable_vagrant_install+x}" ]]; then + wget -q https://releases.hashicorp.com/vagrant/$vagrant_version/vagrant_${vagrant_version}_x86_64.rpm + $INSTALLER_CMD vagrant_${vagrant_version}_x86_64.rpm + rm vagrant_${vagrant_version}_x86_64.rpm + fi + + case $VAGRANT_DEFAULT_PROVIDER in + virtualbox) + wget -q http://download.virtualbox.org/virtualbox/rpm/rhel/virtualbox.repo -P /etc/yum.repos.d + $INSTALLER_CMD --enablerepo=epel dkms + wget -q https://www.virtualbox.org/download/oracle_vbox.asc -O- | rpm --import - + packages+=(VirtualBox-5.1) + ;; + libvirt) + # vagrant-libvirt dependencies + packages+=(qemu libvirt libvirt-devel ruby-devel gcc qemu-kvm) + # NFS + packages+=(nfs-utils nfs-utils-lib) + ;; + esac + ;; + +esac + +# Enable Nested-Virtualization +vendor_id=$(lscpu|grep "Vendor ID") +if [[ $vendor_id == *GenuineIntel* ]]; then + kvm_ok=$(cat /sys/module/kvm_intel/parameters/nested) + if [[ $kvm_ok == 'N' ]]; then + echo "Enable Intel Nested-Virtualization" + sudo rmmod kvm-intel + echo 'options kvm-intel nested=y' | sudo tee --append /etc/modprobe.d/dist.conf + sudo modprobe kvm-intel + fi +else + kvm_ok=$(cat /sys/module/kvm_amd/parameters/nested) + if [[ $kvm_ok == '0' ]]; then + echo "Enable AMD Nested-Virtualization" + sudo rmmod kvm-amd + echo 'options kvm-amd nested=1' | sudo tee --append /etc/modprobe.d/dist.conf + sudo modprobe kvm-amd + fi +fi +sudo modprobe vhost_net + +${INSTALLER_CMD} "${packages[@]}" +if ! which pip; then + curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python +else + sudo -H -E pip install --upgrade pip +fi +sudo -H -E pip install tox +if [[ ${http_proxy+x} ]]; then + vagrant plugin install vagrant-proxyconf +fi +if [ "$VAGRANT_DEFAULT_PROVIDER" == libvirt ]; then + vagrant plugin install vagrant-libvirt + sudo usermod -a -G $libvirt_group "$USER" # This might require to reload user's group assigments + sudo systemctl restart libvirtd + + # Start statd service to prevent NFS lock errors + sudo systemctl enable rpc-statd + sudo systemctl start rpc-statd + + case ${ID,,} in + ubuntu|debian) + kvm-ok + ;; + esac +fi diff --git a/platform/test/e2e-test-crd/test.sh b/platform/test/e2e-test-crd/test.sh new file mode 100755 index 0000000..4a025a6 --- /dev/null +++ b/platform/test/e2e-test-crd/test.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +set -o errexit +set -o nounset +set -o pipefail + + +BASE=$(pwd) + + +clean() +{ +# Cleaning the env +echo "Cleaning the environment..." +echo "Deleting the vms..." +for name in edge-a edge-b sdewan-hub +do + cd $BASE/${name} + vagrant destroy -f & + sleep 10 +done +echo "Cleaning completed" +} + +error_report() +{ + echo "Error on line $1" + echo "End-to-end test failed" +} + +trap 'error_report $LINENO' ERR +trap 'clean' EXIT + +# Install dependencies +echo "Installing dependencies..." +sudo ./setup.sh -p libvirt + +# Bring up 3 vms for edge-a, edge-b and sdewan-hub +cd $BASE +git clone http://gerrit.onap.org/r/multicloud/k8s && cd k8s +echo "Bringing up virtual machines for three clusters..." +for name in edge-a edge-b sdewan-hub +do + cd $BASE/${name} + echo "Start up cluster for ${name}..." + vagrant up && vagrant up installer + sleep 40 +done + + +# Checking vm status... +for name in edge-a edge-b sdewan-hub +do + cd $BASE/${name} + vagrant ssh ${name} -- -t 'mkdir -p /home/vagrant/.kube; sudo cp -i /etc/kubernetes/admin.conf /home/vagrant/.kube/config; sudo chown $(id -u):$(id -g) $HOME/.kube/config' + Status=$(vagrant ssh ${name} -- -t 'kubectl get po -n operator | grep 'nfn-agent'' | grep 'nfn-agent' | awk '{print $3}') + if [ $Status != "Running" ] + then + echo "Virtual machine ${name} provision failed" + exit 1 + else + echo "Virtual machine ${name} provision success" + fi +done + +# Setup ipsec tunnels and applications +echo "Setup configs for the e2e scenario..." +for name in sdewan-hub edge-a edge-b +do + cd $BASE/${name} + vagrant ssh ${name} -- -t 'cd /home/vagrant/scripts; ./setup-cnf.sh' +done + + +echo "Testing the connectivity between applications..." +cd $BASE/edge-a +vagrant ssh edge-a -- -t './scripts/test-connection.sh' +sleep 3 -- 2.16.6