From: hle2 <huifeng.le@intel.com>
Date: Tue, 29 Oct 2019 02:45:06 +0000 (+0800)
Subject: Add SDWAN build and test script
X-Git-Tag: v0.4.0~68
X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=commitdiff_plain;h=7bce432e28247710991b633c43a1c9da7eb7cc57;p=icn.git

Add SDWAN build and test script

Signed-off-by: hle2 <huifeng.le@intel.com>
Change-Id: I9cd855dc2361d28cb2b5669b4631f86577026fc3
---

diff --git a/Makefile b/Makefile
index 3a875ca..c0f3824 100644
--- a/Makefile
+++ b/Makefile
@@ -5,6 +5,7 @@ METAL3DIR:=$(CURDIR)/deploy/metal3/scripts
 METAL3VMDIR:=$(CURDIR)/deploy/metal3-vm
 BPA_OPERATOR:=$(CURDIR)/cmd/bpa-operator/
 KUD_PATH:=$(CURDIR)/deploy/kud
+SDWAN_VERIFIER_PATH:=$(CURDIR)/sdwan/test
 BPA_E2E_SETUP:=https://raw.githubusercontent.com/onap/multicloud-k8s/master/kud/hosting_providers/vagrant/setup.sh
 
 help:
@@ -37,6 +38,9 @@ metal3_prerequisite:
 metal3_vm:
 	pushd $(METAL3VMDIR) && make bmh && popd
 
+sdwan_verifier:
+	pushd $(SDWAN_VERIFIER_PATH) && bash sdwan_verifier.sh && popd
+
 bpa_op_install:
 	pushd $(BPA_OPERATOR) && make docker && make deploy && popd
 
@@ -67,6 +71,7 @@ verify_all: prerequisite \
 verifier: verify_all
 
 verify_nestedk8s: prerequisite \
-	kud_bm_deploy
+	kud_bm_deploy \
+	sdwan_verifier
 
 .PHONY: all bm_preinstall bm_install bashate
diff --git a/sdwan/build/Dockerfile_1806_mwan3.tpl b/sdwan/build/Dockerfile_1806_mwan3.tpl
new file mode 100644
index 0000000..85c7d35
--- /dev/null
+++ b/sdwan/build/Dockerfile_1806_mwan3.tpl
@@ -0,0 +1,26 @@
+FROM openwrt-1806-4-base
+
+#EXPOSE 80
+ENV http_proxy={docker_proxy}
+ENV https_proxy={docker_proxy}
+ENV no_proxy=localhost,120.0.0.1,192.168.*
+
+RUN mkdir /var/lock && \
+    opkg update && \
+    opkg install uhttpd-mod-lua && \
+    uci set uhttpd.main.interpreter='.lua=/usr/bin/lua' && \
+    uci commit uhttpd && \
+    opkg install mwan3 && \
+    opkg install luci-app-mwan3; exit 0
+
+COPY system /etc/config/system
+COPY commands.lua /usr/lib/lua/luci/controller/
+
+ENV http_proxy=
+ENV https_proxy=
+ENV no_proxy=
+
+USER root
+
+# using exec format so that /sbin/init is proc 1 (see procd docs)
+CMD ["/sbin/init"]
diff --git a/sdwan/build/Dockerfile_1806_mwan3_noproxy.tpl b/sdwan/build/Dockerfile_1806_mwan3_noproxy.tpl
new file mode 100644
index 0000000..8b5c57d
--- /dev/null
+++ b/sdwan/build/Dockerfile_1806_mwan3_noproxy.tpl
@@ -0,0 +1,19 @@
+FROM openwrt-1806-4-base
+
+#EXPOSE 80
+
+RUN mkdir /var/lock && \
+    opkg update && \
+    opkg install uhttpd-mod-lua && \
+    uci set uhttpd.main.interpreter='.lua=/usr/bin/lua' && \
+    uci commit uhttpd && \
+    opkg install mwan3 && \
+    opkg install luci-app-mwan3; exit 0
+
+COPY system /etc/config/system
+COPY commands.lua /usr/lib/lua/luci/controller/
+
+USER root
+
+# using exec format so that /sbin/init is proc 1 (see procd docs)
+CMD ["/sbin/init"]
diff --git a/sdwan/build/README.md b/sdwan/build/README.md
new file mode 100644
index 0000000..7691545
--- /dev/null
+++ b/sdwan/build/README.md
@@ -0,0 +1,8 @@
+# Precondition:
+# Docker installed
+
+Use below steps to build openwrt docker image: openwrt-1806-mwan3
+(1) update set_proxy file with proxy used for docker build
+(2) execute build_image.sh
+cd build
+sudo bash build_image.sh
diff --git a/sdwan/build/build_image.sh b/sdwan/build/build_image.sh
new file mode 100644
index 0000000..7ff6e20
--- /dev/null
+++ b/sdwan/build/build_image.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# usage: build_images.sh
+
+set -ex
+base_image_tag=openwrt-1806-4-base
+docker_file=Dockerfile_1806_mwan3
+image_tag=openwrt-1806-mwan3
+package=openwrt-18.06.4-x86-64-generic-rootfs
+
+# build openwrt base docker images
+base_image=`docker images | grep $base_image_tag | awk '{print $1}'`
+if [ -z "$base_image" ]; then
+    # download driver source package
+    if [ ! -e /tmp/$package.tar.gz ]; then
+        wget -P /tmp https://downloads.openwrt.org/releases/18.06.4/targets/x86/64/$package.tar.gz
+    fi
+    cp /tmp/$package.tar.gz .
+
+    docker import $package.tar.gz $base_image_tag
+fi
+
+# generate Dockerfile
+test -f ./set_proxy && . set_proxy
+docker_proxy=${docker_proxy-""}
+if [ -z "$docker_proxy" ]; then
+    cp ${docker_file}_noproxy.tpl $docker_file
+else
+    cp $docker_file.tpl $docker_file
+    sed -i "s,{docker_proxy},$docker_proxy,g" $docker_file
+fi
+
+# build docker images for openwrt with wman3
+docker build --network=host -f $docker_file -t $image_tag .
+
+# clear
+docker image rm $base_image_tag
+rm -rf $docker_file
+rm -rf $package.tar.gz
diff --git a/sdwan/build/commands.lua b/sdwan/build/commands.lua
new file mode 100644
index 0000000..d99f457
--- /dev/null
+++ b/sdwan/build/commands.lua
@@ -0,0 +1,43 @@
+-- Licensed to the public under the GNU General Public License v2.
+
+module("luci.controller.commands", package.seeall)
+
+sys = require "luci.sys"
+ut = require "luci.util"
+io = require "io"
+
+ip = "ip -4 "
+
+function index()
+    entry({"admin", "config", "command"},
+	call("execute")).dependent = false
+end
+
+function trim(s)
+    return s:match("^%s*(.-)%s*$")
+end
+
+function split_and_trim(str, sep)
+    local array = {}
+    local reg = string.format("([^%s]+)", sep)
+    for item in string.gmatch(str, reg) do
+        item_trimed = trim(item)
+        if string.len(item_trimed) > 0 then
+            table.insert(array, item_trimed)
+        end
+    end
+    return array
+end
+
+function execute()
+    local commands = luci.http.formvalue("command")
+    io.stderr:write("Execute command: %s\n" % commands)
+
+    local command_array = split_and_trim(commands, ";")
+    for index, command in ipairs(command_array) do
+        sys.exec(command)
+    end
+
+    luci.http.prepare_content("application/json")
+    luci.http.write_json("{'status':'ok'}")
+end
diff --git a/sdwan/build/set_proxy b/sdwan/build/set_proxy
new file mode 100644
index 0000000..1ad2350
--- /dev/null
+++ b/sdwan/build/set_proxy
@@ -0,0 +1,2 @@
+# set docker proxy  with below line
+#docker_proxy=
diff --git a/sdwan/build/system b/sdwan/build/system
new file mode 100644
index 0000000..5165430
--- /dev/null
+++ b/sdwan/build/system
@@ -0,0 +1,7 @@
+config system
+    option log_file '/var/log/mylog'
+    option timezone 'UTC'
+    option ttylogin '0'
+    option log_size '64'
+    option urandom_seed '0'
+EOF
diff --git a/sdwan/test/ovn-pod.yml b/sdwan/test/ovn-pod.yml
new file mode 100644
index 0000000..bd28174
--- /dev/null
+++ b/sdwan/test/ovn-pod.yml
@@ -0,0 +1,40 @@
+# Create 2 OVN Networks
+---
+apiVersion: k8s.plugin.opnfv.org/v1alpha1
+kind: Network
+metadata:
+  name: ovn-port-net
+spec:
+  cniType : ovn4nfv
+  ipv4Subnets:
+  - subnet: 172.16.33.0/24
+    name: subnet1
+    gateway: 172.16.33.1/24
+
+---
+apiVersion: k8s.plugin.opnfv.org/v1alpha1
+kind: Network
+metadata:
+  name: ovn-priv-net
+spec:
+  cniType : ovn4nfv
+  ipv4Subnets:
+  - subnet: 172.16.44.0/24
+    name: subnet1
+    gateway: 172.16.44.1/24
+
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: ovn-pod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[{ "name": "ovn-networkobj"}]'
+    k8s.plugin.opnfv.org/nfn-network: '{ "type": "ovn4nfv", "interface": [{ "name": "ovn-port-net", "interface": "net0" , "defaultGateway": "false"},
+       { "name": "ovn-priv-net", "interface": "net1" , "defaultGateway": "false"}]}'
+spec:
+  containers:
+  - name: ovn-pod
+    image: docker.io/centos/tools:latest
+    command:
+    - /sbin/init
diff --git a/sdwan/test/sdwan-openwrt-ovn.yml b/sdwan/test/sdwan-openwrt-ovn.yml
new file mode 100644
index 0000000..5c874d5
--- /dev/null
+++ b/sdwan/test/sdwan-openwrt-ovn.yml
@@ -0,0 +1,106 @@
+---
+apiVersion: k8s.plugin.opnfv.org/v1alpha1
+kind: Network
+metadata:
+  name: ovn-port-net
+spec:
+  cniType : ovn4nfv
+  ipv4Subnets:
+  - subnet: 172.16.33.0/24
+    name: subnet1
+    gateway: 172.16.33.1/24
+
+---
+apiVersion: k8s.plugin.opnfv.org/v1alpha1
+kind: Network
+metadata:
+  name: ovn-priv-net
+spec:
+  cniType : ovn4nfv
+  ipv4Subnets:
+  - subnet: 172.16.44.0/24
+    name: subnet1
+    gateway: 172.16.44.1/24
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: sdwan-config-ovn
+data:
+  entrypoint.sh: |
+    #!/bin/bash
+    # Always exit on errors.
+    set -e
+
+    interface0=net0
+    ipaddr0=`ifconfig $interface0 | awk '/inet/{print $2}' | cut -f2 -d ":" | awk 'NR==1 {print $1}'`
+
+    interface1=net1
+    ipaddr1=`ifconfig $interface1 | awk '/inet/{print $2}' | cut -f2 -d ":" | awk 'NR==1 {print $1}'`
+
+    net_config=/etc/config/network
+    cat >> $net_config << EOF
+    config interface 'wan'
+        option ifname '$interface0'
+        option proto 'static'
+        option ipaddr '$ipaddr0'
+        option netmask '255.255.255.0'
+
+    config interface 'wanb'
+        option ifname '$interface1'
+        option proto 'static'
+        option ipaddr '$ipaddr1'
+        option netmask '255.255.255.0'
+    EOF
+
+    /sbin/procd &
+    /sbin/ubusd &
+    iptables -S
+    sleep 1
+    /etc/init.d/rpcd start
+    /etc/init.d/dnsmasq start
+    /etc/init.d/network start
+    /etc/init.d/odhcpd start
+    /etc/init.d/uhttpd start
+    /etc/init.d/log start
+    /etc/init.d/dropbear start
+    /etc/init.d/mwan3 restart
+
+    echo "Entering sleep... (success)"
+
+    # Sleep forever.
+    while true; do sleep 100; done
+
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: sdwan-ovn-pod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[{ "name": "ovn-networkobj"}]'
+    k8s.plugin.opnfv.org/nfn-network: '{ "type": "ovn4nfv", "interface": [{ "name": "ovn-port-net", "interface": "net0" , "defaultGateway": "false"},
+       { "name": "ovn-priv-net", "interface": "net1" , "defaultGateway": "false"}]}'
+spec:
+  containers:
+  - name: sdwan-ovn-pod
+    image: hle2/openwrt-1806-mwan3:v0.1.0
+    ports:
+      - containerPort: 22
+      - containerPort: 80
+    command:
+    - /bin/sh
+    - /init/entrypoint.sh
+    imagePullPolicy: IfNotPresent
+    securityContext:
+      privileged: true
+    volumeMounts:
+      - name: entrypoint-sh
+        mountPath: /init
+  volumes:
+    - name: entrypoint-sh
+      configMap:
+        name: sdwan-config-ovn
+        items:
+        - key: entrypoint.sh
+          path: entrypoint.sh
diff --git a/sdwan/test/sdwan_verifier.sh b/sdwan/test/sdwan_verifier.sh
new file mode 100644
index 0000000..f3ee249
--- /dev/null
+++ b/sdwan/test/sdwan_verifier.sh
@@ -0,0 +1,130 @@
+#!/bin/bash
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+sdwan_pod_name=sdwan-ovn-pod
+ovn_pod_name=ovn-pod
+wan_interface=net0
+
+function login {
+    login_url=http://$1/cgi-bin/luci/
+    echo $(wget -S --spider --post-data "luci_username=root&luci_password=" $login_url 2>&1 | grep sysauth= | sed -r 's/.*sysauth=([^;]+);.*/\1/')
+}
+
+function disable_ping {
+    command_url=http://$2/cgi-bin/luci/admin/config/command
+    command="uci set firewall.@rule[1].target='REJECT';fw3 reload"
+    echo $(wget -S --spider --header="Cookie:sysauth=$1" --post-data "command=$command" $command_url 2>&1)
+}
+
+function enable_ping {
+    command_url=http://$2/cgi-bin/luci/admin/config/command
+    command="uci set firewall.@rule[1].target='ACCEPT';fw3 reload"
+    echo $(wget -S --spider --header="Cookie:sysauth=$1" --post-data "command=$command" $command_url 2>&1)
+}
+
+function wait_for_pod {
+    status_phase=""
+    while [[ "$status_phase" != "Running" ]]; do
+        new_phase="$(kubectl get pods -o wide | grep ^$1 | awk '{print $3}')"
+        if [[ "$new_phase" != "$status_phase" ]]; then
+            status_phase="$new_phase"
+        fi
+        if [[ "$new_phase" == "Err"* ]]; then
+            exit 1
+        fi
+	sleep 2
+    done
+}
+
+function wait_for_pod_namespace {
+    status_phase=""
+    while [[ "$status_phase" != "Running" ]]; do
+        new_phase="$(kubectl get pods -o wide -n $2 | grep ^$1 | awk '{print $3}')"
+        if [[ "$new_phase" != "$status_phase" ]]; then
+            status_phase="$new_phase"
+        fi
+        if [[ "$new_phase" == "Err"* ]]; then
+            exit 1
+        fi
+	sleep 2
+    done
+}
+
+echo "Waiting for multus CNI to be ready ..."
+wait_for_pod_namespace kube-multus-ds kube-system
+
+echo "Create pods ..."
+kubectl apply -f ovn-pod.yml
+kubectl apply -f sdwan-openwrt-ovn.yml
+
+echo "Waiting for pods to be ready ..."
+wait_for_pod $ovn_pod_name
+wait_for_pod $sdwan_pod_name
+echo "* Create pods success"
+
+sdwan_pod_ip=$(kubectl get pods -o wide | grep ^$sdwan_pod_name | awk '{print $6}')
+ovn_pod_ip=$(kubectl get pods -o wide | grep ^$ovn_pod_name | awk '{print $6}')
+echo "SDWAN pod ip:"$sdwan_pod_ip
+echo "OVN pod ip:"$ovn_pod_ip
+
+echo "Login to sdwan ..."
+security_token=""
+while [[ "$security_token" == "" ]]; do
+    echo "Get Security Token ..."
+    security_token=$(login $sdwan_pod_ip)
+    sleep 2
+done
+echo "* Security Token: "$security_token
+
+kubectl exec $sdwan_pod_name ifconfig
+
+sdwan_pod_wan_ip=$(kubectl exec $sdwan_pod_name ifconfig $wan_interface  | awk '/inet/{print $2}' | cut -f2 -d ":" | awk 'NR==1 {print $1}')
+echo "Verify ping is work through wan interface between $sdwan_pod_name and $ovn_pod_name"
+ping_result=$(kubectl exec $ovn_pod_name -- ping -c 3 $sdwan_pod_wan_ip)
+if [[ $ping_result == *", 0% packet loss"* ]]; then
+    echo "* Ping is work through wan interface"
+else
+    echo "* Test failed!"
+    exit 1
+fi
+
+echo "Disable ping rule of wan interface ..."
+ret=$(disable_ping $security_token $sdwan_pod_ip)
+
+echo "Verify ping is not work through wan interface after ping rule disabled"
+ping_result=$(kubectl exec $ovn_pod_name -- ping -c 3 $sdwan_pod_wan_ip 2>&1 || true)
+if [[ $ping_result == *", 100% packet loss"* ]]; then
+    echo "* Ping is disabled"
+else
+    echo "* Test failed!"
+    exit 1
+fi
+
+echo "Enable ping rule of wan interface ..."
+ret=$(enable_ping $security_token $sdwan_pod_ip)
+
+echo "Verify ping is work through wan interface after ping rule enabled"
+ping_result=$(kubectl exec $ovn_pod_name -- ping -c 3 $sdwan_pod_wan_ip)
+if [[ $ping_result == *", 0% packet loss"* ]]; then
+    echo "* Ping is enabled"
+else
+    echo "* Test failed!"
+    exit 1
+fi
+
+echo "Clear pods ..."
+kubectl delete pod $ovn_pod_name
+kubectl delete -f sdwan-openwrt-ovn.yml
+
+echo "Test Completed!"