From f7dab2aa3807639ca5b584bb76b61ec848aa29bc Mon Sep 17 00:00:00 2001 From: davidplunkett Date: Fri, 20 Jul 2018 17:35:39 -0400 Subject: [PATCH] initial push Change-Id: I637128fe227f7ee7c06553d688a39105c5539e4b Signed-off-by: davidplunkett --- ats-demo/ats-demo.yaml | 375 ++++++++++++++++++++++++++++++++++++++++++ ats-demo/run_ats-demo.sh | 115 +++++++++++++ ats-demo/run_openstack_cli.sh | 42 +++++ mv.sh | 19 +++ 4 files changed, 551 insertions(+) create mode 100644 ats-demo/ats-demo.yaml create mode 100755 ats-demo/run_ats-demo.sh create mode 100755 ats-demo/run_openstack_cli.sh create mode 100755 mv.sh diff --git a/ats-demo/ats-demo.yaml b/ats-demo/ats-demo.yaml new file mode 100644 index 0000000..c0f4add --- /dev/null +++ b/ats-demo/ats-demo.yaml @@ -0,0 +1,375 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +heat_template_version: 2015-10-15 + +description: > + Hot Template to deploy an Apache Traffic Server + and locustio client instance. The outputs are + the client ip, server ip, url of the sample video + and url of the client dashboard. + +parameters: + ImageUrl: + type: string + description: URL of Ubuntu cloud image + default: https://cloud-images.ubuntu.com/releases/16.04/release/ubuntu-16.04-server-cloudimg-amd64-disk1.img + ImagePwd: + type: string + description: Password for client VM + default: akraino,d + OriginServerPath: + type: string + description: URL path that will be mapped to http://server-ip:8080/ on Apache Traffic Server instance + default: http://distribution.bbb3d.renderfarming.net/video/mp4/ + OriginFileName: + type: string + description: File name to download from client + default: bbb_sunflower_1080p_30fps_normal.mp4 + Zone: + type: string + description: Name of Availability Zone + default: nova + NetID: + type: string + description: External Network ID + +resources: + + image_name: + type: OS::Heat::Value + properties: + type: string + value: + str_replace: + template: $STACK-image + params: + $STACK: { get_param: "OS::stack_name" } + + glance_image: + type: OS::Glance::Image + properties: + container_format: bare + disk_format: qcow2 + name: { get_attr: [ image_name, value ] } + location: { get_param: ImageUrl } + + client_name: + type: OS::Heat::Value + properties: + type: string + value: + str_replace: + template: $STACK-client + params: + $STACK: { get_param: "OS::stack_name" } + + server_name: + type: OS::Heat::Value + properties: + type: string + value: + str_replace: + template: $STACK-server + params: + $STACK: { get_param: "OS::stack_name" } + + client_flavor_name: + type: OS::Heat::Value + properties: + type: string + value: + str_replace: + template: m1.$STACK + params: + $STACK: { get_param: "OS::stack_name" } + + server_flavor_name: + type: OS::Heat::Value + properties: + type: string + value: + str_replace: + template: x1.$STACK + params: + $STACK: { get_param: "OS::stack_name" } + + client_flavor: + type: OS::Nova::Flavor + properties: + ephemeral: 0 + is_public: true + name: { get_attr: [ client_flavor_name, value ] } + ram: 8192 + vcpus: 4 + disk: 10 + + server_flavor: + type: OS::Nova::Flavor + properties: + ephemeral: 0 + is_public: true + name: { get_attr: [ server_flavor_name, value ] } + ram: 16384 + vcpus: 8 + disk: 20 + + server_init: + type: OS::Heat::CloudConfig + properties: + cloud_config: + chpasswd: + list: + str_replace: + template: | + root:$CPASSWD + ubuntu:$CPASSWD + params: + $CPASSWD: { get_param: ImagePwd } + expire: False + ssh_pwauth: True + timezone: UTC + write_files: + - path: /root/setup_ats.sh + owner: "root:root" + permissions: "0700" + content: + str_replace: + template: | + #!/bin/bash + exec >> /root/setup_ats.log + exec 2>&1 + + echo "### SETTING UP SHELL VARIABLES" + export USER=root + export HOME=/root + cd $HOME + echo "Beginning $0 as user [$USER] in pwd [$PWD] with home [$HOME]" + + MYHOSTNAME=$(hostname) + MYIP=$(ifconfig ens3 | grep -o " inet addr:[^ ]*" | cut -f 2 -d':') + + echo "### INSTALLING REQUIRED PACKAGES" + apt-get update + apt-get install -y curl locales build-essential bzip2 libssl-dev libxml2 libxml2-dev libpcre3 libpcre3-dev tcl tcl-dev libboost-dev speedometer + export LANGUAGE=en_US.UTF-8 + export LANG=en_US.UTF-8 + export LC_ALL=en_US.UTF-8 + locale-gen en_US.UTF-8 + + echo "### INSTALLING APACHE TRAFFIC SERVER" + mkdir /root/trafficserver + cd /root/trafficserver + curl -L http://archive.apache.org/dist/trafficserver/trafficserver-7.1.3.tar.bz2 | tar -xj --strip-components 1 + ./configure --prefix=/opt/trafficserver + make -j 4 install + + echo "### SAVING DEFAULT CONFIG FILES" + cp /opt/trafficserver/etc/trafficserver/remap.config /opt/trafficserver/etc/trafficserver/remap.config.orig + cp /opt/trafficserver/etc/trafficserver/records.config /opt/trafficserver/etc/trafficserver/records.config.orig + cp /opt/trafficserver/etc/trafficserver/storage.config /opt/trafficserver/etc/trafficserver/storage.config.orig + cp /opt/trafficserver/etc/trafficserver/volume.config /opt/trafficserver/etc/trafficserver/volume.config.orig + + echo "### UPDATING CONFIG FILES" + echo "map / $ORIGINPATH" > /opt/trafficserver/etc/trafficserver/remap.config + sed -i -e 's|^CONFIG proxy.config.http.cache.required_headers INT .*$|CONFIG proxy.config.http.cache.required_headers INT 0|' /opt/trafficserver/etc/trafficserver/records.config + echo "/dev/ram0 volume=1" > /opt/trafficserver/etc/trafficserver/storage.config + echo "volume=1 scheme=http" > /opt/trafficserver/etc/trafficserver/volume.config + + echo "### CREATING RAMDISK" + modprobe brd rd_size=8388608 rd_nr=1 + echo "brd rd_size=8388608 rd_nr=1" > /etc/modules + chown nobody:nogroup /dev/ram0 + ls -l /dev/ram* + fdisk -l /dev/ram* + chown nobody:nogroup /dev/ram0 + ls -l /dev/ram* + + echo "### STARTING APACHE TRAFFIC SERVER" + /opt/trafficserver/bin/trafficserver start + + echo "### FINISHED" + params: + $ORIGINPATH: { get_param: OriginServerPath } + - path: /root/firstboot.sh + owner: "root:root" + permissions: "0700" + content: | + #!/bin/bash + echo "userdata started on hostname: $(uname -n) at" `date` + STARTTIME=$(date +%s) + sed -i -e 's|^PermitRootLogin .*$|PermitRootLogin yes|' /etc/ssh/sshd_config + service sshd restart + /root/setup_ats.sh + ENDTIME=$(date +%s) + echo "Monitor network throughput using 'speedometer -r ens3 -t ens3'" + echo "userdata finished on hostname: $(uname -n) at" `date` + echo "userdata elapsed time was $(( ($ENDTIME - $STARTTIME) / 60 )) minutes and $(( ($ENDTIME - $STARTTIME) % 60 )) seconds" + runcmd: + - [ sh, -c, /root/firstboot.sh ] + + server: + type: OS::Nova::Server + properties: + name: { get_attr: [ server_name, value ] } + image: { get_resource: glance_image } + flavor: { get_attr: [ server_flavor_name, value ] } + availability_zone: { get_param: Zone } + networks: + - network: { get_param: NetID } + config_drive: True + user_data_format: SOFTWARE_CONFIG + user_data: + get_resource: server_init + + + client_init: + type: OS::Heat::CloudConfig + properties: + cloud_config: + chpasswd: + list: + str_replace: + template: | + root:$CPASSWD + ubuntu:$CPASSWD + params: + $CPASSWD: { get_param: ImagePwd } + expire: False + ssh_pwauth: True + timezone: UTC + write_files: + - path: /root/test.py + content: + str_replace: + template: | + from locust import HttpLocust, TaskSet, task + + class UserBehavior(TaskSet): + @task(1) + def profile(self): + self.client.get("/$ORIGINFILE") + + class WebsiteUser(HttpLocust): + task_set = UserBehavior + min_wait = 1000 + max_wait = 1000 + params: + $ORIGINFILE: { get_param: OriginFileName } + - path: /root/setup_python.sh + owner: "root:root" + permissions: "0700" + content: + str_replace: + template: | + #!/bin/bash + cd /root + apt-get update + apt-get install -y curl python-pip speedometer + pip install virtualenv + virtualenv ats-demo + source /root/ats-demo/bin/activate + pip install locustio + sed -i 's/ pywsgi/ runners.locust_runner.start_hatching(options.num_clients, options.hatch_rate)\n pywsgi/g' /root/ats-demo/lib/python2.7/site-packages/locust/web.py + python -m compileall + deactivate + params: + $SERVER_IP: { get_attr: [ server, first_address ] } + + - path: /root/run_locust.sh + owner: "root:root" + permissions: "0700" + content: + str_replace: + template: | + #!/bin/bash + RESULTS_FILE=/root/results.txt + CLIENT_IP=$(hostname -I | xargs -n 1 | head -n 1) + cd /root + source /root/ats-demo/bin/activate + i="900" + while ! curl http://$SERVER_IP:8080/$ORIGINFILE >/dev/null && [ "$i" -gt 0 ] ; do + echo "### WAITING UP TO [$i] SECONDS FOR FILE [$ORIGINFILE] TO BE ACCESSIBLE ON APACHE TRAFFIC SERVER [$SERVER_IP:8080]" + sleep 10 + i=$[$i-10] + done + date | tee -a $RESULTS_FILE + echo "Testing with host http://$SERVER_IP:8080" | tee -a $RESULTS_FILE + echo "Test dashboard located at http://$CLIENT_IP:8089/" | tee -a $RESULTS_FILE + echo "Monitor network throughput using 'speedometer -r ens3 -t ens3'" | tee -a $RESULTS_FILE + if [ "$i" -gt 0 ]; then + locust -f /root/test.py --host="http://$SERVER_IP:8080" --web-host=$CLIENT_IP -c 100 -r 10 2>&1 | tee -a $RESULTS_FILE + else + echo "ERROR: Unable to access [http://$SERVER_IP:8080/$ORIGINFILE]. Test aborted!" | tee -a $RESULTS_FILE + fi + deactivate + params: + $SERVER_IP: { get_attr: [ server, first_address ] } + $ORIGINFILE: { get_param: OriginFileName } + - path: /root/firstboot.sh + owner: "root:root" + permissions: "0700" + content: | + #!/bin/bash + echo "userdata started on hostname: $(uname -n) at" `date` + STARTTIME=$(date +%s) + sed -i -e 's|^PermitRootLogin .*$|PermitRootLogin yes|' /etc/ssh/sshd_config + service sshd restart + /root/setup_python.sh + /root/run_locust.sh + ENDTIME=$(date +%s) + echo "Monitor network throughput using 'speedometer -r ens3 -t ens3'" + echo "userdata finished on hostname: $(uname -n) at" `date` + echo "userdata elapsed time was $(( ($ENDTIME - $STARTTIME) / 60 )) minutes and $(( ($ENDTIME - $STARTTIME) % 60 )) seconds" + runcmd: + - [ sh, -c, /root/firstboot.sh ] + + client: + type: OS::Nova::Server + depends_on: [server] + properties: + name: { get_attr: [ client_name, value ] } + image: { get_resource: glance_image } + flavor: { get_attr: [ client_flavor_name, value ] } + availability_zone: { get_param: Zone } + networks: + - network: { get_param: NetID } + config_drive: True + user_data_format: SOFTWARE_CONFIG + user_data: + get_resource: client_init + +outputs: + server_ip: + description: IP of the server + value: { get_attr: [ server, first_address ] } + client_ip: + description: IP of the client + value: { get_attr: [ client, first_address ] } + client_url: + description: URL for client + value: + str_replace: + template: "http://$CLIENT_IP:8089/" + params: + $CLIENT_IP: { get_attr: [ client, first_address ] } + server_url: + description: URL for server + value: + str_replace: + template: "http://$SERVER_IP:8080/$ORIGINFILE" + params: + $SERVER_IP: { get_attr: [ server, first_address ] } + $ORIGINFILE: { get_param: OriginFileName } + diff --git a/ats-demo/run_ats-demo.sh b/ats-demo/run_ats-demo.sh new file mode 100755 index 0000000..e3aa870 --- /dev/null +++ b/ats-demo/run_ats-demo.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Script to run the Apache Traffic Server heat stack and Locustio load generator. +# Only one copy of tempest is allowed to run at a time. +# +# usage: NETWORK_NAME=external_net_name ./run_ats-demo.sh + +# Define Variables +# +# NOTE: User will need to set up the required environment variables +# before executing this script if they differ from the default values. +# +# NOTE: This script calls run_openstack_cli.sh which puts a CR at the end +# of each line of output. Remove using tr -d '\r' + + +# SET DEFAULT VALUES +export OS_USERNAME=${OS_USERNAME:-admin} +export OS_PASSWORD=${OS_PASSWORD:-password} +export OS_REGION_NAME=${OS_REGION_NAME:-RegionOne} +export NAMESPACE="${NAMESPACE:-openstack}" + +STACK_NAME="${STACK_NAME:-ats-demo}" +NETWORK_NAME="${NETWORK_NAME:-external}" +ZONE="${ZONE:-nova}" +TIMEOUT="${TIMEOUT:-300}" + +export SCRIPT_PATH="$( cd "$(dirname "$0")" ; pwd -P )" +export OPENSTACKCLI=$SCRIPT_PATH/run_openstack_cli +echo "$SCRIPT_PATH" + +## CHECK THAT OPENSTACK COMMANDS WORK (also forces download of docker image if needed) +ERROR=$($SCRIPT_PATH/run_openstack_cli.sh stack list) +if [ "$?" -ne 0 ]; then + echo "FAILED: Cannot access openstack." + echo "ERROR: ${ERROR: : -1}" + echo "OS_REGION_NAME = ${OS_REGION_NAME}" + echo "OS_USERNAME = ${OS_USERNAME}" + echo "OS_PASSWORD = ${OS_PASSWORD}" + exit 1 +fi + +## CHECK THAT ATS-DEMO IS NOT ALREADY RUNNING +STATUS=$($SCRIPT_PATH/run_openstack_cli.sh stack list -f csv | tr -d '\r' | grep "\"$STACK_NAME\"" | cut -d ',' -f 3 | sed -e 's/^"//' -e 's/"$//') +if [ -n "$STATUS" ]; then + CTIME=$($SCRIPT_PATH/run_openstack_cli.sh stack show $STACK_NAME -c creation_time -f value | tr -d '\r') + echo "FAILED: ats-demo heat stack was created on [$CTIME] with status [$STATUS]." + if [ "$STATUS" == "CREATE_COMPLETE" ]; then + CLIENTURL=$($SCRIPT_PATH/run_openstack_cli.sh stack output show $STACK_NAME client_url -c output_value -f value | tr -d '\r') + SERVERURL=$($SCRIPT_PATH/run_openstack_cli.sh stack output show $STACK_NAME server_url -c output_value -f value | tr -d '\r') + echo " You can access the demo with client_url: [$CLIENTURL] and server_url: [$SERVERURL]." + echo " Please delete the stack [$STACK_NAME] and redeploy if the urls are not working." + else + echo " Please delete the stack and redeploy." + fi + exit 1 +fi + +## GET EXTERNAL NETWORK +NET_ID=$($SCRIPT_PATH/run_openstack_cli.sh network list -f csv | tr -d '\r' | grep "$NETWORK_NAME" | cut -d ',' -f 1 | sed -e 's/^"//' -e 's/"$//') +if [ -z "$NET_ID" ]; then + echo "FAILED: no network found matching [$NETWORK_NAME]. Available networks are:" + $SCRIPT_PATH/run_openstack_cli.sh network list + exit 1 +fi + +## CREATE HEAT STACK +ERROR=$($SCRIPT_PATH/run_openstack_cli.sh stack create -t ./ats-demo.yaml $STACK_NAME --parameter NetID=$NET_ID --parameter Zone=$ZONE) +if [ "$?" -ne 0 ]; then + echo "FAILED: error creating stack [$STACK_NAME]." + echo "ERROR : $ERROR" + exit 1 +fi + +## WAIT UP TO $TIMEOUT SECONDS FOR STACK TO COMPLETE +i="0" +STATUS=$($SCRIPT_PATH/run_openstack_cli.sh stack show $STACK_NAME -c stack_status -f value | tr -d '\r') +while [ "$STATUS" = "CREATE_IN_PROGRESS" ] && [ $i -lt $TIMEOUT ]; do + sleep 10 + STATUS=$($SCRIPT_PATH/run_openstack_cli.sh stack show $STACK_NAME -c stack_status -f value | tr -d '\r') + #echo -n "." + i=$[$i+10] +done +#echo + +## ABORT IF STACK DID NOT COMPLETE +STATUS=$($SCRIPT_PATH/run_openstack_cli.sh stack show $STACK_NAME -c stack_status -f value | tr -d '\r') +if [ "$STATUS" != "CREATE_COMPLETE" ]; then + echo "FAILED: Stack [$STACK_NAME] did not complete. Please check openstack for more details" + $SCRIPT_PATH/run_openstack_cli.sh stack show $STACK_NAME + exit 1 +fi + +## DISPLAY STACK OUTPUTS +CLIENTURL=$($SCRIPT_PATH/run_openstack_cli.sh stack output show $STACK_NAME client_url -c output_value -f value | tr -d '\r') +SERVERURL=$($SCRIPT_PATH/run_openstack_cli.sh stack output show $STACK_NAME server_url -c output_value -f value | tr -d '\r') +echo "SUCCESS: You can access the demo dashboard with client_url: [$CLIENTURL]" +echo "SUCCESS: You can stream the sample video using server_url: [$SERVERURL]" +exit 0 + diff --git a/ats-demo/run_openstack_cli.sh b/ats-demo/run_openstack_cli.sh new file mode 100755 index 0000000..0179dee --- /dev/null +++ b/ats-demo/run_openstack_cli.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# Define variables +COMMAND='openstack' +OPENSTACK_CLI_IMAGE="${OPENSTACK_CLI_IMAGE:-docker.io/openstackhelm/heat:newton}" + +# Define Base Docker Command +base_docker_command=$(cat << EndOfCommand +sudo docker run -t --rm --net=host +-e http_proxy=${HTTP_PROXY} +-e https_proxy=${HTTPS_PROXY} +-e OS_AUTH_URL=${OS_AUTH_URL:-http://keystone.${NAMESPACE}.svc.cluster.local:80/v3} +-e OS_USERNAME=${OS_USERNAME:-admin} +-e OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME:-default} +-e OS_PASSWORD=${OS_PASSWORD:-password} +-e OS_PROJECT_DOMAIN_NAME=${OS_PROJECT_DOMAIN_NAME:-default} +-e OS_PROJECT_NAME=${OS_PROJECT_NAME:-admin} +-e OS_REGION_NAME=${OS_REGION_NAME:-RegionOne} +-e OS_IDENTITY_API_VERSION=${OS_IDENTITY_API_VERSION:-3} +-w /host/$(pwd) +-v /:/host:rshared +EndOfCommand +) + +# Execute OpenStack CLI +${base_docker_command} ${OPENSTACK_CLI_IMAGE} ${COMMAND} $@ + diff --git a/mv.sh b/mv.sh new file mode 100755 index 0000000..1912dbb --- /dev/null +++ b/mv.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +scp -r ats-demo root@$1:/opt +echo $? +exit 0 -- 2.16.6