Enable installation of TA in CI 75/2275/24
authorCristina Pauna <cristina.pauna@enea.com>
Wed, 4 Mar 2020 10:00:59 +0000 (12:00 +0200)
committerAlexandru Antone <Alexandru.Antone@enea.com>
Tue, 17 Mar 2020 15:06:27 +0000 (17:06 +0200)
For each REC POD, the CD job is tied to a specific jenkins slave
that has access to the POD's lab.

For now, only UNH Ampere1 POD is added and user_config.yaml is stored
on the slave at:
  file:///data/rec_pod/user_config.yaml

Also for CD deployments, admin_user_password in user_config.yaml is
required to be the hash for "admin" to be able to pull post deployment
logs from the first controller.

Jenkins slave package requirements:
  - docker
  - jq

Co-authored-by: Cristina Pauna <cristina.pauna@enea.com>
Co-authored-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>>
Signed-off-by: Alexandru Antone <Alexandru.Antone@enea.com>
Change-Id: Iece4330c3242385c33d8a1daa092f78ce5d46cc8

jjb/akraino-templates/akraino-rec-macros.yaml [new file with mode: 0644]
jjb/akraino-templates/akraino-rec-templates.yaml [new file with mode: 0644]
jjb/rec/rec.yaml
jjb/shell/ta-install.sh [new file with mode: 0644]

diff --git a/jjb/akraino-templates/akraino-rec-macros.yaml b/jjb/akraino-templates/akraino-rec-macros.yaml
new file mode 100644 (file)
index 0000000..bf95721
--- /dev/null
@@ -0,0 +1,36 @@
+---
+##############
+# PARAMETERS #
+##############
+- parameter:
+    name: 'rec-aarch64_baremetal-defaults'
+    parameters:
+      - string:
+          name: REC_ISO_IMAGE_NAME
+          default: 'https://nexus.akraino.org/content/repositories/images-snapshots/TA/images/arm64/latest/install.aarch64.iso'
+          description: 'REC install image'
+      - string:
+          name: REC_PROVISIONING_ISO_NAME
+          default: 'https://nexus.akraino.org/content/repositories/images-snapshots/TA/images/arm64/latest/bootcd.aarch64.iso'
+          description: 'REC provisioning image'
+      - string:
+          name: REC_USER_CONFIG
+          default: 'file:///data/rec_pod/user_config.yaml'
+          description: 'POD user config YAML (hosted on Jumpserver by default)'
+      - string:
+          name: ARCHIVE_ARTIFACTS
+          default: '**/*.log'
+          description: Artifacts to archive to the logs server.
+      - string:
+          name: SCM_REFSPEC_REMOTE_INSTALLER
+          default: 'refs/heads/master'
+          description: 'remote-installer custom git refspec'
+
+########################
+# trigger macros
+########################
+- trigger:
+    name: 'rec-weekly-master-trigger'
+    triggers:
+      - timed: '0 1 * * 6,7'
+
diff --git a/jjb/akraino-templates/akraino-rec-templates.yaml b/jjb/akraino-templates/akraino-rec-templates.yaml
new file mode 100644 (file)
index 0000000..6b26659
--- /dev/null
@@ -0,0 +1,78 @@
+---
+- job-template:
+    id: akraino-rec-deploy
+    name: 'rec-{slave-label}-{job-frequency}-{stream}'
+    disabled: '{obj:disabled}'
+    concurrent: false
+    node: '{slave-label}'
+    properties:
+      - throttle:
+          enabled: true
+          max-total: 4
+          max-per-node: 1
+          option: 'project'
+
+    wrappers:
+      - build-name:
+          name: '$BUILD_NUMBER - Install REC Blueprint'
+
+    triggers:
+      - 'rec-{job-frequency}-{stream}-trigger'
+
+    parameters:
+      - lf-infra-parameters:
+          project: '{project}'
+          stream: '{stream}'
+          branch: '{branch}'
+      - '{project}-{slave-label}-defaults'
+
+    builders:
+      - description-setter:
+          description: "SLAVE: $NODE_NAME"
+      - trigger-builds:
+          - project: 'rec-{slave-label}-install-rec-{job-frequency}-{stream}'
+            current-parameters: true
+            same-node: true
+            block: true
+
+    publishers: &rec-publishers
+      - email:
+          recipients: armband@enea.com
+      - lf-infra-publish
+
+- job-template:
+    id: akraino-rec-install-rec
+    name: 'rec-{slave-label}-install-rec-{job-frequency}-{stream}'
+    concurrent: true
+    node: '{slave-label}'
+    properties:
+      - throttle:
+          enabled: true
+          max-total: 4
+          max-per-node: 1
+          option: 'project'
+
+    parameters:
+      - lf-infra-parameters:
+          project: '{project}'
+          stream: '{stream}'
+          branch: '{branch}'
+      - '{project}-{slave-label}-defaults'
+
+    scm:
+      - ta-lf-infra-gerrit-scm:
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+          git-url: '{git-url}/ta/remote-installer.git'
+          branch: '{branch}'
+          submodule-recursive: false
+          submodule-timeout: 10
+          choosing-strategy: 'gerrit'
+          basedir: 'git/remote-installer'
+          refspec: '$SCM_REFSPEC_REMOTE_INSTALLER'
+    builders:
+      - description-setter:
+          description: "SLAVE: $NODE_NAME"
+      - shell: !include-raw-escape:
+          - ../shell/ta-install.sh
+
+    publishers: *rec-publishers
index 9d87fb9..ae55479 100644 (file)
@@ -1,6 +1,28 @@
 ---
 - project:
-    name: rec-project-view
-    project-name: rec
+    name: 'rec'
+    project-name: 'rec'
+    project: '{project-name}'
     views:
       - project-view
+    # -------------------------------
+    # BRANCH DEFINITIONS
+    # -------------------------------
+    stream:
+      - master:
+          branch: '{stream}'
+          disabled: false
+
+    # -------------------------------
+    # POD DEFINITIONS
+    # -------------------------------
+
+    slave-label:
+      - aarch64_baremetal
+
+    job-frequency:
+      - weekly
+
+    jobs:
+       - akraino-rec-deploy
+       - akraino-rec-install-rec
diff --git a/jjb/shell/ta-install.sh b/jjb/shell/ta-install.sh
new file mode 100644 (file)
index 0000000..7642978
--- /dev/null
@@ -0,0 +1,223 @@
+#!/bin/bash -e
+
+# Copyright 2019 AT&T
+# Copyright 2020 ENEA
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+_yaml2json() {
+    python -c 'import json,sys,yaml; json.dump(yaml.safe_load(sys.stdin.read()), sys.stdout)'
+}
+
+#Work-flow:
+
+#0. Get values for the environment variables
+
+# The following must be provided.
+    REC_ISO_IMAGE_NAME="${REC_ISO_IMAGE_NAME:?'Must be defined!'}"
+    REC_PROVISIONING_ISO_NAME="${REC_PROVISIONING_ISO_NAME:?'Must be defined!'}"
+    REC_USER_CONFIG="${REC_USER_CONFIG:?'Must be defined!'}"
+
+# The next set may be modified if necessary but are best left as-is
+    ADMIN_PASSWD="admin"
+    POD_NAME="REC-$BUILD_ID"
+    HTTPS_PORT=8443
+    API_PORT=15101
+    # Max time (in minutes) to wait for the remote-installer to return completed
+    # Currently 4 hours
+    MAX_TIME=240
+
+    # These should probably not be changed
+    WORKDIR="$(pwd)"
+    BASEDIR="$WORKDIR/rec_basedir"
+
+
+cat <<EOF
+    --------------------------------------------
+    WORKDIR is $WORKDIR
+    BASEDIR is $BASEDIR
+    POD_NAME is $POD_NAME
+    REC_ISO_IMAGE_NAME is $REC_ISO_IMAGE_NAME
+    REC_PROVISIONING_ISO_NAME is $REC_PROVISIONING_ISO_NAME
+    REC_USER_CONFIG is $REC_USER_CONFIG
+    --------------------------------------------
+EOF
+
+# Create cleanup hook
+_cleanup() {
+    exit_status=$?
+    set +e
+    sudo chown -R jenkins:jenkins "$WORKDIR"
+    docker cp 'remote-installer':/var/log/remote-installer.log "$BASEDIR/"
+    docker rm -f 'remote-installer'
+    trap - EXIT ERR HUP INT QUIT TERM
+    exit $exit_status
+}
+trap _cleanup EXIT ERR HUP INT QUIT TERM
+
+# Enable debugging
+    set -x
+
+#1. Create a new directory to be used for holding the installation artifacts.
+
+    # Clean workspace from previous run
+    sudo chown -R jenkins:jenkins "$WORKDIR"
+    rm -rf "$BASEDIR"
+
+    # Create the base directory structure
+    mkdir -p "$BASEDIR/images" \
+             "$BASEDIR/certificates" \
+             "$BASEDIR/installations" \
+             "$BASEDIR/user-configs/$POD_NAME"
+
+#2. Get REC golden and bootcd images from REC Nexus artifacts
+
+    cd "$BASEDIR/images/"
+    curl -sL "$REC_ISO_IMAGE_NAME"        > "$(basename "$REC_ISO_IMAGE_NAME")"
+    curl -sL "$REC_PROVISIONING_ISO_NAME" > "$(basename "$REC_PROVISIONING_ISO_NAME")"
+
+#3. Get the user-config.yaml file and admin_password file for the CD environment from the
+#   cd-environments repo and copy it to the user-configs sub-directory under the directory
+#   created in (1). Copy the files to a cloud-specific directory identified by the cloudname.
+
+    cd "$BASEDIR/user-configs/$POD_NAME"
+    curl -sL "$REC_USER_CONFIG" > user_config.yaml
+    echo "$ADMIN_PASSWD" > admin_passwd
+
+    FIRST_CONTROLLER_IP="$(_yaml2json < user_config.yaml | \
+        jq -r '.hosts[]|select(.service_profiles[]|contains("caas_master", "controller"))|.hwmgmt.address' | \
+        sort | head -1 )"
+    HOST_IP="$(ip route get "$FIRST_CONTROLLER_IP" | grep -Pzo 'src\s+\K([^\s]*)' )"
+
+#4. Copy the sever certificates, the client certificates in addition to CA certificate to
+#  the certificates sub-directory under the directory created in (1).
+#   The following certificates are expected to be available in the directory:
+#
+#   cacert.pem: The CA certificate
+#   servercert.pem: The server certificate signed by the CA
+#   serverkey.pem: The server key
+#   clientcert.pem: The client certificate signed by the CA
+#   clientkey.pem: The client key
+#
+
+    cd "$WORKDIR/git/remote-installer/test/certificates"
+    ./create.sh
+    cp -a {ca,client,server}{cert,key}.pem "${BASEDIR}/certificates/"
+
+#5. Build the remote installer docker-image.
+    cd "$WORKDIR/git/remote-installer/scripts/"
+    ./build.sh "$HTTPS_PORT" "$API_PORT"
+
+#6. Start the remote installer
+
+    cd "$WORKDIR/git/remote-installer/scripts/"
+    if ! ./start.sh -b "$BASEDIR" -e "$HOST_IP" -s "$HTTPS_PORT" -a "$API_PORT" -p "$ADMIN_PASSWD"
+    then
+        echo 'Failed to run workflow.'
+        exit 1
+    fi
+
+#7. Wait for the remote installer to become running.
+#   check every 30 seconds to see if it has it has a status of "running"
+
+    DOCKER_STATUS=""
+
+    while [ ${#DOCKER_STATUS} -eq 0 ]; do
+        sleep 30
+        DOCKER_ID=$(docker ps | grep 'remote-installer' | awk '{print $1}')
+        DOCKER_STATUS=$(docker ps -f status=running | grep "$DOCKER_ID")
+    done
+
+#8. Start the installation by sending the following http request to the installer API
+
+#    POST url: https://localhost:$API_PORT/v1/installations
+#    REQ body json- encoded
+#    {
+#        'cloudname': $POD_NAME,
+#        'iso': $REC_ISO_IMAGE_NAME,
+#        'provisioning-iso': $REC_PROVISIONING_ISO_NAME
+#    }
+#    REP body json-encoded
+#    {
+#        'uuid': $INSTALLATION_UUID
+#    }
+
+INSTALL_ISO="$(basename "$REC_ISO_IMAGE_NAME")"
+BOOT_ISO="$(basename "$REC_PROVISIONING_ISO_NAME")"
+cat >rec_request.json <<EOF
+{
+    "cloud-name": "$POD_NAME",
+    "iso": "$INSTALL_ISO",
+    "provisioning-iso": "$BOOT_ISO"
+}
+EOF
+
+    # Get the IP address of the remote installer container
+    RI_IP=$HOST_IP
+
+    RESPONSE=$(curl -k --silent \
+                    --header "Content-Type: application/json" \
+                    -d "@rec_request.json" \
+                    --cert "$BASEDIR/certificates/clientcert.pem" \
+                    --key  "$BASEDIR/certificates/clientkey.pem" \
+                    "https://${RI_IP}:${API_PORT}/v1/installations")
+    echo "$0: RESPONSE IS $RESPONSE"
+
+    INSTALLATION_UUID="$(echo "$RESPONSE" | jq -r ".uuid")"
+
+#9. Follow the progress of the installation by sending the following http request to the installer API
+
+#    GET url: https://localhost:$API_PORT/v1/installations/$INSTALLATION_UUID
+#
+#    REP body json-encoded
+#    {
+#        'status': <ongoing|completed|failed>,
+#        'description': <description>,
+#        'percentage': <the progess precentage>
+#    }
+#
+#
+
+# check the status every minute until it has become "completed"
+# (for a maximum of MAX_TIME minutes)
+
+    STATUS="ongoing"
+    NTIMES=$MAX_TIME
+    while [ "$STATUS" == "ongoing" -a $NTIMES -gt 0 ]; do
+        sleep 60
+        NTIMES=$((NTIMES - 1))
+        RESPONSE=$(curl -k --silent \
+                        --cert "$BASEDIR/certificates/clientcert.pem" \
+                        --key  "$BASEDIR/certificates/clientkey.pem" \
+                        "https://${RI_IP}:${API_PORT}/v1/installations/$INSTALLATION_UUID/state")
+        STATUS="$(echo "$RESPONSE" | jq -r ".status")"
+        PCT="$(   echo "$RESPONSE" | jq -r ".percentage")"
+        DESCR="$( echo "$RESPONSE" | jq -r ".description")"
+        echo "$(date): Status is $STATUS ($PCT) $DESCR"
+    done
+
+    if [ "$STATUS" == "ongoing" -a $NTIMES -le 0 ]; then
+        echo "Installation failed after $MAX_TIME minutes."
+        echo "RESPONSE: $RESPONSE"
+        exit 1
+    elif [ "$STATUS" != "completed" ]; then
+        echo "Installation failed."
+        echo "RESPONSE: $RESPONSE"
+        exit 1
+    fi
+
+    echo "Installation complete!"
+
+#10. Done
+    exit 0