From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
Date: Wed, 17 Jun 2020 12:52:01 +0000 (+0200)
Subject: iec, rec: Abstract out common seba_on_arm usecase
X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F81%2F3581%2F9;p=ci-management.git

iec, rec: Abstract out common seba_on_arm usecase

- move seba_on_arm IEC usecase to its own macros/templates files;
- enable seba_on_arm install/test jobs for REC deployments running
  on aarch64 baremetal nodes;
- replace dashes with underscores in jjb variables that need to be
  handled by the jinja2 parser, since jinja2 does not allow dashes
  in variable names;

While at it, add vim temporary files ('*.sw?') to .gitignore.

Change-Id: I62ae4ebfd4955fb484876e3a63c8f02a77e90b38
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
---

diff --git a/.gitignore b/.gitignore
index a475179..f28b1ac 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 .tox/
 archives/
 jenkins.ini
+*.sw?
 
 # Packer
 .galaxy/
diff --git a/jjb/akraino-templates/akraino-iec-macros.yaml b/jjb/akraino-templates/akraino-iec-macros.yaml
index 4db8f3a..8c2e20d 100644
--- a/jjb/akraino-templates/akraino-iec-macros.yaml
+++ b/jjb/akraino-templates/akraino-iec-macros.yaml
@@ -79,14 +79,6 @@
           <<: *k8s_ssh_key
           default: ''
 
-# IEC usecase(s) parameters
-- parameter:
-    name: 'seba_on_arm-defaults'
-    parameters:
-      - string:
-          name: PON_TYPE
-          default: 'ponsim'
-
 ########################
 # publisher macros
 ########################
@@ -97,21 +89,6 @@
           recipients: armband@enea.com
       - lf-infra-publish
 
-- publisher:
-    name: 'iec-install-usecase-publisher'
-    publishers:
-      - iec-default-publisher
-
-- publisher:
-    name: 'iec-test-usecase-publisher'
-    publishers:
-      - iec-default-publisher
-      - robot:
-          output-path: 'results'
-          report-html: '**/report.html'
-          log-html: '**/log.html'
-          output-xml: '**/output.xml'
-
 ########################
 # trigger macros
 ########################
diff --git a/jjb/akraino-templates/akraino-iec-templates.yaml b/jjb/akraino-templates/akraino-iec-templates.yaml
index ec2d46e..a3bfa34 100644
--- a/jjb/akraino-templates/akraino-iec-templates.yaml
+++ b/jjb/akraino-templates/akraino-iec-templates.yaml
@@ -4,7 +4,7 @@
     name: 'iec-{iecType}-{installer}-{deploy_type}-{os}-daily-{stream}'
     disabled: '{obj:disabled}'
     concurrent: false
-    node: '{slave-label}'
+    node: '{slave_label}'
     properties:
       - throttle:
           enabled: true
@@ -95,7 +95,7 @@
     id: akraino-iec-deploy
     name: 'iec-{iecType}-deploy-{installer}-{deploy_type}-{os}-daily-{stream}'
     concurrent: true
-    node: '{slave-label}'
+    node: '{slave_label}'
     properties: &properties
       - throttle:
           enabled: true
@@ -171,7 +171,7 @@
     id: akraino-iec-destroy
     name: 'iec-{iecType}-destroy-{installer}-{deploy_type}-{os}-daily-{stream}'
     concurrent: true
-    node: '{slave-label}'
+    node: '{slave_label}'
     properties: *properties
     parameters:
       - lf-infra-parameters:
@@ -201,32 +201,3 @@
 
     publishers:
       - iec-default-publisher
-
-- job-template:
-    id: akraino-iec-phase-usecase
-    name: 'iec-{iecType}-{phase}-{usecase}-{installer}-{deploy_type}-{os}-daily-{stream}'
-    concurrent: true
-    node: '{slave-label}'
-    properties: *properties
-
-    parameters:
-      - lf-infra-parameters:
-          project: '{project}'
-          stream: '{stream}'
-          branch: '{branch}'
-      - '{project}-defaults'
-      - '{installer}-{os}-defaults'
-      - '{usecase}-defaults'
-
-    wrappers:
-      - build-name:
-          name: '$BUILD_NUMBER - Usecase: ${usecase} DeployType: ${installer}-${deploy_type}'
-
-    scm: *scm
-
-    builders:
-      - shell:
-          !include-raw-escape: ../iec/iec-{usecase}.sh
-
-    publishers:
-      - 'iec-{phase}-usecase-publisher'
diff --git a/jjb/akraino-templates/akraino-rec-templates.yaml b/jjb/akraino-templates/akraino-rec-templates.yaml
index 1f02ef8..b92d8e4 100644
--- a/jjb/akraino-templates/akraino-rec-templates.yaml
+++ b/jjb/akraino-templates/akraino-rec-templates.yaml
@@ -1,10 +1,10 @@
 ---
 - job-template:
     id: akraino-rec-deploy
-    name: 'rec-{slave-label}-{job-frequency}-{stream}'
+    name: 'rec-{slave_label}-{job_frequency}-{stream}'
     disabled: '{obj:disabled}'
     concurrent: false
-    node: '{slave-label}'
+    node: '{slave_label}'
     properties:
       - throttle:
           enabled: true
@@ -17,21 +17,21 @@
           name: '$BUILD_NUMBER - Install REC Blueprint'
 
     triggers:
-      - 'rec-{job-frequency}-{stream}-trigger'
+      - 'rec-{job_frequency}-{stream}-trigger'
 
     parameters:
       - lf-infra-parameters:
           project: '{project}'
           stream: '{stream}'
           branch: '{branch}'
-      - 'rec-{slave-label}-defaults'
+      - 'rec-{slave_label}-defaults'
       - 'rec-defaults'
 
     builders:
       - description-setter:
           description: "SLAVE: $NODE_NAME"
       - trigger-builds:
-          - project: 'rec-{slave-label}-install-rec-{job-frequency}-{stream}'
+          - project: 'rec-{slave_label}-install-rec-{job_frequency}-{stream}'
             current-parameters: true
             same-node: true
             block: true
@@ -45,6 +45,22 @@
               OPTIONAL=false
               PULL=yes
             block: true
+          - project:
+              !j2: |
+                {%- for uc in usecase -%}
+                {%- for ph in phase -%}
+                rec-{{ slave_label }}-{{ ph }}-{{ uc }}-{{ job_frequency }}-{{ stream }}
+                {%- if not loop.last -%},{%- endif -%}
+                {%- endfor %}
+                {%- if not loop.last -%},{%- endif -%}
+                {%- endfor %}
+            current-parameters: true
+            same-node: true
+            block: true
+            block-thresholds:
+              build-step-failure-threshold: 'never'
+              failure-threshold: 'never'
+              unstable-threshold: 'FAILURE'
 
     publishers: &rec-publishers
       - email:
@@ -53,9 +69,9 @@
 
 - job-template:
     id: akraino-rec-install-rec
-    name: 'rec-{slave-label}-install-rec-{job-frequency}-{stream}'
+    name: 'rec-{slave_label}-install-rec-{job_frequency}-{stream}'
     concurrent: true
-    node: '{slave-label}'
+    node: '{slave_label}'
     properties:
       - throttle:
           enabled: true
@@ -68,7 +84,7 @@
           project: '{project}'
           stream: '{stream}'
           branch: '{branch}'
-      - 'rec-{slave-label}-defaults'
+      - 'rec-{slave_label}-defaults'
       - 'rec-defaults'
 
     scm:
diff --git a/jjb/akraino-templates/akraino-usecase-macros.yaml b/jjb/akraino-templates/akraino-usecase-macros.yaml
new file mode 100644
index 0000000..f697f9c
--- /dev/null
+++ b/jjb/akraino-templates/akraino-usecase-macros.yaml
@@ -0,0 +1,30 @@
+---
+##############
+# PARAMETERS #
+##############
+- parameter:
+    name: 'seba_on_arm-defaults'
+    parameters:
+      - string:
+          name: PON_TYPE
+          default: 'ponsim'
+
+########################
+# publisher macros
+########################
+- publisher:
+    name: 'seba_on_arm-install-publisher'
+    publishers:
+      - email:
+          recipients: armband@enea.com
+      - lf-infra-publish
+
+- publisher:
+    name: 'seba_on_arm-test-publisher'
+    publishers:
+      - seba_on_arm-install-publisher
+      - robot:
+          output-path: 'results'
+          report-html: '**/report.html'
+          log-html: '**/log.html'
+          output-xml: '**/output.xml'
diff --git a/jjb/akraino-templates/akraino-usecase-templates.yaml b/jjb/akraino-templates/akraino-usecase-templates.yaml
new file mode 100644
index 0000000..2e245ea
--- /dev/null
+++ b/jjb/akraino-templates/akraino-usecase-templates.yaml
@@ -0,0 +1,60 @@
+---
+- job-template: &akraino-usecase-phase-rec
+    id: akraino-usecase-phase-rec
+    name: 'rec-{slave_label}-{phase}-{usecase}-{job_frequency}-{stream}'
+    concurrent: true
+    node: '{slave_label}'
+    # Support usecases that reside in a different project repo, e.g.
+    # seba_on_arm usecase uses the 'iec' project git repo, even if this
+    # job targets a different project, 'rec'.
+    usecase_project: 'iec'
+    properties:
+      - throttle:
+          enabled: true
+          max-total: 4
+          max-per-node: 1
+          option: 'project'
+      - build-blocker:
+          use-build-blocker: true
+          blocking-jobs:
+            - '.*-{usecase}-.*'
+          block-level: 'NODE'
+
+    parameters:
+      - lf-infra-parameters:
+          project: '{usecase_project}'
+          stream: '{stream}'
+          branch: '{branch}'
+      - '{project}-defaults'
+      - '{usecase}-defaults'
+
+    scm:
+      - lf-infra-gerrit-scm:
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+          git-url: '{git-url}/{usecase_project}.git'
+          refspec: '$GERRIT_REFSPEC'
+          branch: '$GERRIT_BRANCH'
+          submodule-recursive: false
+          submodule-timeout: '{submodule-timeout}'
+          submodule-disable: true
+          choosing-strategy: default
+
+    builders:
+      - shell:
+          !include-raw-escape: ../shell/{usecase}.sh
+
+    publishers:
+      - '{usecase}-{phase}-publisher'
+
+- job-template:
+    <<: *akraino-usecase-phase-rec
+    id: akraino-usecase-phase-iec
+    name: 'iec-{iecType}-{phase}-{usecase}-{installer}-{deploy_type}-{os}-daily-{stream}'
+    parameters:
+      - lf-infra-parameters:
+          project: '{usecase_project}'
+          stream: '{stream}'
+          branch: '{branch}'
+      - '{project}-defaults'
+      - '{installer}-{os}-defaults'
+      - '{usecase}-defaults'
diff --git a/jjb/iec/iec.yaml b/jjb/iec/iec.yaml
index f6d9df8..5819d3c 100644
--- a/jjb/iec/iec.yaml
+++ b/jjb/iec/iec.yaml
@@ -43,10 +43,10 @@
     # -------------------------------
     deploy_type:
       - baremetal:
-          slave-label: 'aarch64_baremetal_dev'
+          slave_label: 'aarch64_baremetal_dev'
           scenario: 'k8-calico-iec-noha'
       - virtual:
-          slave-label: 'aarch64_dev'
+          slave_label: 'aarch64_dev'
           scenario: 'k8-calico-iec-noha'
 
     exclude:
@@ -84,7 +84,7 @@
       - akraino-iec-install
       - akraino-iec-deploy
       - akraino-iec-destroy
-      - akraino-iec-phase-usecase
+      - akraino-usecase-phase-iec
       - gerrit-sonar:
           sonar-project-file: ""
           build-node: centos7-builder-2c-1g
diff --git a/jjb/rec/rec.yaml b/jjb/rec/rec.yaml
index ae55479..c41d57f 100644
--- a/jjb/rec/rec.yaml
+++ b/jjb/rec/rec.yaml
@@ -14,15 +14,25 @@
           disabled: false
 
     # -------------------------------
-    # POD DEFINITIONS
+    # USECASE DEFINITIONS
     # -------------------------------
+    usecase:
+      - seba_on_arm
+
+    phase:
+      - install
+      - test
 
-    slave-label:
+    # -------------------------------
+    # POD DEFINITIONS
+    # -------------------------------
+    slave_label:
       - aarch64_baremetal
 
-    job-frequency:
+    job_frequency:
       - weekly
 
     jobs:
-       - akraino-rec-deploy
-       - akraino-rec-install-rec
+      - akraino-rec-deploy
+      - akraino-rec-install-rec
+      - akraino-usecase-phase-rec
diff --git a/jjb/iec/iec-seba_on_arm.sh b/jjb/shell/seba_on_arm.sh
similarity index 78%
rename from jjb/iec/iec-seba_on_arm.sh
rename to jjb/shell/seba_on_arm.sh
index b0a062c..c695f92 100755
--- a/jjb/iec/iec-seba_on_arm.sh
+++ b/jjb/shell/seba_on_arm.sh
@@ -37,24 +37,18 @@ run_on_k8s_master () {
   then
     eval sshpass -p "${K8S_SSH_PASSWORD}" "${_c}" \
          -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-         ${_o} "${K8S_SSH_USER}"@"${K8S_MASTER_IP}${_s}" "$@"
+         "${K8S_SSH_USER}@${K8S_MASTER_IP}${_s}" "$@"
   elif [ -n "${K8S_SSH_KEY}" ]
   then
-    eval ${_c} -i "${K8S_SSH_KEY}" \
+    eval "${_c}" -i "${K8S_SSH_KEY}" \
          -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-         ${_o} "${K8S_SSH_USER}"@"${K8S_MASTER_IP}${_s}" "$@"
+         "${K8S_SSH_USER}@${K8S_MASTER_IP}${_s}" "$@"
   else
     echo "Neither K8S_SSH_PASSWORD nor K8S_SSH_KEY are set"
     exit 1
   fi
 }
 
-if [ -z "$K8S_SSH_USER" ]
-then
-  echo "K8S_SSH_USER not set, cannot ssh to K8S master, aborting"
-  exit 1
-fi
-
 case "${JOB_NAME}" in
   *fuel*)
     K8S_MASTER_IP=$(docker exec fuel cat /etc/hosts | grep iec01 \
@@ -69,18 +63,35 @@ case "${JOB_NAME}" in
     IEC_DIR="/${K8S_SSH_USER}/iec"
     KUBE_DIR="~${K8S_SSH_USER}/.kube"
     ;;
+  *rec*)
+    # CLUSTER_{MASTER_IP,SSH_USER,SSH_PASSWORD} are already set by job params
+    K8S_MASTER_IP=${CLUSTER_MASTER_IP}
+    K8S_SSH_USER=${CLUSTER_SSH_USER}
+    K8S_SSH_PASSWORD=${CLUSTER_SSH_PASSWORD}
+    IEC_DIR="/var/lib/akraino/iec"
+    KUBE_DIR="~${K8S_SSH_USER}/.kube"
+    UPSTREAM_PROJECT='rec'
+    ;;
   *)
     echo "Cannot determine installer from ${JOB_NAME}"
     exit 1
     ;;
 esac
 
+if [ -z "$K8S_SSH_USER" ]
+then
+  echo "K8S_SSH_USER not set, cannot ssh to K8S master, aborting"
+  exit 1
+fi
+
+export K8S_MASTER_IP UPSTREAM_PROJECT="${UPSTREAM_PROJECT:-${PROJECT}}"
+
 case "${JOB_NAME}" in
-  iec-*-install-seba_on_arm*)
-    INSTALL_CMD="'cd ${IEC_DIR}/src/use_cases/seba_on_arm/install; ./install.sh'"
+  *-install-seba_on_arm*)
+    INSTALL_CMD="'cd ${IEC_DIR}/src/use_cases/seba_on_arm/install; ./install.sh ${UPSTREAM_PROJECT}'"
     run_on_k8s_master ssh "${INSTALL_CMD}"
     ;;
-  iec-*-test-seba_on_arm*)
+  *-test-seba_on_arm*)
 
     case "${PON_TYPE}" in
       *ponsim*)
@@ -96,11 +107,11 @@ case "${JOB_NAME}" in
     run_on_k8s_master scp "${KUBE_DIR}" "${WORKSPACE}"
 
     SEBA_TEST_DIR="src/use_cases/seba_on_arm/test"
-    TEST_CMD="${SEBA_TEST_DIR}/${PON_TYPE}/test.sh"
+    TEST_CMD="${SEBA_TEST_DIR}/${PON_TYPE}/test.sh ${UPSTREAM_PROJECT}"
     echo "Issuing command"
     echo "${TEST_CMD}"
 
-    eval ${TEST_CMD}
+    eval "${TEST_CMD}"
     ;;
   *)
     echo "Cannot determine what to do for seba_on_arm from ${JOB_NAME}"