From: wutianwei <wutianwei1@huawei.com>
Date: Thu, 26 Sep 2019 03:53:56 +0000 (+0000)
Subject: Merge "add the jenkins job yaml and scripts"
X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=commitdiff_plain;h=8d0af6a3f7261356224b6329fb8a63418e2dae91;hp=e299906980a364a46c3566a5ea30be86e894e826;p=eliot.git

Merge "add the jenkins job yaml and scripts"
---

diff --git a/jjb/eliot/eliot-ci-jobs.yaml b/jjb/eliot/eliot-ci-jobs.yaml
new file mode 100644
index 0000000..4db508c
--- /dev/null
+++ b/jjb/eliot/eliot-ci-jobs.yaml
@@ -0,0 +1,154 @@
+---
+- project:
+
+    name: eliot-ci
+
+    installer: 'eliot'
+
+    project: 'eliot'
+
+    master: &master
+      stream: master
+      branch: '{stream}'
+      gs-pathname: ''
+      ppa-pathname: '/{stream}'
+      disabled: false
+    # -------------------------------
+    # POD, INSTALLER, AND BRANCH MAPPING
+    # -------------------------------
+    #        CI PODs
+    # -------------------------------
+    pod:
+      - kubeedge-virtual:
+          slave-label: eliot-kubeedge-virtual
+          <<: *master
+          auto-trigger-name: 'eliot-deploy-{pod}-daily-{stream}'
+      - k8s-virtual:
+          slave-label: eliot-k8s-virtual
+          <<: *master
+          auto-trigger-name: 'eliot-deploy-{pod}-daily-{stream}'
+      - k8s-centos-virtual:
+          slave-label: eliot-k8s-centos-virtual
+          <<: *master
+          auto-trigger-name: 'eliot-deploy-{pod}-daily-{stream}'
+
+
+    jobs:
+      - 'eliot-deploy-{pod}-daily-{stream}'
+
+########################
+# job templates
+########################
+- job-template:
+    name: 'eliot-deploy-{pod}-daily-{stream}'
+
+    disabled: '{obj:disabled}'
+
+    concurrent: true
+
+    properties:
+      - logrotate-default
+      - throttle:
+          enabled: true
+          max-per-node: 1
+          option: 'project'
+      - build-blocker:
+          use-build-blocker: true
+          blocking-jobs:
+            - 'eliot-deploy-{pod}-daily-.*?'
+            - 'eliot-verify-deploy-.*?'
+          blocking-level: 'NODE'
+
+
+    wrappers:
+      - build-name:
+          name: '$BUILD_NUMBER'
+      - timeout:
+          timeout: 360
+          abort: true
+      - fix-workspace-permissions
+
+    parameters:
+      - project-parameter:
+          project: '{project}'
+          branch: '{branch}'
+      - eliot-ci-parameter:
+          installer: '{installer}'
+          gs-pathname: '{gs-pathname}'
+          ppa-pathname: '{ppa-pathname}'
+      - '{slave-label}-defaults'
+      - '{installer}-defaults'
+
+    triggers:
+      - '{auto-trigger-name}'
+
+    scm:
+      - git-scm
+
+    builders:
+      - description-setter:
+          description: "POD: $NODE_NAME"
+      - conditional-step:
+          condition-kind: regex-match
+          regex: "k8s-.*"
+          label: '{pod}'
+          steps:
+            - shell:
+                !include-raw-escape: ./eliot-clean.sh
+            - shell:
+                !include-raw-escape: ./eliot-deploy.sh
+      - conditional-step:
+          condition-kind: regex-match
+          regex: "kubeedge-.*"
+          label: '{pod}'
+          steps:
+            - shell:
+                !include-raw-escape: ./eliot-kubeedge-clean.sh
+            - shell:
+                !include-raw-escape: ./eliot-kubeedge-deploy.sh
+
+########################
+# parameter macros
+########################
+- parameter:
+    name: eliot-ci-parameter
+    parameters:
+      - string:
+          name: GS_URL
+          default: '$GS_BASE{gs-pathname}'
+          description: "URL to Google Storage."
+      - string:
+          name: PPA_REPO
+          default: ""
+      - string:
+          name: PPA_CACHE
+          default: "$WORKSPACE/work/repo/"
+      - string:
+          name: LOG_DIRECTORY
+          default: $WORKSPACE/log_output
+          description: ""
+
+
+########################
+# trigger macros
+########################
+
+
+# -----------------
+# virtual-master
+# -----------------
+
+- trigger:
+    name: 'eliot-deploy-kubeedge-virtual-daily-master'
+    triggers:
+      - timed: '0 20 * * *'
+
+- trigger:
+    name: 'eliot-deploy-k8s-virtual-daily-master'
+    triggers:
+      - timed: '0 21 * * *'
+
+- trigger:
+    name: 'eliot-deploy-k8s-centos-virtual-daily-master'
+    triggers:
+      - timed: '0 23 * * *'
diff --git a/jjb/eliot/eliot-clean.sh b/jjb/eliot/eliot-clean.sh
new file mode 100755
index 0000000..2478f21
--- /dev/null
+++ b/jjb/eliot/eliot-clean.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -x
+
+# log info to console
+echo "cc"
+echo "--------------------------------------------------------"
+echo
+
+node_name=$(kubectl get node  |awk '{print $1}' |grep -v NAME) ||true
+for i in $node_name;
+do
+  kubectl drain $i  --delete-local-data --force --ignore-daemonsets
+  kubectl delete node  $i
+done
+
+if [[ "$NODE_NAME" =~ "huawei-eliot-pod1" ]]; then
+    sshpass -f /home/jenkins/pwfile.txt ssh root@10.10.0.99 sudo kubeadm reset -f ||true
+    sshpass -f /home/jenkins/pwfile.txt ssh root@10.10.1.15 sudo kubeadm reset -f ||true
+elif [[ "$NODE_NAME" =~ "huawei-eliot-centos-pod3" ]]; then
+    sshpass -f /home/jenkins/pwfile.txt ssh root@10.10.0.211 sudo kubeadm reset -f ||true
+    sshpass -f /home/jenkins/pwfile.txt ssh root@10.10.0.138 sudo kubeadm reset -f ||true
+
+fi
+
+echo
+echo "--------------------------------------------------------"
+echo "Cleaned"
diff --git a/jjb/eliot/eliot-deploy.sh b/jjb/eliot/eliot-deploy.sh
new file mode 100644
index 0000000..612f32b
--- /dev/null
+++ b/jjb/eliot/eliot-deploy.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -x
+
+echo "CLEANUP Prometheus and Cadvisor started...."
+echo "-------------------------------------------------------"
+cd $WORKSPACE/scripts/ci_management
+./cleanup_prom_cadvisor.sh
+
+sshpass -f /home/jenkins/pwfile.txt ssh root@10.10.0.99 "bash -s" < ./uninstall_cadvisor.sh || true
+
+
+echo "CLEANUP prometheus and Cadvisor ended......"
+echo "-------------------------------------------------------"
+
+# log info to console
+echo "Starting the deployment on baremetal environment using $INSTALLER_TYPE. This could take some time..."
+echo "--------------------------------------------------------"
+echo
+
+rm -rf $WORKSPACE/scripts/nodelist
+
+cp /home/jenkins/nodelist $WORKSPACE/scripts/nodelist
+
+cd $WORKSPACE/scripts/
+./setup.sh
+
+if [ $? -ne 0 ]; then
+    echo "depolyment failed!"
+    deploy_ret=1
+fi
+
+sleep 60
+kubectl get nodes
+
+echo "Logs uploading to nexus repo"
+
+export ARCHIVE_ARTIFACTS="**/*.log"
+export NEXUS_URL=https://nexus.akraino.org
+export SILO=huawei
+export JENKINS_HOSTNAME=http://159.138.47.166:8080
+export BUILD_URL="${JENKINS_HOSTNAME}/job/${JOB_NAME}/${BUILD_NUMBER}"
+export NEXUS_PATH="${SILO}/job/${JOB_NAME}/${BUILD_NUMBER}"
+lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE"
+lftools deploy logs $NEXUS_URL $NEXUS_PATH $BUILD_URL
+
+echo $BUILD_URL
+
+echo "Logs uploaded to $NEXUS_URL/content/sites/logs/$NEXUS_PATH"
+
+echo
+echo "--------------------------------------------------------"
+echo "Done!"
+
+exit $deploy_ret
+
diff --git a/jjb/eliot/eliot-kubeedge-clean.sh b/jjb/eliot/eliot-kubeedge-clean.sh
new file mode 100755
index 0000000..28d5ad2
--- /dev/null
+++ b/jjb/eliot/eliot-kubeedge-clean.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -x
+
+# log info to console
+echo "Cleaning up environment"
+echo "--------------------------------------------------------"
+echo
+
+rm -rf $WORKSPACE/scripts/src/nodelist
+cp /home/jenkins/nodelist_kubeedge $WORKSPACE/scripts/src/nodelist
+
+rm -rf  /root/go/src/github.com/kubeedge/kubeedge
+
+cd $WORKSPACE/ci_management
+./cleanup.sh
+
+node_name=$(kubectl get node  |awk '{print $1}' |grep -v NAME) ||true
+for i in $node_name;
+do
+  kubectl drain $i  --delete-local-data --force --ignore-daemonsets
+  kubectl delete node  $i
+done
+
+sshpass  ssh root@10.10.0.54 sudo kubeadm reset -f ||true
+sshpass  ssh root@10.10.0.45 sudo kubeadm reset -f ||true
+
+echo
+echo "--------------------------------------------------------"
+echo "Cleaned"
+
diff --git a/jjb/eliot/eliot-kubeedge-deploy.sh b/jjb/eliot/eliot-kubeedge-deploy.sh
new file mode 100644
index 0000000..8c2c810
--- /dev/null
+++ b/jjb/eliot/eliot-kubeedge-deploy.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -x
+
+# log info to console
+echo "Starting the deployment on baremetal environment using $INSTALLER_TYPE. This could take some time..."
+echo "--------------------------------------------------------"
+echo
+
+rm -rf $WORKSPACE/scripts/src/config_kubeedge
+
+cp /home/jenkins/nodelist_kubeedge $WORKSPACE/scripts/src/config_kubeedge
+
+cd $WORKSPACE/scripts/src
+source /root/.profile
+sudo ./kubeedge_setup.sh
+
+if [ $? -ne 0 ]; then
+    echo "depolyment failed!"
+    deploy_ret=1
+fi
+
+
+echo "Logs uploading to nexus repo"
+
+export ARCHIVE_ARTIFACTS="**/*.log"
+export NEXUS_URL=https://nexus.akraino.org
+export SILO=huawei
+export JENKINS_HOSTNAME=http://159.138.47.166:8080
+export BUILD_URL="${JENKINS_HOSTNAME}/job/${JOB_NAME}/${BUILD_NUMBER}"
+export NEXUS_PATH="${SILO}/job/${JOB_NAME}/${BUILD_NUMBER}"
+lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE"
+lftools deploy logs $NEXUS_URL $NEXUS_PATH $BUILD_URL
+
+echo $BUILD_URL
+
+echo "Logs uploaded to $NEXUS_URL/content/sites/logs/$NEXUS_PATH"
+
+
+
+echo
+echo "--------------------------------------------------------"
+echo "Done!"
+
+exit $deploy_ret
+
diff --git a/jjb/global/base-macros.yaml b/jjb/global/base-macros.yaml
new file mode 100644
index 0000000..92d315c
--- /dev/null
+++ b/jjb/global/base-macros.yaml
@@ -0,0 +1,285 @@
+---
+# Base macros
+#
+# NOTE: make sure macros are listed in execution ordered.
+#
+# 1. parameters/properties
+# 2. scm
+# 3. triggers
+# 4. wrappers
+# 5. prebuilders (maven only, configured like Builders)
+# 6. builders (maven, freestyle, matrix, etc..)
+# 7. postbuilders (maven only, configured like Builders)
+# 8. publishers/reporters/notifications
+
+- parameter:
+    name: project-parameter
+    parameters:
+      - string:
+          name: PROJECT
+          default: '{project}'
+          description: "JJB configured PROJECT parameter"
+      - string:
+          name: GS_BASE
+          default: artifacts.akraino.org/$PROJECT
+          description: "URL to Google Storage."
+      - string:
+          name: BRANCH
+          default: '{branch}'
+          description: "JJB configured BRANCH parameter"
+      - string:
+          name: GERRIT_BRANCH
+          default: '{branch}'
+          description: "JJB configured GERRIT_BRANCH parameter (deprecated)"
+      - string:
+          name: GERRIT_REFSPEC
+          default: 'refs/heads/{branch}'
+          description: "Default refspec needed for manually triggering."
+
+
+- property:
+    name: logrotate-default
+    properties:
+      - build-discarder:
+          days-to-keep: 60
+          num-to-keep: 200
+          artifact-days-to-keep: 60
+          artifact-num-to-keep: 200
+
+- scm:
+    name: git-scm
+    scm:
+      - git: &git-scm-defaults
+          credentials-id: 'jenkins-ssh'
+          url: '$GIT_BASE'
+          branches:
+            - 'origin/$BRANCH'
+          timeout: 15
+          per-build-tag: false
+          skip-tag: true
+          shallow-clone: false
+          use-author: false
+          ignore-notify: false
+          wipe-workspace: true
+          prune: false
+
+- scm:
+    name: git-scm-gerrit
+    scm:
+      - git:
+          choosing-strategy: 'gerrit'
+          refspec: '$GERRIT_REFSPEC'
+          <<: *git-scm-defaults
+
+- scm:
+    name: git-scm-gerrit-with-submodules
+    scm:
+      - git:
+          choosing-strategy: 'gerrit'
+          refspec: '$GERRIT_REFSPEC'
+          submodule:
+            recursive: true
+            timeout: 20
+          <<: *git-scm-defaults
+
+- trigger:
+    name: 'daily-trigger-disabled'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'weekly-trigger-disabled'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: gerrit-trigger-patchset-created
+    triggers:
+      - gerrit:
+          server-name: 'gerrit.akraino.org'
+          trigger-on:
+            - patchset-created-event:
+                exclude-drafts: 'false'
+                exclude-trivial-rebase: 'false'
+                exclude-no-code-change: 'false'
+            - draft-published-event
+            - comment-added-contains-event:
+                comment-contains-value: 'recheck'
+            - comment-added-contains-event:
+                comment-contains-value: 'reverify'
+          projects:
+            - project-compare-type: 'ANT'
+              project-pattern: '{project}'
+              branches:
+                - branch-compare-type: 'ANT'
+                  branch-pattern: '**/{branch}'
+              file-paths:
+                - compare-type: 'ANT'
+                  pattern: '{files}'
+          skip-vote:
+            successful: false
+            failed: false
+            unstable: false
+            notbuilt: false
+
+- wrapper:
+    name: ssh-agent-wrapper
+    wrappers:
+      - ssh-agent-credentials:
+          users:
+            - 'jenkins-ssh'
+
+- wrapper:
+    name: build-timeout
+    wrappers:
+      - timeout:
+          timeout: '{timeout}'
+          timeout-var: 'BUILD_TIMEOUT'
+          fail: true
+
+- wrapper:
+    name: fix-workspace-permissions
+    wrappers:
+      - pre-scm-buildstep:
+          - shell: |
+             #!/bin/bash
+             sudo chown -R $USER:$USER $WORKSPACE || exit 1
+
+- builder:
+    name: upload-under-review-docs-to-akraino-artifacts
+    builders:
+      - shell: |
+          #!/bin/bash
+          set -o errexit
+          set -o pipefail
+          set -o xtrace
+          export PATH=$PATH:/usr/local/bin/
+
+          [[ $GERRIT_CHANGE_NUMBER =~ .+ ]]
+          [[ -d docs/_build/ ]] || exit 0
+
+          echo
+          echo "###########################"
+          echo "UPLOADING DOCS UNDER REVIEW"
+          echo "###########################"
+          echo
+
+          gs_base="artifacts.akraino.org/$PROJECT/review"
+          gs_path="$gs_base/$GERRIT_CHANGE_NUMBER"
+          local_path="upload/$GERRIT_CHANGE_NUMBER"
+
+          mkdir -p upload
+          mv docs/_build/html/ "$local_path"
+          gsutil -m cp -r "$local_path" "gs://$gs_base"
+
+          gsutil -m setmeta \
+              -h "Content-Type:text/html" \
+              -h "Cache-Control:private, max-age=0, no-transform" \
+              "gs://$gs_path"/**.html > /dev/null 2>&1
+
+          echo "Document link(s):" >> gerrit_comment.txt
+          find "$local_path" | grep -e 'index.html$' -e 'pdf$' | \
+              sed -e "s|^$local_path|    http://$gs_path|" >> gerrit_comment.txt
+
+# To take advantage of this macro, have your build write
+# out the file 'gerrit_comment.txt' with information to post
+# back to gerrit and include this macro in the list of builders.
+- builder:
+    name: report-build-result-to-gerrit
+    builders:
+      - shell: |
+          #!/bin/bash
+          set -o errexit
+          set -o pipefail
+          set -o xtrace
+          export PATH=$PATH:/usr/local/bin/
+          if [[ -e gerrit_comment.txt ]] ; then
+              echo
+              echo "posting review comment to gerrit..."
+              echo
+              cat gerrit_comment.txt
+              echo
+              ssh -p 29418 gerrit.akraino.org \
+                  "gerrit review -p $GERRIT_PROJECT \
+                   -m '$(cat gerrit_comment.txt)' \
+                   $GERRIT_PATCHSET_REVISION \
+                   --notify NONE"
+          fi
+
+- builder:
+    name: remove-old-docs-from-akraino-artifacts
+    builders:
+      - shell: |
+          #!/bin/bash
+          set -o errexit
+          set -o pipefail
+          set -o xtrace
+          export PATH=$PATH:/usr/local/bin/
+
+          [[ $GERRIT_CHANGE_NUMBER =~ .+ ]]
+
+          gs_path="artifacts.akraino.org/$PROJECT/review/$GERRIT_CHANGE_NUMBER"
+
+          if gsutil ls "gs://$gs_path" > /dev/null 2>&1 ; then
+              echo
+              echo "Deleting Out-of-dated Documents..."
+              gsutil -m rm -r "gs://$gs_path"
+          fi
+          gs_path="artifacts.akraino.org/review/$GERRIT_CHANGE_NUMBER"
+
+          if gsutil ls "gs://$gs_path" > /dev/null 2>&1 ; then
+              echo
+              echo "Deleting Out-of-dated Documents..."
+              gsutil -m rm -r "gs://$gs_path"
+          fi
+
+- builder:
+    name: upload-review-docs
+    builders:
+      - upload-under-review-docs-to-akraino-artifacts
+      - report-build-result-to-gerrit
+
+
+- builder:
+    name: clean-workspace
+    builders:
+      - shell: |
+          #!/bin/bash
+          set -o errexit
+          set -o nounset
+          set -o pipefail
+          sudo /bin/rm -rf "$WORKSPACE"
+
+- builder:
+    name: clean-workspace-log
+    builders:
+      - shell: |
+          find $WORKSPACE -type f -name '*.log' | xargs rm -f
+
+- publisher:
+    name: archive-artifacts
+    publishers:
+      - archive:
+          artifacts: '{artifacts}'
+          allow-empty: true
+          fingerprint: true
+          latest-only: true
+
+- publisher:
+    name: publish-coverage
+    publishers:
+      - cobertura:
+          report-file: "coverage.xml"
+          only-stable: "true"
+          health-auto-update: "false"
+          stability-auto-update: "false"
+          zoom-coverage-chart: "true"
+          targets:
+            - files:
+                healthy: 10
+                unhealthy: 20
+                failing: 30
+            - method:
+                healthy: 50
+                unhealthy: 40
+                failing: 30
diff --git a/jjb/global/defaults.yaml b/jjb/global/defaults.yaml
new file mode 100644
index 0000000..3e9f8e9
--- /dev/null
+++ b/jjb/global/defaults.yaml
@@ -0,0 +1,10 @@
+---
+# jjb defaults
+- defaults:
+    name: global
+
+    gerrit-server-name: Primary
+    git-url: 'ssh://akraino-jobbuilder@gerrit.akraino.org:29418'
+    jenkins-ssh-credential: jenkins-ssh
+    lftools-version: '<1.0.0'
+    submodule-timeout: 10
diff --git a/jjb/global/project-params.yaml b/jjb/global/project-params.yaml
new file mode 100644
index 0000000..a23a57c
--- /dev/null
+++ b/jjb/global/project-params.yaml
@@ -0,0 +1,16 @@
+---
+- parameter:
+    name: 'eliot-defaults'
+    parameters:
+      - string:
+          name: NODE_IP
+          default: ""
+          description: 'IP of the node'
+      - string:
+          name: USERNAME
+          default: ""
+          description: 'username of the node'
+      - string:
+          name: PASSWORD
+          default: ""
+          description: 'password of the node'
diff --git a/jjb/global/slave-params.yaml b/jjb/global/slave-params.yaml
new file mode 100644
index 0000000..b56fd46
--- /dev/null
+++ b/jjb/global/slave-params.yaml
@@ -0,0 +1,39 @@
+---
+- parameter:
+    name: 'eliot-kubeedge-virtual-defaults'
+    parameters:
+      - label:
+          name: SLAVE_LABEL
+          default: 'eliot-kubeedge-virtual'
+          all-nodes: false
+          node-eligibility: 'ignore-offline'
+      - string:
+          name: GIT_BASE
+          default: https://gerrit.akraino.org/r/$PROJECT
+          description: 'Git URL to use on this Jenkins Slave'
+
+- parameter:
+    name: 'eliot-k8s-virtual-defaults'
+    parameters:
+      - label:
+          name: SLAVE_LABEL
+          default: 'eliot-k8s-virtual'
+          all-nodes: false
+          node-eligibility: 'ignore-offline'
+      - string:
+          name: GIT_BASE
+          default: https://gerrit.akraino.org/r/$PROJECT
+          description: 'Git URL to use on this Jenkins Slave'
+
+- parameter:
+    name: 'eliot-k8s-centos-virtual-defaults'
+    parameters:
+      - label:
+          name: SLAVE_LABEL
+          default: 'eliot-k8s-centos-virtual'
+          all-nodes: false
+          node-eligibility: 'ignore-offline'
+      - string:
+          name: GIT_BASE
+          default: https://gerrit.akraino.org/r/$PROJECT
+          description: 'Git URL to use on this Jenkins Slave'