From: wutianwei <wutianwei1@huawei.com>
Date: Wed, 7 Aug 2019 03:45:53 +0000 (+0800)
Subject: add the jenkins job yaml and scripts
X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=commitdiff_plain;h=08061086d68d8c4f0a027c910c5e498b1016e12a;hp=--cc;p=eliot.git

add the jenkins job yaml and scripts

when need create or update the jenkins job
follow the steps as below
1. git clone the code to jenkins server
2. modify the yaml and scripts
3. run the command "jenkins-jobs update  global/:eliot/" in eliot/jjb directory.

Signed-off-by: wutianwei <wutianwei1@huawei.com>
Change-Id: Ia692f66bc3824f71c07abf3b389e0b5b8e336257
---

08061086d68d8c4f0a027c910c5e498b1016e12a
diff --git a/jjb/eliot/eliot-ci-jobs.yaml b/jjb/eliot/eliot-ci-jobs.yaml
new file mode 100644
index 0000000..4db508c
--- /dev/null
+++ b/jjb/eliot/eliot-ci-jobs.yaml
@@ -0,0 +1,154 @@
+---
+- project:
+
+    name: eliot-ci
+
+    installer: 'eliot'
+
+    project: 'eliot'
+
+    master: &master
+      stream: master
+      branch: '{stream}'
+      gs-pathname: ''
+      ppa-pathname: '/{stream}'
+      disabled: false
+    # -------------------------------
+    # POD, INSTALLER, AND BRANCH MAPPING
+    # -------------------------------
+    #        CI PODs
+    # -------------------------------
+    pod:
+      - kubeedge-virtual:
+          slave-label: eliot-kubeedge-virtual
+          <<: *master
+          auto-trigger-name: 'eliot-deploy-{pod}-daily-{stream}'
+      - k8s-virtual:
+          slave-label: eliot-k8s-virtual
+          <<: *master
+          auto-trigger-name: 'eliot-deploy-{pod}-daily-{stream}'
+      - k8s-centos-virtual:
+          slave-label: eliot-k8s-centos-virtual
+          <<: *master
+          auto-trigger-name: 'eliot-deploy-{pod}-daily-{stream}'
+
+
+    jobs:
+      - 'eliot-deploy-{pod}-daily-{stream}'
+
+########################
+# job templates
+########################
+- job-template:
+    name: 'eliot-deploy-{pod}-daily-{stream}'
+
+    disabled: '{obj:disabled}'
+
+    concurrent: true
+
+    properties:
+      - logrotate-default
+      - throttle:
+          enabled: true
+          max-per-node: 1
+          option: 'project'
+      - build-blocker:
+          use-build-blocker: true
+          blocking-jobs:
+            - 'eliot-deploy-{pod}-daily-.*?'
+            - 'eliot-verify-deploy-.*?'
+          blocking-level: 'NODE'
+
+
+    wrappers:
+      - build-name:
+          name: '$BUILD_NUMBER'
+      - timeout:
+          timeout: 360
+          abort: true
+      - fix-workspace-permissions
+
+    parameters:
+      - project-parameter:
+          project: '{project}'
+          branch: '{branch}'
+      - eliot-ci-parameter:
+          installer: '{installer}'
+          gs-pathname: '{gs-pathname}'
+          ppa-pathname: '{ppa-pathname}'
+      - '{slave-label}-defaults'
+      - '{installer}-defaults'
+
+    triggers:
+      - '{auto-trigger-name}'
+
+    scm:
+      - git-scm
+
+    builders:
+      - description-setter:
+          description: "POD: $NODE_NAME"
+      - conditional-step:
+          condition-kind: regex-match
+          regex: "k8s-.*"
+          label: '{pod}'
+          steps:
+            - shell:
+                !include-raw-escape: ./eliot-clean.sh
+            - shell:
+                !include-raw-escape: ./eliot-deploy.sh
+      - conditional-step:
+          condition-kind: regex-match
+          regex: "kubeedge-.*"
+          label: '{pod}'
+          steps:
+            - shell:
+                !include-raw-escape: ./eliot-kubeedge-clean.sh
+            - shell:
+                !include-raw-escape: ./eliot-kubeedge-deploy.sh
+
+########################
+# parameter macros
+########################
+- parameter:
+    name: eliot-ci-parameter
+    parameters:
+      - string:
+          name: GS_URL
+          default: '$GS_BASE{gs-pathname}'
+          description: "URL to Google Storage."
+      - string:
+          name: PPA_REPO
+          default: ""
+      - string:
+          name: PPA_CACHE
+          default: "$WORKSPACE/work/repo/"
+      - string:
+          name: LOG_DIRECTORY
+          default: $WORKSPACE/log_output
+          description: ""
+
+
+########################
+# trigger macros
+########################
+
+
+# -----------------
+# virtual-master
+# -----------------
+
+- trigger:
+    name: 'eliot-deploy-kubeedge-virtual-daily-master'
+    triggers:
+      - timed: '0 20 * * *'
+
+- trigger:
+    name: 'eliot-deploy-k8s-virtual-daily-master'
+    triggers:
+      - timed: '0 21 * * *'
+
+- trigger:
+    name: 'eliot-deploy-k8s-centos-virtual-daily-master'
+    triggers:
+      - timed: '0 23 * * *'
diff --git a/jjb/eliot/eliot-clean.sh b/jjb/eliot/eliot-clean.sh
new file mode 100755
index 0000000..2478f21
--- /dev/null
+++ b/jjb/eliot/eliot-clean.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -x
+
+# log info to console
+echo "cc"
+echo "--------------------------------------------------------"
+echo
+
+node_name=$(kubectl get node  |awk '{print $1}' |grep -v NAME) ||true
+for i in $node_name;
+do
+  kubectl drain $i  --delete-local-data --force --ignore-daemonsets
+  kubectl delete node  $i
+done
+
+if [[ "$NODE_NAME" =~ "huawei-eliot-pod1" ]]; then
+    sshpass -f /home/jenkins/pwfile.txt ssh root@10.10.0.99 sudo kubeadm reset -f ||true
+    sshpass -f /home/jenkins/pwfile.txt ssh root@10.10.1.15 sudo kubeadm reset -f ||true
+elif [[ "$NODE_NAME" =~ "huawei-eliot-centos-pod3" ]]; then
+    sshpass -f /home/jenkins/pwfile.txt ssh root@10.10.0.211 sudo kubeadm reset -f ||true
+    sshpass -f /home/jenkins/pwfile.txt ssh root@10.10.0.138 sudo kubeadm reset -f ||true
+
+fi
+
+echo
+echo "--------------------------------------------------------"
+echo "Cleaned"
diff --git a/jjb/eliot/eliot-deploy.sh b/jjb/eliot/eliot-deploy.sh
new file mode 100644
index 0000000..612f32b
--- /dev/null
+++ b/jjb/eliot/eliot-deploy.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -x
+
+echo "CLEANUP Prometheus and Cadvisor started...."
+echo "-------------------------------------------------------"
+cd $WORKSPACE/scripts/ci_management
+./cleanup_prom_cadvisor.sh
+
+sshpass -f /home/jenkins/pwfile.txt ssh root@10.10.0.99 "bash -s" < ./uninstall_cadvisor.sh || true
+
+
+echo "CLEANUP prometheus and Cadvisor ended......"
+echo "-------------------------------------------------------"
+
+# log info to console
+echo "Starting the deployment on baremetal environment using $INSTALLER_TYPE. This could take some time..."
+echo "--------------------------------------------------------"
+echo
+
+rm -rf $WORKSPACE/scripts/nodelist
+
+cp /home/jenkins/nodelist $WORKSPACE/scripts/nodelist
+
+cd $WORKSPACE/scripts/
+./setup.sh
+
+if [ $? -ne 0 ]; then
+    echo "depolyment failed!"
+    deploy_ret=1
+fi
+
+sleep 60
+kubectl get nodes
+
+echo "Logs uploading to nexus repo"
+
+export ARCHIVE_ARTIFACTS="**/*.log"
+export NEXUS_URL=https://nexus.akraino.org
+export SILO=huawei
+export JENKINS_HOSTNAME=http://159.138.47.166:8080
+export BUILD_URL="${JENKINS_HOSTNAME}/job/${JOB_NAME}/${BUILD_NUMBER}"
+export NEXUS_PATH="${SILO}/job/${JOB_NAME}/${BUILD_NUMBER}"
+lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE"
+lftools deploy logs $NEXUS_URL $NEXUS_PATH $BUILD_URL
+
+echo $BUILD_URL
+
+echo "Logs uploaded to $NEXUS_URL/content/sites/logs/$NEXUS_PATH"
+
+echo
+echo "--------------------------------------------------------"
+echo "Done!"
+
+exit $deploy_ret
+
diff --git a/jjb/eliot/eliot-kubeedge-clean.sh b/jjb/eliot/eliot-kubeedge-clean.sh
new file mode 100755
index 0000000..28d5ad2
--- /dev/null
+++ b/jjb/eliot/eliot-kubeedge-clean.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -x
+
+# log info to console
+echo "Cleaning up environment"
+echo "--------------------------------------------------------"
+echo
+
+rm -rf $WORKSPACE/scripts/src/nodelist
+cp /home/jenkins/nodelist_kubeedge $WORKSPACE/scripts/src/nodelist
+
+rm -rf  /root/go/src/github.com/kubeedge/kubeedge
+
+cd $WORKSPACE/ci_management
+./cleanup.sh
+
+node_name=$(kubectl get node  |awk '{print $1}' |grep -v NAME) ||true
+for i in $node_name;
+do
+  kubectl drain $i  --delete-local-data --force --ignore-daemonsets
+  kubectl delete node  $i
+done
+
+sshpass  ssh root@10.10.0.54 sudo kubeadm reset -f ||true
+sshpass  ssh root@10.10.0.45 sudo kubeadm reset -f ||true
+
+echo
+echo "--------------------------------------------------------"
+echo "Cleaned"
+
diff --git a/jjb/eliot/eliot-kubeedge-deploy.sh b/jjb/eliot/eliot-kubeedge-deploy.sh
new file mode 100644
index 0000000..8c2c810
--- /dev/null
+++ b/jjb/eliot/eliot-kubeedge-deploy.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -x
+
+# log info to console
+echo "Starting the deployment on baremetal environment using $INSTALLER_TYPE. This could take some time..."
+echo "--------------------------------------------------------"
+echo
+
+rm -rf $WORKSPACE/scripts/src/config_kubeedge
+
+cp /home/jenkins/nodelist_kubeedge $WORKSPACE/scripts/src/config_kubeedge
+
+cd $WORKSPACE/scripts/src
+source /root/.profile
+sudo ./kubeedge_setup.sh
+
+if [ $? -ne 0 ]; then
+    echo "depolyment failed!"
+    deploy_ret=1
+fi
+
+
+echo "Logs uploading to nexus repo"
+
+export ARCHIVE_ARTIFACTS="**/*.log"
+export NEXUS_URL=https://nexus.akraino.org
+export SILO=huawei
+export JENKINS_HOSTNAME=http://159.138.47.166:8080
+export BUILD_URL="${JENKINS_HOSTNAME}/job/${JOB_NAME}/${BUILD_NUMBER}"
+export NEXUS_PATH="${SILO}/job/${JOB_NAME}/${BUILD_NUMBER}"
+lftools deploy archives -p "$ARCHIVE_ARTIFACTS" "$NEXUS_URL" "$NEXUS_PATH" "$WORKSPACE"
+lftools deploy logs $NEXUS_URL $NEXUS_PATH $BUILD_URL
+
+echo $BUILD_URL
+
+echo "Logs uploaded to $NEXUS_URL/content/sites/logs/$NEXUS_PATH"
+
+
+
+echo
+echo "--------------------------------------------------------"
+echo "Done!"
+
+exit $deploy_ret
+
diff --git a/jjb/global/base-macros.yaml b/jjb/global/base-macros.yaml
new file mode 100644
index 0000000..92d315c
--- /dev/null
+++ b/jjb/global/base-macros.yaml
@@ -0,0 +1,285 @@
+---
+# Base macros
+#
+# NOTE: make sure macros are listed in execution ordered.
+#
+# 1. parameters/properties
+# 2. scm
+# 3. triggers
+# 4. wrappers
+# 5. prebuilders (maven only, configured like Builders)
+# 6. builders (maven, freestyle, matrix, etc..)
+# 7. postbuilders (maven only, configured like Builders)
+# 8. publishers/reporters/notifications
+
+- parameter:
+    name: project-parameter
+    parameters:
+      - string:
+          name: PROJECT
+          default: '{project}'
+          description: "JJB configured PROJECT parameter"
+      - string:
+          name: GS_BASE
+          default: artifacts.akraino.org/$PROJECT
+          description: "URL to Google Storage."
+      - string:
+          name: BRANCH
+          default: '{branch}'
+          description: "JJB configured BRANCH parameter"
+      - string:
+          name: GERRIT_BRANCH
+          default: '{branch}'
+          description: "JJB configured GERRIT_BRANCH parameter (deprecated)"
+      - string:
+          name: GERRIT_REFSPEC
+          default: 'refs/heads/{branch}'
+          description: "Default refspec needed for manually triggering."
+
+
+- property:
+    name: logrotate-default
+    properties:
+      - build-discarder:
+          days-to-keep: 60
+          num-to-keep: 200
+          artifact-days-to-keep: 60
+          artifact-num-to-keep: 200
+
+- scm:
+    name: git-scm
+    scm:
+      - git: &git-scm-defaults
+          credentials-id: 'jenkins-ssh'
+          url: '$GIT_BASE'
+          branches:
+            - 'origin/$BRANCH'
+          timeout: 15
+          per-build-tag: false
+          skip-tag: true
+          shallow-clone: false
+          use-author: false
+          ignore-notify: false
+          wipe-workspace: true
+          prune: false
+
+- scm:
+    name: git-scm-gerrit
+    scm:
+      - git:
+          choosing-strategy: 'gerrit'
+          refspec: '$GERRIT_REFSPEC'
+          <<: *git-scm-defaults
+
+- scm:
+    name: git-scm-gerrit-with-submodules
+    scm:
+      - git:
+          choosing-strategy: 'gerrit'
+          refspec: '$GERRIT_REFSPEC'
+          submodule:
+            recursive: true
+            timeout: 20
+          <<: *git-scm-defaults
+
+- trigger:
+    name: 'daily-trigger-disabled'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'weekly-trigger-disabled'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: gerrit-trigger-patchset-created
+    triggers:
+      - gerrit:
+          server-name: 'gerrit.akraino.org'
+          trigger-on:
+            - patchset-created-event:
+                exclude-drafts: 'false'
+                exclude-trivial-rebase: 'false'
+                exclude-no-code-change: 'false'
+            - draft-published-event
+            - comment-added-contains-event:
+                comment-contains-value: 'recheck'
+            - comment-added-contains-event:
+                comment-contains-value: 'reverify'
+          projects:
+            - project-compare-type: 'ANT'
+              project-pattern: '{project}'
+              branches:
+                - branch-compare-type: 'ANT'
+                  branch-pattern: '**/{branch}'
+              file-paths:
+                - compare-type: 'ANT'
+                  pattern: '{files}'
+          skip-vote:
+            successful: false
+            failed: false
+            unstable: false
+            notbuilt: false
+
+- wrapper:
+    name: ssh-agent-wrapper
+    wrappers:
+      - ssh-agent-credentials:
+          users:
+            - 'jenkins-ssh'
+
+- wrapper:
+    name: build-timeout
+    wrappers:
+      - timeout:
+          timeout: '{timeout}'
+          timeout-var: 'BUILD_TIMEOUT'
+          fail: true
+
+- wrapper:
+    name: fix-workspace-permissions
+    wrappers:
+      - pre-scm-buildstep:
+          - shell: |
+             #!/bin/bash
+             sudo chown -R $USER:$USER $WORKSPACE || exit 1
+
+- builder:
+    name: upload-under-review-docs-to-akraino-artifacts
+    builders:
+      - shell: |
+          #!/bin/bash
+          set -o errexit
+          set -o pipefail
+          set -o xtrace
+          export PATH=$PATH:/usr/local/bin/
+
+          [[ $GERRIT_CHANGE_NUMBER =~ .+ ]]
+          [[ -d docs/_build/ ]] || exit 0
+
+          echo
+          echo "###########################"
+          echo "UPLOADING DOCS UNDER REVIEW"
+          echo "###########################"
+          echo
+
+          gs_base="artifacts.akraino.org/$PROJECT/review"
+          gs_path="$gs_base/$GERRIT_CHANGE_NUMBER"
+          local_path="upload/$GERRIT_CHANGE_NUMBER"
+
+          mkdir -p upload
+          mv docs/_build/html/ "$local_path"
+          gsutil -m cp -r "$local_path" "gs://$gs_base"
+
+          gsutil -m setmeta \
+              -h "Content-Type:text/html" \
+              -h "Cache-Control:private, max-age=0, no-transform" \
+              "gs://$gs_path"/**.html > /dev/null 2>&1
+
+          echo "Document link(s):" >> gerrit_comment.txt
+          find "$local_path" | grep -e 'index.html$' -e 'pdf$' | \
+              sed -e "s|^$local_path|    http://$gs_path|" >> gerrit_comment.txt
+
+# To take advantage of this macro, have your build write
+# out the file 'gerrit_comment.txt' with information to post
+# back to gerrit and include this macro in the list of builders.
+- builder:
+    name: report-build-result-to-gerrit
+    builders:
+      - shell: |
+          #!/bin/bash
+          set -o errexit
+          set -o pipefail
+          set -o xtrace
+          export PATH=$PATH:/usr/local/bin/
+          if [[ -e gerrit_comment.txt ]] ; then
+              echo
+              echo "posting review comment to gerrit..."
+              echo
+              cat gerrit_comment.txt
+              echo
+              ssh -p 29418 gerrit.akraino.org \
+                  "gerrit review -p $GERRIT_PROJECT \
+                   -m '$(cat gerrit_comment.txt)' \
+                   $GERRIT_PATCHSET_REVISION \
+                   --notify NONE"
+          fi
+
+- builder:
+    name: remove-old-docs-from-akraino-artifacts
+    builders:
+      - shell: |
+          #!/bin/bash
+          set -o errexit
+          set -o pipefail
+          set -o xtrace
+          export PATH=$PATH:/usr/local/bin/
+
+          [[ $GERRIT_CHANGE_NUMBER =~ .+ ]]
+
+          gs_path="artifacts.akraino.org/$PROJECT/review/$GERRIT_CHANGE_NUMBER"
+
+          if gsutil ls "gs://$gs_path" > /dev/null 2>&1 ; then
+              echo
+              echo "Deleting Out-of-dated Documents..."
+              gsutil -m rm -r "gs://$gs_path"
+          fi
+          gs_path="artifacts.akraino.org/review/$GERRIT_CHANGE_NUMBER"
+
+          if gsutil ls "gs://$gs_path" > /dev/null 2>&1 ; then
+              echo
+              echo "Deleting Out-of-dated Documents..."
+              gsutil -m rm -r "gs://$gs_path"
+          fi
+
+- builder:
+    name: upload-review-docs
+    builders:
+      - upload-under-review-docs-to-akraino-artifacts
+      - report-build-result-to-gerrit
+
+
+- builder:
+    name: clean-workspace
+    builders:
+      - shell: |
+          #!/bin/bash
+          set -o errexit
+          set -o nounset
+          set -o pipefail
+          sudo /bin/rm -rf "$WORKSPACE"
+
+- builder:
+    name: clean-workspace-log
+    builders:
+      - shell: |
+          find $WORKSPACE -type f -name '*.log' | xargs rm -f
+
+- publisher:
+    name: archive-artifacts
+    publishers:
+      - archive:
+          artifacts: '{artifacts}'
+          allow-empty: true
+          fingerprint: true
+          latest-only: true
+
+- publisher:
+    name: publish-coverage
+    publishers:
+      - cobertura:
+          report-file: "coverage.xml"
+          only-stable: "true"
+          health-auto-update: "false"
+          stability-auto-update: "false"
+          zoom-coverage-chart: "true"
+          targets:
+            - files:
+                healthy: 10
+                unhealthy: 20
+                failing: 30
+            - method:
+                healthy: 50
+                unhealthy: 40
+                failing: 30
diff --git a/jjb/global/defaults.yaml b/jjb/global/defaults.yaml
new file mode 100644
index 0000000..3e9f8e9
--- /dev/null
+++ b/jjb/global/defaults.yaml
@@ -0,0 +1,10 @@
+---
+# jjb defaults
+- defaults:
+    name: global
+
+    gerrit-server-name: Primary
+    git-url: 'ssh://akraino-jobbuilder@gerrit.akraino.org:29418'
+    jenkins-ssh-credential: jenkins-ssh
+    lftools-version: '<1.0.0'
+    submodule-timeout: 10
diff --git a/jjb/global/project-params.yaml b/jjb/global/project-params.yaml
new file mode 100644
index 0000000..a23a57c
--- /dev/null
+++ b/jjb/global/project-params.yaml
@@ -0,0 +1,16 @@
+---
+- parameter:
+    name: 'eliot-defaults'
+    parameters:
+      - string:
+          name: NODE_IP
+          default: ""
+          description: 'IP of the node'
+      - string:
+          name: USERNAME
+          default: ""
+          description: 'username of the node'
+      - string:
+          name: PASSWORD
+          default: ""
+          description: 'password of the node'
diff --git a/jjb/global/slave-params.yaml b/jjb/global/slave-params.yaml
new file mode 100644
index 0000000..b56fd46
--- /dev/null
+++ b/jjb/global/slave-params.yaml
@@ -0,0 +1,39 @@
+---
+- parameter:
+    name: 'eliot-kubeedge-virtual-defaults'
+    parameters:
+      - label:
+          name: SLAVE_LABEL
+          default: 'eliot-kubeedge-virtual'
+          all-nodes: false
+          node-eligibility: 'ignore-offline'
+      - string:
+          name: GIT_BASE
+          default: https://gerrit.akraino.org/r/$PROJECT
+          description: 'Git URL to use on this Jenkins Slave'
+
+- parameter:
+    name: 'eliot-k8s-virtual-defaults'
+    parameters:
+      - label:
+          name: SLAVE_LABEL
+          default: 'eliot-k8s-virtual'
+          all-nodes: false
+          node-eligibility: 'ignore-offline'
+      - string:
+          name: GIT_BASE
+          default: https://gerrit.akraino.org/r/$PROJECT
+          description: 'Git URL to use on this Jenkins Slave'
+
+- parameter:
+    name: 'eliot-k8s-centos-virtual-defaults'
+    parameters:
+      - label:
+          name: SLAVE_LABEL
+          default: 'eliot-k8s-centos-virtual'
+          all-nodes: false
+          node-eligibility: 'ignore-offline'
+      - string:
+          name: GIT_BASE
+          default: https://gerrit.akraino.org/r/$PROJECT
+          description: 'Git URL to use on this Jenkins Slave'