ci/ci.sh

   1 #!/usr/bin/env bash
   2 set -eux -o pipefail
   3
   4 SCRIPT_DIR="$(readlink -f $(dirname ${BASH_SOURCE[0]}))"
   5
   6 WITH_VAGRANT="${WITH_VAGRANT:-yes}"
   7 JENKINS_HOSTNAME="${JENKINS_HOSTNAME:-localhost}"
   8 JENKINS_ADMIN_USERNAME="${JENKINS_ADMIN_USERNAME:-admin}"
   9 JENKINS_ADMIN_PASSWORD="${JENKINS_ADMIN_PASSWORD:-admin}"
  10 JENKINS_LFTOOLS_USERNAME="${JENKINS_LFTOOLS_USERNAME:-icn.jenkins}"
  11 JENKINS_SSH_USERNAME="${JENKINS_SSH_USERNAME:-icn.jenkins}"
  12 CLUSTER_MASTER_IP="${CLUSTER_MASTER_IP:-localhost}"
  13 CLUSTER_SSH_USER="${CLUSTER_SSH_USER:-root}"
  14
  15 BUILD_DIR=${SCRIPT_DIR/icn/icn/build}
  16 mkdir -p ${BUILD_DIR}
  17
  18 ICN_DIR="${SCRIPT_DIR}/.."
  19 # The ci-management repo must be a sibling of the icn repo
  20 CI_MANAGEMENT_DIR="${ICN_DIR}/../ci-management"
  21 JJB_PATH="${CI_MANAGEMENT_DIR}/jjb:${ICN_DIR}/ci/jjb"
  22
  23 # Workaround for KuD installer which messes with /etc/environment
  24 function cleanup_after_kud {
  25     sed -i -e '/ANSIBLE_CONFIG/d' /etc/environment
  26 }
  27
  28 function install_jenkins {
  29     # Prerequisites
  30     export DEBIAN_FRONTEND=noninteractive
  31     apt-get update
  32     apt-get install -y ca-certificates python3-pip
  33     pip3 install ansible jenkins-job-builder lftools
  34
  35     # Jenkins
  36     ansible-galaxy install -r ${SCRIPT_DIR}/galaxy-requirements.yaml --roles-path /etc/ansible/roles
  37     ansible-playbook ${SCRIPT_DIR}/site_jenkins.yaml --extra-vars "@${SCRIPT_DIR}/vars.yaml" -v
  38
  39     # The Bluval job requires docker access
  40     usermod -aG docker jenkins
  41
  42     # Restart Jenkins to take into account any group changes above
  43     systemctl restart jenkins
  44
  45     # Jenkins jobs
  46     mkdir -p ${HOME}/.config/jenkins_jobs
  47     cp ${SCRIPT_DIR}/jenkins_jobs.ini ${HOME}/.config/jenkins_jobs/jenkins_jobs.ini
  48     git clone --recursive https://gerrit.akraino.org/r/ci-management "${CI_MANAGEMENT_DIR}"
  49
  50     # TODO Figure out how to automate this, it doesn't appear to be exposed with jenkins-cli.jar
  51     cat <<EOF
  52 Git plugin 4.4 removes the second fetch operation in most cases. This
  53 prevents the CI jobs from checking out the correct version.
  54
  55 To enable the second fetch, check the following in the Jenkins web UI:
  56   Manage Jenkins -> Configure System -> [X] Preserve second fetch during checkout
  57 EOF
  58 }
  59
  60 function install_credentials {
  61     if [[ ! -f ${JENKINS_SSH_PRIVATE_KEY} ]]; then
  62         echo "JENKINS_SSH_PRIVATE_KEY must be set to the path of the private key of ${JENKINS_SSH_USERNAME}"
  63         exit 1
  64     fi
  65
  66     wget http://${JENKINS_HOSTNAME}:8080/jnlpJars/jenkins-cli.jar -O ${BUILD_DIR}/jenkins-cli.jar
  67     cat <<EOF >${BUILD_DIR}/jenkins-ssh-credential.xml
  68 <com.cloudbees.jenkins.plugins.sshcredentials.impl.BasicSSHUserPrivateKey plugin="ssh-credentials@1.13">
  69   <scope>GLOBAL</scope>
  70   <id>jenkins-ssh</id>
  71   <description></description>
  72   <username>${JENKINS_SSH_USERNAME}</username>
  73   <privateKeySource class="com.cloudbees.jenkins.plugins.sshcredentials.impl.BasicSSHUserPrivateKey\$DirectEntryPrivateKeySource">
  74     <privateKey>$(cat ${JENKINS_SSH_PRIVATE_KEY})</privateKey>
  75   </privateKeySource>
  76 </com.cloudbees.jenkins.plugins.sshcredentials.impl.BasicSSHUserPrivateKey>
  77 EOF
  78     java -jar ${BUILD_DIR}/jenkins-cli.jar -auth ${JENKINS_ADMIN_USERNAME}:${JENKINS_ADMIN_PASSWORD} -s http://${JENKINS_HOSTNAME}:8080/ create-credentials-by-xml system::system::jenkins _  <${BUILD_DIR}/jenkins-ssh-credential.xml
  79 }
  80
  81 function install_lftools_credentials {
  82     cat <<EOF >/var/lib/jenkins/.netrc
  83 machine nexus.akraino.org login ${JENKINS_LFTOOLS_USERNAME} password ${JENKINS_LFTOOLS_PASSWORD}
  84 EOF
  85     chown jenkins:jenkins /var/lib/jenkins/.netrc
  86     chmod 0600 /var/lib/jenkins/.netrc
  87 }
  88
  89 function update_jobs {
  90     # TODO Find a better way to do this without touching files under
  91     # source control
  92     sed -i -e "s!git-url: .*!git-url: 'ssh://${JENKINS_SSH_USERNAME}@gerrit.akraino.org:29418'!" ${SCRIPT_DIR}/jjb/defaults.yaml
  93     sed -i -e "s!bluval-cluster-master-ip: .*!bluval-cluster-master-ip: ${CLUSTER_MASTER_IP}!" ${SCRIPT_DIR}/jjb/defaults.yaml
  94     sed -i -e "s!bluval-cluster-ssh-user: .*!bluval-cluster-ssh-user: ${CLUSTER_SSH_USER}!" ${SCRIPT_DIR}/jjb/defaults.yaml
  95
  96     # This will create all 344 jobs:
  97     # jenkins-jobs update ${JJB_PATH}:${ICN_DIR}/ci/jjb/project.yaml
  98
  99     # These are the ICN jobs we are interested in
 100     if [[ ${WITH_VAGRANT} == "yes" ]]; then
 101         jenkins-jobs update ${JJB_PATH} icn-master-vagrant-verify-verifier
 102     fi
 103     jenkins-jobs update ${JJB_PATH} icn-master-bm-verify-bm_verifier
 104     jenkins-jobs update ${JJB_PATH} icn-bluval-daily-master
 105     #jenkins-jobs update ${JJB_PATH} icn-master-verify
 106
 107     # These are additional ICN jobs:
 108     # if [[ ${WITH_VAGRANT} == "yes" ]]; then
 109     #     jenkins-jobs update ${JJB_PATH} icn-master-vagrant-verify-verify_nestedk8s
 110     # fi
 111 }
 112
 113 function install_jenkins_id {
 114     # Create a new key if one does not exist
 115     ssh-keygen -q -t rsa -N '' -f /var/lib/jenkins/jenkins-rsa -C jenkins@$(hostname) <<<n >/dev/null 2>&1 || true
 116     ssh-copy-id -i /var/lib/jenkins/jenkins-rsa -f ${CLUSTER_SSH_USER}@${CLUSTER_MASTER_IP}
 117     chown jenkins:jenkins /var/lib/jenkins/jenkins-rsa*
 118     chmod 600 /var/lib/jenkins/jenkins-rsa*
 119 }
 120
 121 case $1 in
 122     "cleanup-after-kud") cleanup_after_kud ;;
 123     "install-credentials") install_credentials ;;
 124     "install-jenkins") install_jenkins ;;
 125     "install-jenkins-id") install_jenkins_id ;;
 126     "install-lftools-credentials") install_lftools_credentials ;;
 127     "update-jobs") update_jobs ;;
 128     *) cat <<EOF
 129 Usage: $(basename $0) COMMAND
 130
 131 Commands:
 132   cleanup-after-kud            - Cleanup after KuD install
 133   install-credentials          - Install credentials into Jenkins
 134   install-jenkins              - Install Jenkins
 135   install-jenkins-id           - Install Jenkins ID into test cluster
 136   install-lftools-credentials  - Install lftools credentials
 137   update-jobs                  - Install or update ICN jobs into Jenkins
 138
 139 Environment variables used by the commands:
 140   WITH_VAGRANT=[yes|no]    - Install components needed to run the VM
 141                              verifier job
 142   JENKINS_HOSTNAME         - jenkins_hostname in vars.yaml
 143   JENKINS_ADMIN_USERNAME   - jenkins_admin_username in vars.yaml
 144   JENKINS_ADMIN_PASSWORD   - jenkins_admin_password in vars.yaml
 145   JENKINS_LFTOOLS_USERNAME - The .netrc login
 146   JENKINS_LFTOOLS_PASSWORD - The .netrc password
 147   JENKINS_SSH_USERNAME     - The gerrit account username
 148   JENKINS_SSH_PRIVATE_KEY  - The gerrit account private key file
 149   CLUSTER_MASTER_IP        - The cluster under test
 150   CLUSTER_SSH_USER         - The cluster account username
 151 EOF
 152        ;;
 153 esac