Add new jobs for blueprint-pae repo 10/1210/4
authorYolanda Robla <yroblamo@redhat.com>
Tue, 16 Jul 2019 15:01:26 +0000 (17:01 +0200)
committerYolanda Robla <yroblamo@redhat.com>
Wed, 17 Jul 2019 08:02:03 +0000 (10:02 +0200)
The content from kni/templates has been moved to
kni/blueprint-pae. So we are creating new jobs for it
and disable the older ones. We are also updating
the shell for those jobs, to match with the current
workflow.

Signed-off-by: Yolanda Robla <yroblamo@redhat.com>
Change-Id: I597bb960b40d2f33ee78cf4129c7aed7479f4d54

jjb/akraino-templates/akraino-jjb-kni.yaml
jjb/kni/kni_installer.yaml
jjb/shell/kni_deploy_aws.sh
jjb/shell/kni_deploy_libvirt.sh

index 1194fe9..e207263 100644 (file)
       - lf-infra-publish
 
 - job-template:
-    id: kni-installer-merge-deploy-aws
-    name: '{project-name}-merge-deploy-aws'
+    id: kni-blueprint-pae-verify-deploy-aws
+    name: '{project-name}-verify-deploy-aws'
 
     ######################
     # Default parameters #
     ######################
 
-    archive-artifacts: >
-      **/*.log
     build-days-to-keep: 30
     build-timeout: 60
-    build-node: centos7-builder-8c-8g
+    build-node: kni-libvirt
     submodule-recursive: true
 
     #####################
       - lf-infra-properties:
           build-days-to-keep: '{build-days-to-keep}'
 
+    archive-artifacts: >
+      **/*.log
+
     parameters:
       - lf-infra-parameters:
           project: '{project}'
 
     gerrit_trigger_file_paths:
       - compare-type: REG_EXP
-        pattern: '.*'
+        pattern: 'profiles/production.aws/*'
+      - compare-type: REG_EXP
+        pattern: 'sites/testing.aws.edge-sites.net/*'
 
-    gerrit_merge_triggers:
-      - change-merged-event
+    gerrit_verify_triggers:
+      - patchset-created-event:
+          exclude-drafts: true
+          exclude-trivial-rebase: false
+          exclude-no-code-change: false
+      - draft-published-event
       - comment-added-contains-event:
-          comment-contains-value: remerge$
+          comment-contains-value: '^Patch Set[ ]+[0-9]+:([ ]+|[\n]+)(recheck|reverify)$'
 
     scm:
       - lf-infra-gerrit-scm:
     triggers:
       - gerrit:
           server-name: '{gerrit-server-name}'
-          trigger-on: '{obj:gerrit_merge_triggers}'
+          trigger-on: '{obj:gerrit_verify_triggers}'
           projects:
             - project-compare-type: ANT
               project-pattern: '{project}'
       - lf-infra-pre-build
       - config-file-provider:
           files:
-            - file-id: kni-aws-access-key-id
-              target: akraino-secrets/aws-access-key-id
-            - file-id: kni-aws-secret-access-key
-              target: akraino-secrets/aws-secret-access-key
             - file-id: kni-coreos-pull-secret
               target: akraino-secrets/coreos-pull-secret
-            - file-id: kni-ssh-pub-key
-              target: akraino-secrets/ssh-pub-key
       - shell: !include-raw-escape:
           - ../shell/install_go.sh
       - shell: !include-raw-escape:
       - lf-infra-publish
 
 - job-template:
-    id: kni-installer-merge-deploy-libvirt
-    name: '{project-name}-merge-deploy-libvirt'
+    id: kni-blueprint-pae-verify-deploy-libvirt
+    name: '{project-name}-verify-deploy-libvirt'
 
     ######################
     # Default parameters #
     ######################
 
-    archive-artifacts: >
-      **/*.log
     build-days-to-keep: 30
     build-timeout: 60
     build-node: kni-libvirt
           default: '{archive-artifacts}'
           description: Artifacts to archive to the logs server.
 
-    wrappers:
-      - lf-infra-wrappers:
-          build-timeout: '{build-timeout}'
-          jenkins-ssh-credential: '{jenkins-ssh-credential}'
-      - credentials-binding:
-          - ssh-user-private-key:
-              credential-id: kni-secrets
-              key-file-variable: KNI_KEY_PATH
-
-    gerrit_trigger_file_paths:
-      - compare-type: REG_EXP
-        pattern: '.*'
-
-    gerrit_merge_triggers:
-      - change-merged-event
-      - comment-added-contains-event:
-          comment-contains-value: remerge$
-
-    scm:
-      - lf-infra-gerrit-scm:
-          jenkins-ssh-credential: '{jenkins-ssh-credential}'
-          git-url: '{git-url}/{project}.git'
-          refspec: '$GERRIT_REFSPEC'
-          branch: '$GERRIT_BRANCH'
-          submodule-recursive: '{submodule-recursive}'
-          submodule-timeout: '{submodule-timeout}'
-          submodule-disable: false
-          choosing-strategy: default
-
-    triggers:
-      - gerrit:
-          server-name: '{gerrit-server-name}'
-          trigger-on: '{obj:gerrit_merge_triggers}'
-          projects:
-            - project-compare-type: ANT
-              project-pattern: '{project}'
-              branches:
-                - branch-compare-type: ANT
-                  branch-pattern: '**/{branch}'
-              file-paths: '{obj:gerrit_trigger_file_paths}'
-
-    builders:
-      - lf-infra-pre-build
-      - config-file-provider:
-          files:
-            - file-id: kni-coreos-pull-secret
-              target: akraino-secrets/coreos-pull-secret
-            - file-id: kni-ssh-pub-key
-              target: akraino-secrets/ssh-pub-key
-      - shell: !include-raw-escape:
-          - ../shell/install_go.sh
-      - shell: !include-raw-escape:
-          - ../shell/kni_deploy_libvirt.sh
-
-    publishers:
-      - lf-infra-publish
-
-- job-template:
-    id: kni-templates-verify-deploy-aws
-    name: '{project-name}-verify-deploy-aws'
-
-    ######################
-    # Default parameters #
-    ######################
-
-    build-days-to-keep: 30
-    build-timeout: 60
-    build-node: centos7-builder-8c-8g
-    submodule-recursive: true
-
-    #####################
-    # Job Configuration #
-    #####################
-
-    project-type: freestyle
-    node: '{build-node}'
-
-    properties:
-      - lf-infra-properties:
-          build-days-to-keep: '{build-days-to-keep}'
-
-    parameters:
-      - lf-infra-parameters:
-          project: '{project}'
-          branch: '{branch}'
-          stream: '{stream}'
-          lftools-version: '{lftools-version}'
+    archive-artifacts: >
+      **/*.log
 
     wrappers:
       - lf-infra-wrappers:
 
     gerrit_trigger_file_paths:
       - compare-type: REG_EXP
-        pattern: 'aws/*'
-
-    scm:
-      - lf-infra-gerrit-scm:
-          jenkins-ssh-credential: '{jenkins-ssh-credential}'
-          git-url: '{git-url}/kni/installer.git'
-          refspec: '$GERRIT_REFSPEC'
-          branch: '$GERRIT_BRANCH'
-          submodule-recursive: '{submodule-recursive}'
-          submodule-timeout: '{submodule-timeout}'
-          submodule-disable: false
-          choosing-strategy: default
+        pattern: 'profiles/testing.libvirt/*'
+      - compare-type: REG_EXP
+        pattern: 'sites/testing.virt.edge-sites.net/*'
 
     gerrit_verify_triggers:
       - patchset-created-event:
                   branch-pattern: '**/{branch}'
               file-paths: '{obj:gerrit_trigger_file_paths}'
 
-    builders:
-      - lf-infra-pre-build
-      - config-file-provider:
-          files:
-            - file-id: kni-aws-access-key-id
-              target: akraino-secrets/aws-access-key-id
-            - file-id: kni-aws-secret-access-key
-              target: akraino-secrets/aws-secret-access-key
-            - file-id: kni-coreos-pull-secret
-              target: akraino-secrets/coreos-pull-secret
-            - file-id: kni-ssh-pub-key
-              target: akraino-secrets/ssh-pub-key
-      - shell: !include-raw-escape:
-          - ../shell/install_go.sh
-      - shell: !include-raw-escape:
-          - ../shell/kni_deploy_aws.sh
-
-    publishers:
-      - lf-infra-publish
-
-- job-template:
-    id: kni-templates-verify-deploy-libvirt
-    name: '{project-name}-verify-deploy-libvirt'
-
-    ######################
-    # Default parameters #
-    ######################
-
-    build-days-to-keep: 30
-    build-timeout: 60
-    build-node: kni-libvirt
-    submodule-recursive: true
-
-    #####################
-    # Job Configuration #
-    #####################
-
-    project-type: freestyle
-    node: '{build-node}'
-
-    properties:
-      - lf-infra-properties:
-          build-days-to-keep: '{build-days-to-keep}'
-
-    parameters:
-      - lf-infra-parameters:
-          project: '{project}'
-          branch: '{branch}'
-          stream: '{stream}'
-          lftools-version: '{lftools-version}'
-
-    wrappers:
-      - lf-infra-wrappers:
-          build-timeout: '{build-timeout}'
-          jenkins-ssh-credential: '{jenkins-ssh-credential}'
-      - credentials-binding:
-          - ssh-user-private-key:
-              credential-id: kni-secrets
-              key-file-variable: KNI_KEY_PATH
-
-    gerrit_trigger_file_paths:
-      - compare-type: REG_EXP
-        pattern: 'aws/*'
-
     scm:
       - lf-infra-gerrit-scm:
           jenkins-ssh-credential: '{jenkins-ssh-credential}'
-          git-url: '{git-url}/kni/installer.git'
+          git-url: '{git-url}/{project}.git'
           refspec: '$GERRIT_REFSPEC'
           branch: '$GERRIT_BRANCH'
           submodule-recursive: '{submodule-recursive}'
           submodule-disable: false
           choosing-strategy: default
 
-    gerrit_verify_triggers:
-      - patchset-created-event:
-          exclude-drafts: true
-          exclude-trivial-rebase: false
-          exclude-no-code-change: false
-      - draft-published-event
-      - comment-added-contains-event:
-          comment-contains-value: '^Patch Set[ ]+[0-9]+:([ ]+|[\n]+)(recheck|reverify)$'
-
-    triggers:
-      - gerrit:
-          server-name: '{gerrit-server-name}'
-          trigger-on: '{obj:gerrit_verify_triggers}'
-          projects:
-            - project-compare-type: ANT
-              project-pattern: '{project}'
-              branches:
-                - branch-compare-type: ANT
-                  branch-pattern: '**/{branch}'
-              file-paths: '{obj:gerrit_trigger_file_paths}'
-
     builders:
       - lf-infra-pre-build
       - config-file-provider:
index 262236f..62e7993 100644 (file)
@@ -9,14 +9,13 @@
       - kni-installer-verify-installer
       - kni-installer-verify-binary
       - kni-installer-publish-openshift-install-libvirt-binary
-      - kni-installer-merge-deploy-aws
-      - kni-installer-merge-deploy-libvirt
+
 - project:
-    name: kni-templates
-    project: 'kni/templates'
-    project-name: kni-templates
+    name: kni-blueprint-pae
+    project: 'kni/blueprint-pae'
+    project-name: kni-blueprint-pae
     stream: master
     branch: master
     jobs:
-      - kni-templates-verify-deploy-aws
-      - kni-templates-verify-deploy-libvirt
+      - kni-blueprint-pae-verify-deploy-aws
+      - kni-blueprint-pae-verify-deploy-libvirt
index ce8eec5..d9c468d 100755 (executable)
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-export PATH=$PATH:/usr/local/go/bin
-KNI_PATH='go/src/gerrit.akraino.org/kni/'
-
 set -e -u -x -o pipefail
 
+export PATH=$PATH:/usr/local/go/bin:/usr/local/bin
+KNI_PATH='src/gerrit.akraino.org/kni/'
+SITE_NAME='edge-sites-testing.devcluster.openshift.com'
+
 echo '---> Starting kni installer generation'
+export GOPATH=${WORKSPACE}/
 
-mkdir -p $HOME/${KNI_PATH}/installer
-export GOPATH=$HOME/go
+# move the blueprint to an inner directory
+mkdir ${WORKSPACE}/blueprint-pae
+mv base profiles sites tools ${WORKSPACE}/blueprint-pae/
 
-# move clone to gopath
-cp -R ${WORKSPACE}/* $HOME/${KNI_PATH}/installer/
+# clone installer in the right directory
+sudo rm -rf ${WORKSPACE}/${KNI_PATH}
+mkdir -p ${WORKSPACE}/${KNI_PATH}
+pushd ${WORKSPACE}/${KNI_PATH}/
+git clone https://gerrit.akraino.org/r/kni/installer
+pushd installer
 
 # first build kni installer
-pushd $HOME/${KNI_PATH}/installer
 make build 2>&1 | tee ${WORKSPACE}/build.log
 
-# then start aws deploy
-export CREDENTIALS=file://$(pwd)/akraino-secrets
-export BASE_REPO="git::https://gerrit.akraino.org/r/kni/templates"
-export BASE_PATH="aws/3-node"
-export SITE_REPO="git::https://gerrit.akraino.org/r/kni/templates"
-export SETTINGS_PATH="aws/sample_settings.yaml"
-make deploy 2>&1 | tee ${WORKSPACE}/aws_deploy.log
+# add the right credentials to kni
+mkdir $HOME/.kni || true
+cp $WORKSPACE/akraino-secrets/coreos-pull-secret $HOME/.kni/pull-secret.json || true
+
+# start the workflow
+./knictl fetch_requirements file://${WORKSPACE}/blueprint-pae//sites/${SITE_NAME} 2>&1 | tee ${WORKSPACE}/aws_requirements.log
+./knictl prepare_manifests ${SITE_NAME} 2>&1 | tee ${WORKSPACE}/aws_manifests.log
+
+# now run the cluster
+source $HOME/.kni/${SITE_NAME}/profile.env
+$HOME/.kni/${SITE_NAME}/requirements/openshift-install create cluster --dir=/$HOME/.kni/${SITE_NAME}/final_manifests 2>&1 | tee ${WORKSPACE}/aws_deploy.log
 STATUS=$?
 
 # output tfstate
 echo "metadata.json for removing cluster"
-cat $(pwd)/build/metadata.json
+cat $HOME/.kni/${SITE_NAME}/final_manifests/metadata.json
 
 if [ $STATUS -ne 0 ]; then
     echo "Error deploying in AWS"
     exit 1
 fi
 
-echo "Cluster successfully deployed! Starting cleanup"
-make clean
+echo "Cluster successfully deployed! Start applying workloads"
+./knictl apply_workloads ${SITE_NAME} 2>&1 | tee ${WORKSPACE}/aws_workloads.go
+STATUS=$?
+
+if [ $STATUS -ne 0 ]; then
+    echo "Error applying workloads to AWS"
+    exit 1
+fi
+
+echo "Workloads successfully applied! Starting cleanup"
+$HOME/.kni/${SITE_NAME}/requirements/openshift-install destroy cluster --dir=/$HOME/.kni/${SITE_NAME}/final_manifests 2>&1 | tee ${WORKSPACE}/aws_destroy.log
 STATUS=$?
 
 popd
index 527bc71..f5d1932 100755 (executable)
 set -e -u -x -o pipefail
 
 export PATH=$PATH:/usr/local/go/bin:/usr/local/bin
-KNI_PATH='src/gerrit.akraino.org/kni/installer'
+KNI_PATH='src/gerrit.akraino.org/kni/'
+SITE_NAME='testing.virt.edge-sites.net'
 
 echo '---> Starting kni installer generation'
+export GOPATH=${WORKSPACE}/
 
-# move to right directory in GOPATH
+function approve_certs() {
+    # sleep for the first 10 min
+    sleep 600
+
+    # temporary fix, autoapprove certificates on background
+    while /bin/true; do
+        export KUBECONFIG=$HOME/.kni/${SITE_NAME}/final_manifests/auth/kubeconfig
+        oc get csr | grep worker | grep Pending | awk '{print $1}' | xargs -n 1 oc adm certificate approve || true
+        sleep 60
+    done
+}
+
+# move the blueprint to an inner directory
+mkdir ${WORKSPACE}/blueprint-pae
+mv base profiles sites tools ${WORKSPACE}/blueprint-pae/
+
+# clone installer in the right directory
+sudo rm -rf ${WORKSPACE}/${KNI_PATH}
 mkdir -p ${WORKSPACE}/${KNI_PATH}
-export GOPATH=${WORKSPACE}
-mv cmd pkg vendor ${WORKSPACE}/${KNI_PATH}/
+pushd ${WORKSPACE}/${KNI_PATH}/
+git clone https://gerrit.akraino.org/r/kni/installer
+pushd installer
+
+# first build kni installer
+make build 2>&1 | tee ${WORKSPACE}/build.log
 
 # do a host preparation and cleanup
-bash utils/prep_host.sh
+bash utils/prep_host.sh virt.edge-sites.net
 wget https://raw.githubusercontent.com/openshift/installer/master/scripts/maintenance/virsh-cleanup.sh
 chmod a+x ./virsh-cleanup.sh
-sudo bash -c "yes Y | ./virsh-cleanup.sh"
+sudo -E bash -c "yes Y | ./virsh-cleanup.sh"
 
-# first build kni installer
-make build 2>&1 | tee ${WORKSPACE}/build.log
+# add the right credentials to kni
+mkdir $HOME/.kni || true
+cp $WORKSPACE/akraino-secrets/coreos-pull-secret $HOME/.kni/pull-secret.json || true
+cp $HOME/.ssh/id_rsa.pub $HOME/.kni/id_rsa.pub || true
 
-# now build the openshift-install binary and copy to gopath
-make binary 2>&1 | tee ${WORKSPACE}/binary.log
-
-# then start libvirt deploy
-export MASTER_MEMORY_MB=24000
-export CREDENTIALS=file://$(pwd)/akraino-secrets
-export BASE_REPO="git::https://gerrit.akraino.org/r/kni/templates"
-export BASE_PATH="libvirt/1-node"
-export SITE_REPO="git::https://gerrit.akraino.org/r/kni/templates"
-export SETTINGS_PATH="libvirt/sample_settings.yaml"
-export INSTALLER_PATH="file://${WORKSPACE}/bin/openshift-install"
-make deploy 2>&1 | tee ${WORKSPACE}/libvirt_deploy.log
+# start the workflow
+sudo rm -rf /$HOME/.kni/${SITE_NAME}/final_manifests || true
+./knictl fetch_requirements file://${WORKSPACE}/blueprint-pae//sites/${SITE_NAME} 2>&1 | tee ${WORKSPACE}/libvirt_requirements.log
+./knictl prepare_manifests ${SITE_NAME} 2>&1 | tee ${WORKSPACE}/libvirt_manifests.log
+
+# now run the cluster
+source $HOME/.kni/${SITE_NAME}/profile.env
+approve_certs &
+FUNCTION_PID=$!
+sudo -E $HOME/.kni/${SITE_NAME}/requirements/openshift-install create cluster --dir=/$HOME/.kni/${SITE_NAME}/final_manifests 2>&1 | tee ${WORKSPACE}/libvirt_deploy.log
 STATUS=$?
+kill $FUNCTION_PID || true
 
 # output tfstate
 echo "metadata.json for removing cluster"
-cat $(pwd)/build/metadata.json
+cat $HOME/.kni/${SITE_NAME}/final_manifests/metadata.json
 
 if [ $STATUS -ne 0 ]; then
     echo "Error deploying in libvirt"
     exit 1
 fi
 
+echo "Cluster successfully deployed! Start applying workloads"
+./knictl apply_workloads ${SITE_NAME} 2>&1 | tee ${WORKSPACE}/libvirt_workloads.log
+STATUS=$?
+
+if [ $STATUS -ne 0 ]; then
+    echo "Error applying workloads to libvirt"
+    exit 1
+fi
+
+popd
+popd
+
 exit $STATUS