From 4c123de0ab834e6f9a4e6fb7f732338301cee2eb Mon Sep 17 00:00:00 2001 From: "Eby, Robert (re2429)" Date: Mon, 3 Dec 2018 14:13:31 -0500 Subject: [PATCH] Update for staging docker builds Added new shell to check autostage directory Updated triggers for staging docker builds Change-Id: I9b9bfe4fb1b837016a0ab2d9fb7b6ec6e85be6e6 Signed-off-by: Eby, Robert (re2429) --- jjb/akraino-templates/akraino-jjb-docker.yaml | 46 +++++++++++++ jjb/camunda_workflow/camunda_workflow-docker.yaml | 1 + .../portal_user_interface-docker.yaml | 1 + jjb/postgres_db_schema/postgres_db_schema.yaml | 1 + jjb/redfish/redfish-docker.yaml | 1 + jjb/shell/build-docker.sh | 78 ++++++++++++++++------ jjb/shell/get-upstream-autostage.sh | 42 ++++++++++++ 7 files changed, 148 insertions(+), 22 deletions(-) create mode 100644 jjb/shell/get-upstream-autostage.sh diff --git a/jjb/akraino-templates/akraino-jjb-docker.yaml b/jjb/akraino-templates/akraino-jjb-docker.yaml index b80a8a4..6c6afa5 100644 --- a/jjb/akraino-templates/akraino-jjb-docker.yaml +++ b/jjb/akraino-templates/akraino-jjb-docker.yaml @@ -51,3 +51,49 @@ triggers: - timed: '@hourly' + +- job-template: + id: 'akraino-project-stream-stage-docker' + name: '{project}-{stream}-stage-docker' + project-type: freestyle + node: '{build-node}' + + parameters: + - lf-infra-parameters: + project: '{project}' + branch: '{branch}' + stream: '{stream}' + lftools-version: '{lftools-version}' + - string: + name: STAGING_BUILD + default: 'true' + description: Mark this as a staging build. + + scm: + - lf-infra-gerrit-scm: + jenkins-ssh-credential: '{jenkins-ssh-credential}' + git-url: '{git-url}/{project}.git' + refspec: '' + branch: '{branch}' + submodule-recursive: '{submodule-recursive}' + submodule-timeout: '{submodule-timeout}' + choosing-strategy: default + + builders: + - lf-update-java-alternatives: + java-version: '{java-version}' + - lf-infra-docker-login: + global-settings-file: '{global-settings-file}' + settings-file: '{mvn-settings}' + - inject: + properties-content: | + PROJECT={project} + - shell: !include-raw-escape: + - ../shell/get-upstream-autostage.sh + - ../shell/build-docker.sh + + triggers: + - reverse: + jobs: + - '{project}-maven-stage-{stream}' + - 'yaml_builds-{stream}-stage-config' diff --git a/jjb/camunda_workflow/camunda_workflow-docker.yaml b/jjb/camunda_workflow/camunda_workflow-docker.yaml index 9bda997..2d6fff6 100644 --- a/jjb/camunda_workflow/camunda_workflow-docker.yaml +++ b/jjb/camunda_workflow/camunda_workflow-docker.yaml @@ -34,3 +34,4 @@ submodule-recursive: true jobs: - 'akraino-project-stream-docker' + - 'akraino-project-stream-stage-docker' diff --git a/jjb/portal_user_interface/portal_user_interface-docker.yaml b/jjb/portal_user_interface/portal_user_interface-docker.yaml index 6d8168e..ea64d5e 100644 --- a/jjb/portal_user_interface/portal_user_interface-docker.yaml +++ b/jjb/portal_user_interface/portal_user_interface-docker.yaml @@ -34,3 +34,4 @@ submodule-recursive: true jobs: - 'akraino-project-stream-docker' + - 'akraino-project-stream-stage-docker' diff --git a/jjb/postgres_db_schema/postgres_db_schema.yaml b/jjb/postgres_db_schema/postgres_db_schema.yaml index 9411819..feddb06 100644 --- a/jjb/postgres_db_schema/postgres_db_schema.yaml +++ b/jjb/postgres_db_schema/postgres_db_schema.yaml @@ -34,5 +34,6 @@ submodule-recursive: true jobs: - 'akraino-project-stream-docker' + - 'akraino-project-stream-stage-docker' views: - project-view diff --git a/jjb/redfish/redfish-docker.yaml b/jjb/redfish/redfish-docker.yaml index 177f977..7cf6486 100644 --- a/jjb/redfish/redfish-docker.yaml +++ b/jjb/redfish/redfish-docker.yaml @@ -34,3 +34,4 @@ submodule-recursive: true jobs: - 'akraino-project-stream-docker' + - 'akraino-project-stream-stage-docker' diff --git a/jjb/shell/build-docker.sh b/jjb/shell/build-docker.sh index b5041d3..62a2c65 100644 --- a/jjb/shell/build-docker.sh +++ b/jjb/shell/build-docker.sh @@ -15,6 +15,7 @@ # limitations under the License. DOCKER_REPO='nexus3.akraino.org:10003' +STAGING_BUILD=${STAGING_BUILD:=''} set -e -u -x -o pipefail @@ -31,13 +32,23 @@ else case "$PROJECT" in portal_user_interface) CON_NAME='akraino-portal' - VERSION=`xmlstarlet sel -N "x=http://maven.apache.org/POM/4.0.0" -t -v "/x:project/x:version" AECPortalMgmt/pom.xml` - - XMLFILE="${NEXUS_URL}/service/local/repositories/snapshots/content/org/akraino/${PROJECT}/${VERSION}/maven-metadata.xml" - curl -O "${XMLFILE}" - V2=`grep value maven-metadata.xml | sed -e 's;;;' -e 's;.*;;' | uniq` - WARFILE="${NEXUS_URL}/service/local/repositories/snapshots/content/org/akraino/${PROJECT}/${VERSION}/${PROJECT}-${V2}.war" - curl -O "${WARFILE}" + if [ -n "$STAGING_BUILD" -a -n "$AUTOSTAGING" ] + then + # For a staging build, the $VERSION is fixed + VERSION=`xmlstarlet sel -N "x=http://maven.apache.org/POM/4.0.0" -t -v "/x:project/x:version" AECPortalMgmt/pom.xml` + VERSION=$(echo "$VERSION" | sed 's/-SNAPSHOT//') + WARFILE="${NEXUS_URL}/content/repositories/autostaging-${AUTOSTAGING}/org/akraino/${PROJECT}/${VERSION}/${PROJECT}-${VERSION}.war" + DOCKER_REPO='nexus3.akraino.org:10004' + curl -O "${WARFILE}" + else + # For a snapshot build - find the latest snapshot + VERSION=`xmlstarlet sel -N "x=http://maven.apache.org/POM/4.0.0" -t -v "/x:project/x:version" AECPortalMgmt/pom.xml` + XMLFILE="${NEXUS_URL}/service/local/repositories/snapshots/content/org/akraino/${PROJECT}/${VERSION}/maven-metadata.xml" + curl -O "${XMLFILE}" + V2=`grep value maven-metadata.xml | sed -e 's;;;' -e 's;.*;;' | uniq` + WARFILE="${NEXUS_URL}/service/local/repositories/snapshots/content/org/akraino/${PROJECT}/${VERSION}/${PROJECT}-${V2}.war" + curl -O "${WARFILE}" + fi ln $(basename ${WARFILE}) AECPortalMgmt.war ( @@ -48,25 +59,48 @@ else camunda_workflow) CON_NAME='akraino-camunda-workflow-engine' - VERSION=`xmlstarlet sel -N "x=http://maven.apache.org/POM/4.0.0" -t -v "/x:project/x:version" akraino/pom.xml` - - XMLFILE="${NEXUS_URL}/service/local/repositories/snapshots/content/org/akraino/${PROJECT}/${VERSION}/maven-metadata.xml" - curl -O "${XMLFILE}" - V2=`grep value maven-metadata.xml | sed -e 's;;;' -e 's;.*;;' | uniq` - JARFILE="${NEXUS_URL}/service/local/repositories/snapshots/content/org/akraino/${PROJECT}/${VERSION}/${PROJECT}-${V2}.jar" - curl -O ${JARFILE} + if [ -n "$STAGING_BUILD" -a -n "$AUTOSTAGING" ] + then + # For a staging build, the $VERSION is fixed + VERSION=`xmlstarlet sel -N "x=http://maven.apache.org/POM/4.0.0" -t -v "/x:project/x:version" akraino/pom.xml` + VERSION=$(echo "$VERSION" | sed 's/-SNAPSHOT//') + JARFILE="${NEXUS_URL}/content/repositories/autostaging-${AUTOSTAGING}/org/akraino/${PROJECT}/${VERSION}/${PROJECT}-${VERSION}.jar" + DOCKER_REPO='nexus3.akraino.org:10004' + curl -O "${JARFILE}" + else + # For a snapshot build - find the latest snapshot + VERSION=`xmlstarlet sel -N "x=http://maven.apache.org/POM/4.0.0" -t -v "/x:project/x:version" akraino/pom.xml` + XMLFILE="${NEXUS_URL}/service/local/repositories/snapshots/content/org/akraino/${PROJECT}/${VERSION}/maven-metadata.xml" + curl -O "${XMLFILE}" + V2=`grep value maven-metadata.xml | sed -e 's;;;' -e 's;.*;;' | uniq` + JARFILE="${NEXUS_URL}/service/local/repositories/snapshots/content/org/akraino/${PROJECT}/${VERSION}/${PROJECT}-${V2}.jar" + curl -O "${JARFILE}" + fi ;; postgres_db_schema) CON_NAME='akraino_schema_db' - source $WORKSPACE/version.properties - - # Note: for some reason the project name is in the path twice for tar files - XMLFILE="${NEXUS_URL}/service/local/repositories/snapshots/content/org/akraino/yaml_builds/yaml_builds/${VERSION}/maven-metadata.xml" - curl -O "${XMLFILE}" - V2=`grep value maven-metadata.xml | sed -e 's;;;' -e 's;.*;;' | uniq` - TGZFILE="${NEXUS_URL}/service/local/repositories/snapshots/content/org/akraino/yaml_builds/yaml_builds/${VERSION}/yaml_builds-${V2}.tgz" - curl -O "${TGZFILE}" + sudo yum install -y dos2unix + dos2unix "${WORKSPACE}/version.properties" + source "$WORKSPACE/version.properties" + + if [ -n "$STAGING_BUILD" -a -n "$AUTOSTAGING" ] + then + # For a staging build, the $VERSION is fixed + # Note: yaml_builds version MUST match the postgres_db_schema version + VERSION=$(echo "$VERSION" | sed 's/-SNAPSHOT//') + TGZFILE="${NEXUS_URL}/content/repositories/autostaging-${AUTOSTAGING}/yaml_builds-${VERSION}.tgz" + DOCKER_REPO='nexus3.akraino.org:10004' + curl -O "${TGZFILE}" + else + # For a snapshot build - find the latest snapshot + # Note: for some reason the project name is in the path twice for tar files + XMLFILE="${NEXUS_URL}/service/local/repositories/snapshots/content/org/akraino/yaml_builds/yaml_builds/${VERSION}/maven-metadata.xml" + curl -O "${XMLFILE}" + V2=`grep value maven-metadata.xml | sed -e 's;;;' -e 's;.*;;' | uniq` + TGZFILE="${NEXUS_URL}/service/local/repositories/snapshots/content/org/akraino/yaml_builds/yaml_builds/${VERSION}/yaml_builds-${V2}.tgz" + curl -O "${TGZFILE}" + fi (mkdir yaml_builds; cd yaml_builds; tar xfv ../$(basename ${TGZFILE})) mv yaml_builds/templates akraino-j2templates ;; diff --git a/jjb/shell/get-upstream-autostage.sh b/jjb/shell/get-upstream-autostage.sh new file mode 100644 index 0000000..8c2f0f5 --- /dev/null +++ b/jjb/shell/get-upstream-autostage.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# +# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copied (and modified slightly) from edgexfoundry +# Do not fail this job as this is a stopgap for now. +set +e + +# Look in our console log for the triggering build number +T=/tmp/ups.$$ +curl $BUILD_URL/consoleText | grep "Started by upstream project" > $T +if [ -s $T ] +then + # $T should contain "06:25:53 Started by upstream project "projname" build number 111" + upstream_name=$( awk -F '"' '{print $2}' < $T) + upstream_number=$( awk -F '"' '{print $3}' < $T | awk -F ' ' '{print$3}') + + # This is the URL of the console log for the triggering job + upstream_url="https://jenkins.akraino.org/job/$upstream_name/$upstream_number/consoleText" + + # This identifies the staging directory for the staging URL + # https://nexus.akraino.org/content/repositories/autostaging-${AUTOSTAGING}/org/akraino/${PROJECT}/${VERSION}/${PROJECT}-${VERSION}.[jw]ar + # or + # https://nexus.akraino.org/content/repositories/autostaging-${AUTOSTAGING}/${PROJECT}-${VERSION}.tgz + # e.g. + # https://nexus.akraino.org/content/repositories/autostaging-1100/org/akraino/portal_user_interface/0.0.2/portal_user_interface-0.0.2.war + # https://nexus.akraino.org/content/repositories/autostaging-1099/org/akraino/camunda_workflow/0.0.2/camunda_workflow-0.0.2.jar + # https://nexus.akraino.org/content/repositories/autostaging-1091/airshipinabottle_deploy-0.0.1.tgz + AUTOSTAGING=$(curl $upstream_url | grep "Completed uploading files to autostaging" | sed 's/.*-//' | tr -d '.') +fi -- 2.16.6