From: Saku Chydenius <saku.chydenius@nokia.com>
Date: Sat, 11 May 2019 13:53:13 +0000 (+0300)
Subject: Add a script to build REC images
X-Git-Url: https://gerrit.akraino.org/r/gitweb?p=ta%2Fbuild-tools.git;a=commitdiff_plain;h=900738828f48bade06f69c1e3a8f6fb988b97950

Add a script to build REC images

This can be executed locally as well as in Jenkins to create REC images.

To create REC image also manifest RPM must be built to get build
information included to the REC runtime environment. This information
can be then used within the REC to indentify which build has been
installed.

The "rpmbuilder" tool (that uses "mock") is way too heavy for manifest
RPM creation thus Alpine based build environment was chosen. Manifest
RPM has no dependencies thus this should be ok although experimental.

Change-Id: Ic2ca3115bbfbbac89bf3139c47f8467a76ebd0b6
Signed-off-by: Saku Chydenius <saku.chydenius@nokia.com>
---

diff --git a/README b/README
index 4efc9eb..879250b 100644
--- a/README
+++ b/README
@@ -6,3 +6,9 @@ Execute with:
 Execute only subset with verbose diff on assertion errors:
   $ tox -e py27 -- -vv tools/script/process_rpmdata_test.py::test_components
 
+# Building REC
+
+  $ git clone <manifest-repo>
+  $ git clone <build-tools-repo>
+  $ build-tools/build_images.sh -m manifest -w work
+
diff --git a/build_images.sh b/build_images.sh
new file mode 100755
index 0000000..9d94f59
--- /dev/null
+++ b/build_images.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+set -eu
+
+usage() {
+    echo "Usage: $0 -m <manifest-path> -w <work-dir-path>"
+    exit 1
+}
+
+[ "$#" -ne 4 ] && usage
+while getopts "m:w:" OPT; do
+    case $OPT in
+    m)
+        export MANIFEST_PATH=$(readlink -f $OPTARG)
+        ;;
+    w)
+        export WORK=$OPTARG
+        ;;
+    *)
+        usage
+        ;;
+    esac
+done
+
+scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
+source $scriptdir/lib.sh
+
+_initialize_work_dirs
+
+# Create manifest RPM
+$LIBDIR/create_manifest_rpm.sh
+
+# Create repo config
+$LIBDIR/build_step_create_yum_repo_files.sh
+
+# QCOW
+$LIBDIR/build_step_golden_image.sh
+sha1short=$(grep -v product-manifest $RPMLISTS/rpmlist | sha1sum | cut -c-8)
+
+# ISO images
+$LIBDIR/build_step_create_install_cd.sh
+
+echo "=== SUCCESS ==="
+echo "Build results are in $WORKRESULTS"
+echo "Installed RPMS checksum: ${sha1short} (this will change if list of installed RPMs changes)"
diff --git a/build_step_create_install_cd.sh b/build_step_create_install_cd.sh
index 40de1cc..f2f7dab 100755
--- a/build_step_create_install_cd.sh
+++ b/build_step_create_install_cd.sh
@@ -23,11 +23,9 @@ _read_manifest_vars
 tmp=$WORKTMP/install_cd
 iso_build_dir=$tmp/build
 
-input_image="$WORKTMP/goldenimage/${GOLDEN_IMAGE_NAME}"
-output_image_path="$1"
-[[ $output_image_path =~ ^/ ]] || output_image_path=$(pwd)/$output_image_path
-output_bootcd_path="$2"
-[[ $output_bootcd_path =~ ^/ ]] || output_bootcd_path=$(pwd)/$output_bootcd_path
+input_image=$(readlink -f ${1:-$WORKTMP/goldenimage/$GOLDEN_IMAGE_NAME})
+output_image_path=${2:-$RESULT_IMAGES_DIR/rec.iso}
+output_bootcd_path=${3:-$RESULT_IMAGES_DIR/bootcd.iso}
 mkdir -p $tmp
 rm -rf $iso_build_dir
 mkdir -p $iso_build_dir
diff --git a/build_step_golden_image.sh b/build_step_golden_image.sh
index 2dc1a37..dad25b4 100755
--- a/build_step_golden_image.sh
+++ b/build_step_golden_image.sh
@@ -19,10 +19,7 @@ set -e
 scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
 source $scriptdir/lib.sh
 
-output_image_path="$1"
-[[ $output_image_path =~ ^/ ]] || output_image_path=$(pwd)/$output_image_path
-rpm_info_output_dir=$2
-[[ $rpm_info_output_dir =~ ^/ ]] || rpm_info_output_dir=$(pwd)/$rpm_info_output_dir
+output_image_path=${1:-$WORKTMP/goldenimage/$GOLDEN_IMAGE_NAME}
 
 docker_dib_image=dib:2.0
 _load_docker_image $docker_dib_image
@@ -32,10 +29,15 @@ docker run \
   --privileged \
   -v /dev:/dev \
   -v $WORK:/work \
+  -e WORK=/work \
+  -v $MANIFEST_PATH:/manifest \
+  -e MANIFEST_PATH=/manifest \
+  -v $scriptdir:/tools \
   $docker_dib_image \
-  $(realpath --relative-to $WORK $scriptdir)/create_golden_image.sh
+  /tools/create_golden_image.sh
 
-_publish_image ${TMP_GOLDEN_IMAGE}.qcow2 $output_image_path/${GOLDEN_IMAGE_NAME}
+mkdir -p $(dirname $output_image_path)
+mv -f ${TMP_GOLDEN_IMAGE}.qcow2 $output_image_path
 
 input_dir=$WORKTMP/rpmdata
 mkdir $input_dir
diff --git a/prepare_manifest.sh b/create_manifest_rpm.sh
similarity index 50%
rename from prepare_manifest.sh
rename to create_manifest_rpm.sh
index e34536d..b7ac213 100755
--- a/prepare_manifest.sh
+++ b/create_manifest_rpm.sh
@@ -14,15 +14,29 @@
 # limitations under the License.
 
 set -x
+set -eu
 
 scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
 source $scriptdir/lib.sh
-_read_manifest_vars
 
-pushd $MANIFEST_PATH
-sed -i -e "s/^Version: .*/Version: $PRODUCT_RELEASE_BUILD_ID/" rpmbuild.spec
-cat > product-release <<EOF
-release=$PRODUCT_RELEASE_LABEL
-build=$PRODUCT_RELEASE_BUILD_ID
-EOF
-popd
+work=${1:-$WORKTMP/manifest-work}
+work=$(readlink -f $work)
+rm -rf $work
+mkdir -p $work
+cp -f $MANIFEST_PATH/*.spec $work
+
+$scriptdir/create_mock_config.sh $MANIFEST_PATH/build_config.ini $work/mock_config
+
+rpm_macros=$work/rpmmacros
+$scriptdir/mock2rpmbuild_config.py --mock-config $work/mock_config/mock.cfg --output-file-path $rpm_macros
+
+docker run --rm \
+    -v $rpm_macros:/root/.rpmmacros \
+    -v $work:/work \
+    alpine:3.9.4 \
+    sh -c '\
+        apk add rpm && \
+        rpmbuild --build-in-place --rmspec -ba /work/*.spec && \
+        find /root/rpmbuild -name "*.rpm" | xargs -I "{}" mv {} /work'
+
+_add_rpms_to_localrepo $(find $work -name '*.rpm')
diff --git a/create_mock_config.sh b/create_mock_config.sh
index 701cbc1..c7db5cf 100755
--- a/create_mock_config.sh
+++ b/create_mock_config.sh
@@ -17,6 +17,7 @@ set -e
 
 scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
 source $scriptdir/lib.sh
+_read_manifest_vars
 
 config_ini=${1:-$BUILD_CONFIG_INI}
 output_repo_files_dir=${2:-$REPO_FILES}
@@ -37,6 +38,8 @@ sed -i \
   -e "s/#RPM_VENDOR#/\"$(_read_build_config $config_ini rpm vendor)\"/" \
   -e "s/#RPM_LICENSE#/\"$(_read_build_config $config_ini rpm license)\"/" \
   -e "s/#RPM_RELEASE_ID#/\"$(_read_build_config $config_ini rpm release_id)\"/" \
+  -e "s/#PRODUCT_RELEASE_BUILD_ID#/\"$PRODUCT_RELEASE_BUILD_ID\"/" \
+  -e "s/#PRODUCT_RELEASE_LABEL#/\"$PRODUCT_RELEASE_LABEL\"/" \
   $mock_cfg
 
 docker_sock=/var/run/docker.sock
diff --git a/create_rpmdata_in_docker.sh b/create_rpmdata_in_docker.sh
index 84d2472..3e0885d 100755
--- a/create_rpmdata_in_docker.sh
+++ b/create_rpmdata_in_docker.sh
@@ -39,7 +39,7 @@ output_mp=/output
 docker run \
   --rm \
   -e PYTHONPATH=/work \
-  -e BUILD_URL -e JENKINS_USERNAME -e JENKINS_TOKEN -e WORKSPACE \
+  -e BUILD_URL -e JENKINS_USERNAME -e JENKINS_TOKEN -e WORK \
   -v $scriptdir:/work \
   -v $input_dir:$input_mp \
   -v $output_dir:$output_mp \
diff --git a/lib.sh b/lib.sh
index 671d041..9bc0e16 100644
--- a/lib.sh
+++ b/lib.sh
@@ -18,23 +18,18 @@ set -e
 
 LIBDIR="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
 
-PUBLISH_RESULTS="${PUBLISH_RESULTS:-false}"
-VIRT_CUSTOMIZE_MEM="${VIRT_CUSTOMIZE_MEM:-}"
-VIRT_CUSTOMIZE_SMP="${VIRT_CUSTOMIZE_SMP:-}"
-PARALLEL_BUILD_TIMEOUT="${PARALLEL_BUILD_TIMEOUT:-0}"
-ENABLE_GOLDEN_IMAGE_ROOT_PASSWORD="${ENABLE_GOLDEN_IMAGE_ROOT_PASSWORD:-true}"
-GOLDEN_BASE_IMAGE_TAR_URL=${GOLDEN_BASE_IMAGE_TAR_URL:-}
 GOLDEN_BASE_IMAGE_FETCH_USER=${GOLDEN_BASE_IMAGE_FETCH_USER:-}
 GOLDEN_BASE_IMAGE_FETCH_PASSWORD=${GOLDEN_BASE_IMAGE_FETCH_PASSWORD:-}
 
-WORK=$(dirname $(dirname $LIBDIR))
+WORK=$(readlink -f ${WORK:-$(dirname $(dirname $LIBDIR))})
+mkdir -p $WORK
 RPM_BUILDER=$(find $WORK -maxdepth 2 -type d -name rpmbuilder)
 
 WORKTMP=$WORK/tmp
 WORKLOGS=$WORKTMP/logs
 DURATION_LOG=$WORKLOGS/durations.log
-MANIFEST_PATH=$WORK/.repo/manifests
-BUILD_CONFIG_INI=$WORK/.repo/manifests/build_config.ini
+MANIFEST_PATH=$(readlink -f ${MANIFEST_PATH:-$WORK/.repo/manifests})
+BUILD_CONFIG_INI=${BUILD_CONFIG_INI:-$MANIFEST_PATH/build_config.ini}
 GOLDEN_IMAGE_NAME=guest-image.img
 TMP_GOLDEN_IMAGE=$WORKTMP/$GOLDEN_IMAGE_NAME
 
@@ -59,14 +54,15 @@ function _read_build_config()
 
 function _read_manifest_vars()
 {
-  PRODUCT_RELEASE_BUILD_ID="${BUILD_NUMBER:?0}"
+  PRODUCT_RELEASE_BUILD_ID="${BUILD_NUMBER:-0}"
   PRODUCT_RELEASE_LABEL="$(_read_build_config DEFAULT product_release_label)"
 }
 
 function _initialize_work_dirs()
 {
+  mkdir -p $WORK
   rm -rf $WORKRESULTS
-  mkdir -p $WORKRESULTS $REPO_FILES $REPO_DIR $RPMLISTS $CHECKSUM_DIR
+  mkdir -p $WORKRESULTS $REPO_FILES $REPO_DIR $SRC_REPO_DIR $RPMLISTS $CHECKSUM_DIR
   # dont clear tmp, can be used for caching
   mkdir -p $WORKTMP
   rm -rf $WORKLOGS
@@ -140,8 +136,22 @@ function _run_cmd_as_step()
   _success $step
 }
 
+function _add_rpms_to_localrepo()
+{
+  local rpms=$@
+  mkdir -p $REPO_DIR
+  mkdir -p $SRC_REPO_DIR
+  for rpm in $@; do
+    if grep ".src.rpm" <<< "$rpm"; then
+      cp -f $rpm $SRC_REPO_DIR
+    else
+      cp -f $rpm $REPO_DIR
+    fi
+  done
+  _create_localrepo
+}
 
-function _add_rpms_to_repo()
+function _add_rpms_dir_to_repo()
 {
   local repo_dir=$1
   local rpm_dir=$2
@@ -161,8 +171,8 @@ function _create_localrepo()
 
 function _add_rpms_to_repos_from_workdir()
 {
-  _add_rpms_to_repo $REPO_DIR $1/buildrepository/mock/rpm
-  _add_rpms_to_repo $SRC_REPO_DIR $1/buildrepository/mock/srpm
+  _add_rpms_dir_to_repo $REPO_DIR $1/buildrepository/mock/rpm
+  _add_rpms_dir_to_repo $SRC_REPO_DIR $1/buildrepository/mock/srpm
   #find $1/ -name '*.tar.gz' | xargs rm -f
   true
 }
diff --git a/mock/mock.cfg.template b/mock/mock.cfg.template
index 17800b5..395c4a6 100644
--- a/mock/mock.cfg.template
+++ b/mock/mock.cfg.template
@@ -32,6 +32,8 @@ config_opts['exclude_from_homedir_cleanup'] = ('build/SOURCES', '.bash_history',
 # Common RPM directive values
 config_opts['macros']['%packager'] = #RPM_PACKAGER#
 config_opts['macros']['%dist'] = ".el7.centos%{_platform_product}"
+config_opts['macros']['%_platform_product_release_build_id'] = #PRODUCT_RELEASE_BUILD_ID#
+config_opts['macros']['%_platform_product_release_label'] = #PRODUCT_RELEASE_LABEL#
 config_opts['macros']['%_platform_product'] = #RPM_RELEASE_ID#
 config_opts['macros']['%_platform_dist'] = ".el7.centos"
 config_opts['macros']['%_platform_vendor'] = #RPM_VENDOR#
diff --git a/mock2rpmbuild_config.py b/mock2rpmbuild_config.py
new file mode 100755
index 0000000..101b8d1
--- /dev/null
+++ b/mock2rpmbuild_config.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import sys
+import argparse
+
+
+def parse(args):
+    p = argparse.ArgumentParser(
+        description='Parse RPM macro definitions from mock '
+                    'config and write a file for "rpmbuild" to process')
+    p.add_argument('--mock-config', required=True)
+    p.add_argument('--output-file-path', required=True)
+    args = p.parse_args(args)
+    return args
+
+
+def main(input_args):
+    args = parse(input_args)
+    with open(args.mock_config, 'r') as f:
+        data = f.read()
+    with open(args.output_file_path, 'w') as f:
+        for macro, expr in re.findall(
+                r'^\s*config_opts\[[\'"]macros[\'"]\]\[[\'"](.*)[\'"]\]\s*=\s*[\'"](.*)[\'"]\s*$',
+                data, re.MULTILINE):
+            f.write('{} {}\n'.format(macro, expr))
+    print('Wrote ' + args.output_file_path)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])
diff --git a/tools/repository.py b/tools/repository.py
index 19000e9..7f0511a 100755
--- a/tools/repository.py
+++ b/tools/repository.py
@@ -47,7 +47,7 @@ class RepositoryConfig(object):
             validate_environ(['BUILD_URL'])
             baseurl = os.path.join(os.environ['BUILD_URL'], 'artifact/results', dirname)
         else:
-            validate_environ(['WORKSPACE'])
+            validate_environ(['WORK'])
             baseurl = 'file://' + \
-                      os.path.join(os.environ['WORKSPACE'], 'results', dirname)
+                      os.path.join(os.environ['WORK'], 'results', dirname)
         return dict(name='localrepo', baseurl=baseurl)
diff --git a/tools/script/create_rpm_data.py b/tools/script/create_rpm_data.py
index 9c4b747..82beabf 100755
--- a/tools/script/create_rpm_data.py
+++ b/tools/script/create_rpm_data.py
@@ -191,7 +191,10 @@ class RpmDataBuilder(object):
     def _read_configured_repos(self):
         repos = self.repoconfig.read_sections(
             ['baseimage-repositories', 'repositories'])
-        repos.append(self.repoconfig.get_localrepo(remote=True))
+        if 'BUILD_URL' in os.environ:
+            repos.append(self.repoconfig.get_localrepo(remote=True))
+        else:
+            repos.append(self.repoconfig.get_localrepo(remote=False))
         logging.debug('Configured repos: {}'.format(pformat(repos)))
         return repos
 
diff --git a/tools/script/create_rpm_data_test.py b/tools/script/create_rpm_data_test.py
index 1e47e89..c9f82b0 100755
--- a/tools/script/create_rpm_data_test.py
+++ b/tools/script/create_rpm_data_test.py
@@ -59,7 +59,7 @@ def test_complete_parse(mock_config, mock_reporeader, _):
                                    Result(0, '', ''),  # rm -rf /var/yum/cache
                                    Result(0, yum_available_output, '')]
     os.environ['BUILD_URL'] = 'test-url/'
-    os.environ['WORKSPACE'] = '/foo/path'
+    os.environ['WORK'] = '/foo/path'
     result = RpmDataBuilder('fake_build_config_path',
                             yum_installed_output,
                             rpm_info_output,
diff --git a/tools/statics.py b/tools/statics.py
index f1ebd9c..12950aa 100755
--- a/tools/statics.py
+++ b/tools/statics.py
@@ -20,7 +20,7 @@ WORK_ROOT = os.path.dirname(
             os.path.dirname(
                 os.path.realpath(__file__)))))
 
-MAINFEST_PATH = os.path.join(WORK_ROOT, '.repo/manifests')
+MAINFEST_PATH = os.environ.get('MANIFEST_PATH', os.path.join(WORK_ROOT, '.repo/manifests'))
 
 BUILD_CONFIG_PATH = os.path.join(MAINFEST_PATH, 'build_config.ini')
 PACKAGES_CONFIG_PATH = os.path.join(MAINFEST_PATH, 'packages.yaml')