From eb6df414a1b5d1b7b67d3dd5f9b7d0529b8cf5fc Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Wed, 7 Aug 2019 14:57:59 +0000 Subject: [PATCH] AArch64 (arm64) support for image building, mock - mock: * extend config_opts['legal_host_arches'] to include 'aarch64'; * parameterize RPM target architecture based on the output of $(uname -m) in create_mock_config.sh; - Dockerfile-dib: * Bump base to centos:7.6.1810, but since centos:7.x.y tags are broken on arm64 [1], apply an arch-specific patch to the DIB Dockerfile which pins the FROM image to a specific sha256; * Use AArch64 altsig repository for openstack-queens packages; * Bump diskimage-builder to 2.26.1 to include arm64 critical fixes; * Add tools required for UEFI partition handling and other AArch64 specific requirements to the arm64 container image (gdisk, dosfstools, util-linux, qemu-img); - dib elements: * finalise.d/99-collect-rpm-info: AArch64 workaround for packages initially installed from 'updates' repo residing in 'base' repo after all distro/repo handling take place; - create_golden_image.sh: * dib: Add additional element block-device-efi on AArch64 (only); - build_step_create_install_cd.sh: * parameterize CentOS artifacts URI based on machine architecture; * skip isolinux artifacts download/handling for AArch64, since PC BIOS is x86 specific; * ISO(s) build: AArch64 only supports UEFI booting, skip isolinux; * AArch64: use legacy method of copying out the contents of "/boot" from the golden image since the AArch64 image has a more complex partition layout and would require mounting both "/boot" and ESP; - create_rpmdata_in_docker.sh: * fix bash shebang since script uses bash-isms; - tools/buildconfig.py: * add support for replacing the magic string '#ARCH#' with current platform architecture in option names; Signed-off-by: Alexandru Avadanii Co-authored-by: Valentin Radulescu Change-Id: I32cfdc2adc43fb703843383a1e94a807919f4e8c --- build_images.sh | 3 + build_step_create_install_cd.sh | 78 +++++++++++++--------- build_step_golden_image.sh | 2 +- create_golden_image.sh | 3 +- create_mock_config.sh | 1 + create_rpmdata_in_docker.sh | 2 +- .../myproduct/finalise.d/99-collect-rpm-info | 10 +++ docker-context/Dockerfile-dib | 19 ++++-- docker-context/Dockerfile-dib.aarch64.patch | 13 ++++ mock/mock.cfg.template | 4 +- tools/buildconfig.py | 7 +- 11 files changed, 101 insertions(+), 41 deletions(-) create mode 100644 docker-context/Dockerfile-dib.aarch64.patch diff --git a/build_images.sh b/build_images.sh index bbaaf00..0d7968b 100755 --- a/build_images.sh +++ b/build_images.sh @@ -54,6 +54,9 @@ source $scriptdir/lib.sh _initialize_work_dirs +dib_patch="$scriptdir/docker-context/Dockerfile-dib.$(uname -m).patch" +if [ -e "${dib_patch}" ]; then patch "$scriptdir/docker-context/Dockerfile-dib" "${dib_patch}"; fi + docker build -f $scriptdir/docker-context/Dockerfile-dib -t dib $scriptdir/docker-context docker build -f $scriptdir/docker-context/Dockerfile-buildtools -t buildtools $scriptdir/docker-context diff --git a/build_step_create_install_cd.sh b/build_step_create_install_cd.sh index 78bafd3..40d3880 100755 --- a/build_step_create_install_cd.sh +++ b/build_step_create_install_cd.sh @@ -30,9 +30,10 @@ mkdir -p $tmp rm -rf $iso_build_dir mkdir -p $iso_build_dir +iso_arch="$(uname -m)" reposnap_base=$(_read_build_config DEFAULT centos_reposnap) release_version=$PRODUCT_RELEASE_LABEL -reposnap_base_dir="${reposnap_base}/os/x86_64/" +reposnap_base_dir="${reposnap_base}/os/${iso_arch}/" iso_image_label=$(_read_build_config DEFAULT iso_image_label) cd_efi_dir="${reposnap_base_dir}/EFI" cd_images_dir="${reposnap_base_dir}/images" @@ -59,48 +60,63 @@ wget_dir ${cd_efi_dir}/ wget_dir ${cd_images_dir}/ rm -f images/boot.iso sync -wget_dir ${cd_isolinux_dir}/ -chmod +w -R isolinux/ EFI/ images/ - -if [ -e $scriptdir/isolinux/isolinux.cfg ]; then - cp $scriptdir/isolinux/isolinux.cfg isolinux/isolinux.cfg -else - sed -i "s/^timeout.*/timeout 100/" isolinux/isolinux.cfg - sed -i "s/^ - Press.*/Beginning the cloud installation process/" isolinux/boot.msg - sed -i "s/^#menu hidden/menu hidden/" isolinux/isolinux.cfg - sed -i "s/menu default//" isolinux/isolinux.cfg - sed -i "/^label linux/amenu default" isolinux/isolinux.cfg - sed -i "/append initrd/ s/$/ console=tty0 console=ttyS1,115200/" isolinux/isolinux.cfg +chmod +w -R EFI/ images/ +# AArch64 does not support PC-BIOS, so skip all isolinux processing +if [ "${iso_arch}" != 'aarch64' ]; then + wget_dir ${cd_isolinux_dir}/ + chmod +w -R isolinux/ + + if [ -e $scriptdir/isolinux/isolinux.cfg ]; then + cp $scriptdir/isolinux/isolinux.cfg isolinux/isolinux.cfg + else + sed -i "s/^timeout.*/timeout 100/" isolinux/isolinux.cfg + sed -i "s/^ - Press.*/Beginning the cloud installation process/" isolinux/boot.msg + sed -i "s/^#menu hidden/menu hidden/" isolinux/isolinux.cfg + sed -i "s/menu default//" isolinux/isolinux.cfg + sed -i "/^label linux/amenu default" isolinux/isolinux.cfg + sed -i "/append initrd/ s/$/ console=tty0 console=ttyS1,115200/" isolinux/isolinux.cfg + fi + cp -f $scriptdir/akraino_splash.png isolinux/splash.png fi -cp -f $scriptdir/akraino_splash.png isolinux/splash.png popd pushd $tmp # Copy latest kernel and initrd-provisioning from boot dir -qemu-img convert $input_image guest-image.raw -myloop=$(sudo losetup -fP --show guest-image.raw) -mkdir mnt -sudo mount -o loop ${myloop}p1 mnt/ -sudo rsync -avA mnt/boot . -sudo chown -R $(id -u):$(id -g) boot -sudo umount mnt -sudo losetup -d ${myloop} -rm -f guest-image.raw +if [ "${iso_arch}" != 'aarch64' ]; then + qemu-img convert $input_image guest-image.raw + myloop=$(sudo losetup -fP --show guest-image.raw) + mkdir mnt + sudo mount -o loop ${myloop}p1 mnt/ + sudo rsync -avA mnt/boot . + sudo chown -R $(id -u):$(id -g) boot + sudo umount mnt + sudo losetup -d ${myloop} + rm -f guest-image.raw +else + export LIBGUESTFS_BACKEND=direct + virt-copy-out -a $input_image /boot/ ./ +fi chmod u+w boot/ -rm -f $iso_build_dir/isolinux/vmlinuz $iso_build_dir/isolinux/initrd.img KVER=`ls -lrt boot/vmlinuz-* |grep -v rescue |tail -n1 |awk -F 'boot/vmlinuz-' '{print $2}'` -cp -fp boot/vmlinuz-${KVER} $iso_build_dir/isolinux/vmlinuz -cp -fp boot/initrd-provisioning.img $iso_build_dir/isolinux/initrd.img +if [ "${iso_arch}" != 'aarch64' ]; then + rm -f $iso_build_dir/isolinux/vmlinuz $iso_build_dir/isolinux/initrd.img + cp -fp boot/vmlinuz-${KVER} $iso_build_dir/isolinux/vmlinuz + cp -fp boot/initrd-provisioning.img $iso_build_dir/isolinux/initrd.img +fi rm -rf boot/ echo "Generating boot iso" +if [ "${iso_arch}" != 'aarch64' ]; then + bios_specific_args="-b isolinux/isolinux.bin -c isolinux/boot.cat \ + -no-emul-boot -boot-load-size 4 -boot-info-table" +fi genisoimage -U -r -v -T -J -joliet-long \ -V "${release_version}" -A "${release_version}" -P ${iso_image_label} \ - -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 \ - -boot-info-table -eltorito-alt-boot -e images/efiboot.img -no-emul-boot \ + ${bios_specific_args:-} \ + -eltorito-alt-boot -e images/efiboot.img -no-emul-boot \ -o boot.iso $iso_build_dir _publish_image $tmp/boot.iso $output_bootcd_path @@ -112,10 +128,10 @@ mkdir -p $iso_build_dir/rpms echo "Generating product iso" genisoimage -U -r -v -T -J -joliet-long \ -V "${release_version}" -A "${release_version}" -P ${iso_image_label} \ - -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 \ - -boot-info-table -eltorito-alt-boot -e images/efiboot.img -no-emul-boot \ + ${bios_specific_args:-} \ + -eltorito-alt-boot -e images/efiboot.img -no-emul-boot \ -o release.iso $iso_build_dir -isohybrid $tmp/release.iso +[ "${iso_arch}" = 'aarch64' ] || isohybrid $tmp/release.iso _publish_image $tmp/release.iso $output_image_path echo "Clean up to preserve workspace footprint" diff --git a/build_step_golden_image.sh b/build_step_golden_image.sh index ceb9a90..e409d35 100755 --- a/build_step_golden_image.sh +++ b/build_step_golden_image.sh @@ -40,7 +40,7 @@ mkdir -p $(dirname $output_image_path) mv -f ${TMP_GOLDEN_IMAGE}.qcow2 $output_image_path input_dir=$WORKTMP/rpmdata -mkdir $input_dir +mkdir -p $input_dir cp -r \ $BUILD_CONFIG_INI $RPMLISTS/rpm_info_installed $RPMLISTS/yum_info_installed $RPMLISTS/crypto_rpms.json $RPMLISTS/boms \ $input_dir diff --git a/create_golden_image.sh b/create_golden_image.sh index 307da83..2a9d66a 100755 --- a/create_golden_image.sh +++ b/create_golden_image.sh @@ -25,6 +25,7 @@ CENTOS_SNAP=${CENTOS_SNAP:-$(_read_build_config DEFAULT centos_reposnap)} BASE_IMAGE_NAME=`echo $BASE_IMAGE_URL | awk -F "/" '{print $NF}'` BASE_IMAGE_SIZE="8GiB" +[ "$(uname -m)" != aarch64 ] || EXTRA_ELEMENTS="block-device-efi" wget_args="" [ -n "$GOLDEN_BASE_IMAGE_FETCH_USER" ] && wget_args="$wget_args --http-user=$GOLDEN_BASE_IMAGE_FETCH_USER" @@ -55,6 +56,6 @@ DIB_DEBUG_TRACE=1 \ DIB_YUM_REPO_CONF="${REPO_FILES}/repositories.repo ${REPO_FILES}/localrepo.repo" \ DIB_LOCAL_IMAGE=$WORKTMP/base-img/$BASE_IMAGE_NAME \ ELEMENTS_PATH=$scriptdir/dib_elements/ \ - /usr/bin/disk-image-create --root-label img-rootfs --image-size $BASE_IMAGE_SIZE vm centos7 selinux-permissive myproduct -o $TMP_GOLDEN_IMAGE + /usr/bin/disk-image-create --root-label img-rootfs --image-size $BASE_IMAGE_SIZE vm centos7 selinux-permissive myproduct ${EXTRA_ELEMENTS:-} -o $TMP_GOLDEN_IMAGE rm -rf $WORKTMP/base-img diff --git a/create_mock_config.sh b/create_mock_config.sh index fd39fed..f05dfdc 100755 --- a/create_mock_config.sh +++ b/create_mock_config.sh @@ -34,6 +34,7 @@ cp $LIBDIR/mock/site-defaults.cfg $output_mock_dir/ mock_cfg=$output_mock_dir/mock.cfg sed -e "/#REPOSITORIES#/r $output_repo_files_dir/repositories.repo" $LIBDIR/mock/mock.cfg.template > $mock_cfg sed -i \ + -e "s/#RPM_ARCH#/\"$(uname -m)\"/" \ -e "s/#RPM_PACKAGER#/\"$(_read_build_config $config_ini rpm packager)\"/" \ -e "s/#RPM_VENDOR#/\"$(_read_build_config $config_ini rpm vendor)\"/" \ -e "s/#RPM_LICENSE#/\"$(_read_build_config $config_ini rpm license)\"/" \ diff --git a/create_rpmdata_in_docker.sh b/create_rpmdata_in_docker.sh index 26b6159..82c7bcc 100755 --- a/create_rpmdata_in_docker.sh +++ b/create_rpmdata_in_docker.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Copyright 2019 Nokia # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/dib_elements/myproduct/finalise.d/99-collect-rpm-info b/dib_elements/myproduct/finalise.d/99-collect-rpm-info index f3cc6b8..439fe4b 100755 --- a/dib_elements/myproduct/finalise.d/99-collect-rpm-info +++ b/dib_elements/myproduct/finalise.d/99-collect-rpm-info @@ -19,6 +19,16 @@ fi set -eu set -o pipefail +# On AArch64, since the cloud image is very old, some packages are obsolete and +# get upgraded from the 'updates' repository. However, after the whole distro +# is upgraded and the repos are pointing to the latest repositories, the same +# packages previously available in the 'updates' repo are now listed in the +# 'base' repo, breaking our RPM availability check below. +# +# Work around this by forcing a reinstall of the problematic packages, which +# will also update the 'From repo' fields. +[ "$(uname -m)" != aarch64 ] || yum reinstall -y ncurses ncurses-base ncurses-libs + yum info installed > /root/yum_info_installed rpm -qai --queryformat "Obsoletes : [%{OBSOLETES},]\n" > /root/rpm_info_installed diff --git a/docker-context/Dockerfile-dib b/docker-context/Dockerfile-dib index 1a339f0..33b0e2b 100644 --- a/docker-context/Dockerfile-dib +++ b/docker-context/Dockerfile-dib @@ -14,12 +14,19 @@ # Use newer than the CentOS stock diskimage-builder via "cloud" repo -FROM centos:7.5.1804 +FROM centos:7.6.1810 RUN \ - yum-config-manager --add-repo http://mirror.centos.org/centos/7/cloud/x86_64/openstack-queens/ && \ - yum install --nogpgcheck -y diskimage-builder \ + if [ "$(uname -m)" = aarch64 ]; then \ + REPO_URI=http://mirror.centos.org/altarch/7/cloud/aarch64/openstack-queens/; \ + EXTRA_RPM="gdisk dosfstools util-linux"; \ + else \ + REPO_URI=http://mirror.centos.org/centos/7/cloud/x86_64/openstack-queens/; \ + fi && \ + yum-config-manager --add-repo $REPO_URI && \ + yum install --nogpgcheck -y \ git \ python \ + python-pip \ wget \ which \ findutils \ @@ -28,7 +35,11 @@ RUN \ parted \ sudo \ e2fsprogs \ - xfsprogs + xfsprogs \ + qemu-img \ + ${EXTRA_RPM:-} && \ + pip install diskimage-builder==2.26.1 && \ + yum remove -y python-pip RUN find -name 99-selinux-fixfiles-restore -exec rm {} \; WORKDIR /work diff --git a/docker-context/Dockerfile-dib.aarch64.patch b/docker-context/Dockerfile-dib.aarch64.patch new file mode 100644 index 0000000..5af6d04 --- /dev/null +++ b/docker-context/Dockerfile-dib.aarch64.patch @@ -0,0 +1,13 @@ +diff --git a/docker-context/Dockerfile-dib b/docker-context/Dockerfile-dib +index 038b64d..393f10f 100644 +--- a/docker-context/Dockerfile-dib ++++ b/docker-context/Dockerfile-dib +@@ -14,7 +14,7 @@ + + + # Use newer than the CentOS stock diskimage-builder via "cloud" repo +-FROM centos:7.6.1810 ++FROM centos@sha256:df89b0a0b42916b5b31b334fd52d3e396c226ad97dfe772848bdd6b00fb42bf0 + RUN \ + if [ "$(uname -m)" = aarch64 ]; then \ + REPO_URI=http://mirror.centos.org/altarch/7/cloud/aarch64/openstack-queens/; \ diff --git a/mock/mock.cfg.template b/mock/mock.cfg.template index c5b3496..26ce44e 100644 --- a/mock/mock.cfg.template +++ b/mock/mock.cfg.template @@ -15,8 +15,8 @@ # Root name to be used for chroot and caching, must differ between products config_opts['root'] = 'akrainolite' -config_opts['target_arch'] = 'x86_64' -config_opts['legal_host_arches'] = ('x86_64',) +config_opts['target_arch'] = #RPM_ARCH# +config_opts['legal_host_arches'] = ('x86_64','aarch64',) config_opts['dist'] = 'el7' # only useful for --resultdir variable subst config_opts['chroot_setup_cmd'] = 'install createrepo yum-utils bison byacc cscope ctags cvs diffstat doxygen flex gcc gcc-c++ gcc-gfortran gettext git indent intltool libtool patch patchutils rcs redhat-rpm-config rpm-build subversion swig systemtap sudo' config_opts['plugin_conf']['yum_cache_enable'] = False diff --git a/tools/buildconfig.py b/tools/buildconfig.py index a43fa92..1fae876 100755 --- a/tools/buildconfig.py +++ b/tools/buildconfig.py @@ -13,15 +13,20 @@ # limitations under the License. import ConfigParser +import platform from tools.statics import BUILD_CONFIG_PATH +def optionxform_arch(option): + return str(option).replace('#ARCH#', platform.machine()) + + class BuildConfigParser(ConfigParser.ConfigParser): def __init__(self, ini_file=BUILD_CONFIG_PATH): ConfigParser.ConfigParser.__init__(self) self.ini_file = ini_file - self.optionxform = str + self.optionxform = optionxform_arch self.read(self.ini_file) def items(self, section): # pylint: disable=arguments-differ -- 2.16.6