AArch64 (arm64) support for image building, mock

- mock:
  * extend config_opts['legal_host_arches'] to include 'aarch64';
  * parameterize RPM target architecture based on the output of
    $(uname -m) in create_mock_config.sh;
- Dockerfile-dib:
  * Bump base to centos:7.6.1810, but since centos:7.x.y tags are
    broken on arm64 [1], apply an arch-specific patch to the DIB
    Dockerfile which pins the FROM image to a specific sha256;
  * Use AArch64 altsig repository for openstack-queens packages;
  * Bump diskimage-builder to 2.26.1 to include arm64 critical fixes;
  * Add tools required for UEFI partition handling and other AArch64
    specific requirements to the arm64 container image (gdisk,
    dosfstools, util-linux, qemu-img);
- dib elements:
  * finalise.d/99-collect-rpm-info: AArch64 workaround for packages
    initially installed from 'updates' repo residing in 'base' repo
    after all distro/repo handling take place;
- create_golden_image.sh:
  * dib: Add additional element block-device-efi on AArch64 (only);
- build_step_create_install_cd.sh:
  * parameterize CentOS artifacts URI based on machine architecture;
  * skip isolinux artifacts download/handling for AArch64, since
    PC BIOS is x86 specific;
  * ISO(s) build: AArch64 only supports UEFI booting, skip isolinux;
  * AArch64: use legacy method of copying out the contents of "/boot"
    from the golden image since the AArch64 image has a more complex
    partition layout and would require mounting both "/boot" and ESP;
- create_rpmdata_in_docker.sh:
  * fix bash shebang since script uses bash-isms;
- tools/buildconfig.py:
  * add support for replacing the magic string '#ARCH#' with current
    platform architecture in option names;

Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
Co-authored-by: Valentin Radulescu <Valentin.Radulescu@enea.com>
Change-Id: I32cfdc2adc43fb703843383a1e94a807919f4e8c

diff --git a/build_images.sh b/build_images.sh
index bbaaf00..0d7968b 100755 (executable)
--- a/build_images.sh
+++ b/build_images.sh
@@ -54,6 +54,9 @@ source $scriptdir/lib.sh
 
 _initialize_work_dirs
 
+dib_patch="$scriptdir/docker-context/Dockerfile-dib.$(uname -m).patch"
+if [ -e "${dib_patch}" ]; then patch "$scriptdir/docker-context/Dockerfile-dib" "${dib_patch}"; fi
+
 docker build -f $scriptdir/docker-context/Dockerfile-dib -t dib $scriptdir/docker-context
 docker build -f $scriptdir/docker-context/Dockerfile-buildtools -t buildtools $scriptdir/docker-context
 

diff --git a/build_step_create_install_cd.sh b/build_step_create_install_cd.sh
index 78bafd3..40d3880 100755 (executable)
--- a/build_step_create_install_cd.sh
+++ b/build_step_create_install_cd.sh
@@ -30,9 +30,10 @@ mkdir -p $tmp
 rm -rf $iso_build_dir
 mkdir -p $iso_build_dir
 
+iso_arch="$(uname -m)"
 reposnap_base=$(_read_build_config DEFAULT centos_reposnap)
 release_version=$PRODUCT_RELEASE_LABEL
-reposnap_base_dir="${reposnap_base}/os/x86_64/"
+reposnap_base_dir="${reposnap_base}/os/${iso_arch}/"
 iso_image_label=$(_read_build_config DEFAULT iso_image_label)
 cd_efi_dir="${reposnap_base_dir}/EFI"
 cd_images_dir="${reposnap_base_dir}/images"
@@ -59,48 +60,63 @@ wget_dir ${cd_efi_dir}/
 wget_dir ${cd_images_dir}/
 rm -f images/boot.iso
 sync
-wget_dir ${cd_isolinux_dir}/
-chmod +w -R isolinux/ EFI/ images/
-
-if [ -e $scriptdir/isolinux/isolinux.cfg ]; then
-    cp $scriptdir/isolinux/isolinux.cfg isolinux/isolinux.cfg
-else
-    sed -i "s/^timeout.*/timeout 100/" isolinux/isolinux.cfg
-    sed -i "s/^ -  Press.*/Beginning the cloud installation process/" isolinux/boot.msg
-    sed -i "s/^#menu hidden/menu hidden/" isolinux/isolinux.cfg
-    sed -i "s/menu default//" isolinux/isolinux.cfg
-    sed -i "/^label linux/amenu default" isolinux/isolinux.cfg
-    sed -i "/append initrd/ s/$/ console=tty0 console=ttyS1,115200/" isolinux/isolinux.cfg
+chmod +w -R EFI/ images/
+# AArch64 does not support PC-BIOS, so skip all isolinux processing
+if [ "${iso_arch}" != 'aarch64' ]; then
+    wget_dir ${cd_isolinux_dir}/
+    chmod +w -R isolinux/
+
+    if [ -e $scriptdir/isolinux/isolinux.cfg ]; then
+        cp $scriptdir/isolinux/isolinux.cfg isolinux/isolinux.cfg
+    else
+        sed -i "s/^timeout.*/timeout 100/" isolinux/isolinux.cfg
+        sed -i "s/^ -  Press.*/Beginning the cloud installation process/" isolinux/boot.msg
+        sed -i "s/^#menu hidden/menu hidden/" isolinux/isolinux.cfg
+        sed -i "s/menu default//" isolinux/isolinux.cfg
+        sed -i "/^label linux/amenu default" isolinux/isolinux.cfg
+        sed -i "/append initrd/ s/$/ console=tty0 console=ttyS1,115200/" isolinux/isolinux.cfg
+    fi
+    cp -f $scriptdir/akraino_splash.png isolinux/splash.png
 fi
-cp -f $scriptdir/akraino_splash.png isolinux/splash.png
 
 popd
 
 pushd $tmp
 
  # Copy latest kernel and initrd-provisioning from boot dir
-qemu-img convert $input_image guest-image.raw
-myloop=$(sudo losetup -fP --show guest-image.raw)
-mkdir mnt
-sudo mount -o loop ${myloop}p1 mnt/
-sudo rsync -avA mnt/boot .
-sudo chown -R $(id -u):$(id -g) boot
-sudo umount mnt
-sudo losetup -d ${myloop}
-rm -f guest-image.raw
+if [ "${iso_arch}" != 'aarch64' ]; then
+    qemu-img convert $input_image guest-image.raw
+    myloop=$(sudo losetup -fP --show guest-image.raw)
+    mkdir mnt
+    sudo mount -o loop ${myloop}p1 mnt/
+    sudo rsync -avA mnt/boot .
+    sudo chown -R $(id -u):$(id -g) boot
+    sudo umount mnt
+    sudo losetup -d ${myloop}
+    rm -f guest-image.raw
+else
+    export LIBGUESTFS_BACKEND=direct
+    virt-copy-out -a $input_image /boot/ ./
+fi
 
 chmod u+w boot/
-rm -f $iso_build_dir/isolinux/vmlinuz $iso_build_dir/isolinux/initrd.img
 KVER=`ls -lrt boot/vmlinuz-* |grep -v rescue |tail -n1 |awk -F 'boot/vmlinuz-' '{print $2}'`
-cp -fp boot/vmlinuz-${KVER} $iso_build_dir/isolinux/vmlinuz
-cp -fp boot/initrd-provisioning.img $iso_build_dir/isolinux/initrd.img
+if [ "${iso_arch}" != 'aarch64' ]; then
+    rm -f $iso_build_dir/isolinux/vmlinuz $iso_build_dir/isolinux/initrd.img
+    cp -fp boot/vmlinuz-${KVER} $iso_build_dir/isolinux/vmlinuz
+    cp -fp boot/initrd-provisioning.img $iso_build_dir/isolinux/initrd.img
+fi
 rm -rf boot/
 
 echo "Generating boot iso"
+if [ "${iso_arch}" != 'aarch64' ]; then
+    bios_specific_args="-b isolinux/isolinux.bin -c isolinux/boot.cat \
+                        -no-emul-boot -boot-load-size 4 -boot-info-table"
+fi
 genisoimage  -U -r -v -T -J -joliet-long \
   -V "${release_version}" -A "${release_version}" -P ${iso_image_label} \
-  -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 \
-  -boot-info-table -eltorito-alt-boot -e images/efiboot.img -no-emul-boot \
+  ${bios_specific_args:-} \
+  -eltorito-alt-boot -e images/efiboot.img -no-emul-boot \
   -o boot.iso $iso_build_dir
 _publish_image $tmp/boot.iso $output_bootcd_path
 
@@ -112,10 +128,10 @@ mkdir -p $iso_build_dir/rpms
 echo "Generating product iso"
 genisoimage  -U -r -v -T -J -joliet-long \
   -V "${release_version}" -A "${release_version}" -P ${iso_image_label} \
-  -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 \
-  -boot-info-table -eltorito-alt-boot -e images/efiboot.img -no-emul-boot \
+  ${bios_specific_args:-} \
+  -eltorito-alt-boot -e images/efiboot.img -no-emul-boot \
   -o release.iso $iso_build_dir
-isohybrid $tmp/release.iso
+[ "${iso_arch}" = 'aarch64' ] || isohybrid $tmp/release.iso
 _publish_image $tmp/release.iso $output_image_path
 
 echo "Clean up to preserve workspace footprint"

diff --git a/build_step_golden_image.sh b/build_step_golden_image.sh
index ceb9a90..e409d35 100755 (executable)
--- a/build_step_golden_image.sh
+++ b/build_step_golden_image.sh
@@ -40,7 +40,7 @@ mkdir -p $(dirname $output_image_path)
 mv -f ${TMP_GOLDEN_IMAGE}.qcow2 $output_image_path
 
 input_dir=$WORKTMP/rpmdata
-mkdir $input_dir
+mkdir -p $input_dir
 cp -r \
   $BUILD_CONFIG_INI $RPMLISTS/rpm_info_installed $RPMLISTS/yum_info_installed $RPMLISTS/crypto_rpms.json $RPMLISTS/boms \
   $input_dir

diff --git a/create_golden_image.sh b/create_golden_image.sh
index 307da83..2a9d66a 100755 (executable)
--- a/create_golden_image.sh
+++ b/create_golden_image.sh
@@ -25,6 +25,7 @@ CENTOS_SNAP=${CENTOS_SNAP:-$(_read_build_config DEFAULT centos_reposnap)}
 
 BASE_IMAGE_NAME=`echo $BASE_IMAGE_URL | awk -F "/" '{print $NF}'`
 BASE_IMAGE_SIZE="8GiB"
+[ "$(uname -m)" != aarch64 ] || EXTRA_ELEMENTS="block-device-efi"
 
 wget_args=""
 [ -n "$GOLDEN_BASE_IMAGE_FETCH_USER" ] && wget_args="$wget_args --http-user=$GOLDEN_BASE_IMAGE_FETCH_USER"
@@ -55,6 +56,6 @@ DIB_DEBUG_TRACE=1 \
   DIB_YUM_REPO_CONF="${REPO_FILES}/repositories.repo ${REPO_FILES}/localrepo.repo" \
   DIB_LOCAL_IMAGE=$WORKTMP/base-img/$BASE_IMAGE_NAME \
   ELEMENTS_PATH=$scriptdir/dib_elements/ \
-  /usr/bin/disk-image-create --root-label img-rootfs --image-size $BASE_IMAGE_SIZE vm centos7 selinux-permissive myproduct -o $TMP_GOLDEN_IMAGE
+  /usr/bin/disk-image-create --root-label img-rootfs --image-size $BASE_IMAGE_SIZE vm centos7 selinux-permissive myproduct ${EXTRA_ELEMENTS:-} -o $TMP_GOLDEN_IMAGE
 
 rm -rf $WORKTMP/base-img

diff --git a/create_mock_config.sh b/create_mock_config.sh
index fd39fed..f05dfdc 100755 (executable)
--- a/create_mock_config.sh
+++ b/create_mock_config.sh
@@ -34,6 +34,7 @@ cp $LIBDIR/mock/site-defaults.cfg $output_mock_dir/
 mock_cfg=$output_mock_dir/mock.cfg
 sed -e "/#REPOSITORIES#/r $output_repo_files_dir/repositories.repo" $LIBDIR/mock/mock.cfg.template > $mock_cfg
 sed -i \
+  -e "s/#RPM_ARCH#/\"$(uname -m)\"/" \
   -e "s/#RPM_PACKAGER#/\"$(_read_build_config $config_ini rpm packager)\"/" \
   -e "s/#RPM_VENDOR#/\"$(_read_build_config $config_ini rpm vendor)\"/" \
   -e "s/#RPM_LICENSE#/\"$(_read_build_config $config_ini rpm license)\"/" \

diff --git a/create_rpmdata_in_docker.sh b/create_rpmdata_in_docker.sh
index 26b6159..82c7bcc 100755 (executable)
--- a/create_rpmdata_in_docker.sh
+++ b/create_rpmdata_in_docker.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # Copyright 2019 Nokia
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

diff --git a/dib_elements/myproduct/finalise.d/99-collect-rpm-info b/dib_elements/myproduct/finalise.d/99-collect-rpm-info
index f3cc6b8..439fe4b 100755 (executable)
--- a/dib_elements/myproduct/finalise.d/99-collect-rpm-info
+++ b/dib_elements/myproduct/finalise.d/99-collect-rpm-info
@@ -19,6 +19,16 @@ fi
 set -eu
 set -o pipefail
 
+# On AArch64, since the cloud image is very old, some packages are obsolete and
+# get upgraded from the 'updates' repository. However, after the whole distro
+# is upgraded and the repos are pointing to the latest repositories, the same
+# packages previously available in the 'updates' repo are now listed in the
+# 'base' repo, breaking our RPM availability check below.
+#
+# Work around this by forcing a reinstall of the problematic packages, which
+# will also update the 'From repo' fields.
+[ "$(uname -m)" != aarch64 ] || yum reinstall -y ncurses ncurses-base ncurses-libs
+
 yum info installed > /root/yum_info_installed
 rpm -qai --queryformat "Obsoletes   : [%{OBSOLETES},]\n" > /root/rpm_info_installed
 

diff --git a/docker-context/Dockerfile-dib b/docker-context/Dockerfile-dib
index 1a339f0..33b0e2b 100644 (file)
--- a/docker-context/Dockerfile-dib
+++ b/docker-context/Dockerfile-dib
@@ -14,12 +14,19 @@
 
 
 # Use newer than the CentOS stock diskimage-builder via "cloud" repo
-FROM centos:7.5.1804
+FROM centos:7.6.1810
 RUN \
-    yum-config-manager --add-repo http://mirror.centos.org/centos/7/cloud/x86_64/openstack-queens/ && \
-    yum install --nogpgcheck -y diskimage-builder \
+    if [ "$(uname -m)" = aarch64 ]; then \
+        REPO_URI=http://mirror.centos.org/altarch/7/cloud/aarch64/openstack-queens/; \
+        EXTRA_RPM="gdisk dosfstools util-linux"; \
+    else \
+        REPO_URI=http://mirror.centos.org/centos/7/cloud/x86_64/openstack-queens/; \
+    fi && \
+    yum-config-manager --add-repo $REPO_URI && \
+    yum install --nogpgcheck -y \
         git \
         python \
+        python-pip \
         wget \
         which \
         findutils \
@@ -28,7 +35,11 @@ RUN \
         parted \
         sudo \
         e2fsprogs \
-        xfsprogs
+        xfsprogs \
+        qemu-img \
+        ${EXTRA_RPM:-} && \
+    pip install diskimage-builder==2.26.1 && \
+    yum remove -y python-pip
 RUN find -name 99-selinux-fixfiles-restore -exec rm {} \;
 
 WORKDIR /work

diff --git a/docker-context/Dockerfile-dib.aarch64.patch b/docker-context/Dockerfile-dib.aarch64.patch
new file mode 100644 (file)
index 0000000..5af6d04
--- /dev/null
+++ b/docker-context/Dockerfile-dib.aarch64.patch
@@ -0,0 +1,13 @@
+diff --git a/docker-context/Dockerfile-dib b/docker-context/Dockerfile-dib
+index 038b64d..393f10f 100644
+--- a/docker-context/Dockerfile-dib
++++ b/docker-context/Dockerfile-dib
+@@ -14,7 +14,7 @@
+
+
+ # Use newer than the CentOS stock diskimage-builder via "cloud" repo
+-FROM centos:7.6.1810
++FROM centos@sha256:df89b0a0b42916b5b31b334fd52d3e396c226ad97dfe772848bdd6b00fb42bf0
+ RUN \
+     if [ "$(uname -m)" = aarch64 ]; then \
+         REPO_URI=http://mirror.centos.org/altarch/7/cloud/aarch64/openstack-queens/; \

diff --git a/mock/mock.cfg.template b/mock/mock.cfg.template
index c5b3496..26ce44e 100644 (file)
--- a/mock/mock.cfg.template
+++ b/mock/mock.cfg.template
@@ -15,8 +15,8 @@
 # Root name to be used for chroot and caching, must differ between products
 config_opts['root'] = 'akrainolite'
 
-config_opts['target_arch'] = 'x86_64'
-config_opts['legal_host_arches'] = ('x86_64',)
+config_opts['target_arch'] = #RPM_ARCH#
+config_opts['legal_host_arches'] = ('x86_64','aarch64',)
 config_opts['dist'] = 'el7'  # only useful for --resultdir variable subst
 config_opts['chroot_setup_cmd'] = 'install createrepo yum-utils bison byacc cscope ctags cvs diffstat doxygen flex gcc gcc-c++ gcc-gfortran gettext git indent intltool libtool patch patchutils rcs redhat-rpm-config rpm-build subversion swig systemtap sudo'
 config_opts['plugin_conf']['yum_cache_enable'] = False

diff --git a/tools/buildconfig.py b/tools/buildconfig.py
index a43fa92..1fae876 100755 (executable)
--- a/tools/buildconfig.py
+++ b/tools/buildconfig.py
@@ -13,15 +13,20 @@
 # limitations under the License.
 
 import ConfigParser
+import platform
 
 from tools.statics import BUILD_CONFIG_PATH
 
 
+def optionxform_arch(option):
+    return str(option).replace('#ARCH#', platform.machine())
+
+
 class BuildConfigParser(ConfigParser.ConfigParser):
     def __init__(self, ini_file=BUILD_CONFIG_PATH):
         ConfigParser.ConfigParser.__init__(self)
         self.ini_file = ini_file
-        self.optionxform = str
+        self.optionxform = optionxform_arch
         self.read(self.ini_file)
 
     def items(self, section):  # pylint: disable=arguments-differ