From: Alexandru Avadanii Date: Wed, 21 Aug 2019 13:24:09 +0000 (+0200) Subject: ta: multiarch build: ISO, python-peewee RPM X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=commitdiff_plain;ds=inline;h=c109904304d487c973b3bcfab90425661826a401;hp=605a84f23c42ebe91add58935901d4ac2ec3bfa9;p=ci-management.git ta: multiarch build: ISO, python-peewee RPM Enable both amd64 and arm64 builds for python-peewee, which will later be extended to all arch-dependent packages in TA. To accomodate non-x86 ISO builds (triggered on RPM changes merge): - rename artifacts to include the machine architecture in their names (e.g. bootcd.aarch64.iso, bootcd.aarch64.iso.md5 etc.), only applicable if $(uname -m) is not 'x86_64' so x86 artifact names remain unchanged (e.g. bootcd.iso); - akraino-project-stream-type-ta: new 'arch' job parameter, defaulting to 'amd64', which will allow selecting the proper ISO build job (for arm64 TA RPM merge jobs, 'arch' should be set to 'arm64', while for amd64 no action is required); - create arch-specific ISO build jobs (ta-ci-build-{amd64,arm64}) and let the TA RPM merge jobs trigger the appropiate one; While at it, fix some related issues: - adjust ta-rpm-deploy.sh to publish artifacts for the current machine architecture (x86_64 or aarch64); - explicitly set shell to bash where needed, e.g. where `pushd` bash builtin is used (otherwise it will fail on systems using sh as default shell); - cleanup git repository destination before clone to allow re-running the jobs on non-volatile slaves; - extend dependency package installation for Debian systems, not only RHEL; Depends-on: I32cfdc2adc43fb703843383a1e94a807919f4e8c Depends-on: I3a940e2e0ece6febc9fc90d6bbbc6ff8ef212e14 Change-Id: If3adcae723aa4932fc095cc71debdef2d28840ff Signed-off-by: Alexandru Avadanii --- diff --git a/jjb/akraino-templates/akraino-ta-common-macros.yaml b/jjb/akraino-templates/akraino-ta-common-macros.yaml index 7006eac..af0b769 100644 --- a/jjb/akraino-templates/akraino-ta-common-macros.yaml +++ b/jjb/akraino-templates/akraino-ta-common-macros.yaml @@ -89,6 +89,7 @@ - lf-pip-install: pip-packages: tox - shell: |2- + #!/bin/bash -ex source /tmp/v/tox/bin/activate for f in $(find $GERRIT_PROJECT -name tox.ini); do pushd $(dirname $f) @@ -100,10 +101,32 @@ name: ta-builder-rpm builders: - shell: |2- + #!/bin/bash -ex # Skip RPM build in case no *.spec file is found within the {project-to-build} if [ -n "$(find {project-to-build} -name '*.spec')" ] then - sudo yum install -y createrepo + if apt --version 2>1 >/dev/null; then + sudo groupadd mock || true + sudo usermod -aG mock $USER + sudo apt update + # Explicitly install mock dependencies + sudo apt install -y createrepo alien systemd-container python-pip \ + python-decoratortools yum-utils usermode pigz + # We need a specific version of mock (1.4.14) and devtools + # not available in Ubuntu/Debian repos, so fetch them from RHEL mirrors + wget http://mirror.centos.org/centos/7.6.1810/os/x86_64/Packages/rpmdevtools-8.3-5.el7.noarch.rpm \ + https://kojipkgs.fedoraproject.org/packages/mock/1.4.14/2.el7/noarch/mock-1.4.14-2.el7.noarch.rpm + sudo alien -d *.rpm + sudo dpkg -i *.deb + rm -rf *.rpm *.deb + # RHEL tools hardcode the full path for certain distro tools + sudo ln -sf /bin/tar /usr/bin/gtar + sudo ln -sf /bin/machinectl /bin/df /usr/bin/ + # Some mock dependencies are only available via pip + sudo pip install distro pyroute2 + else + sudo yum install -y createrepo + fi ta/build-tools/build_rpms.sh \ -m ta/manifest \ -r ta/rpmbuilder \ @@ -116,6 +139,8 @@ name: ta-builder-scm builders: - shell: |2- + #!/bin/bash -ex + rm -rf {project} git clone {git-url}/{project} {project} pushd {project} if echo {ref} | grep -q "refs/"; then @@ -230,8 +255,18 @@ name: ta-builder-install-docker builders: - shell: |2- - sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo - sudo yum -y install docker-ce docker-ce-cli containerd.io + #!/bin/bash -ex + if apt --version 2>1 >/dev/null; then + sudo apt update + sudo apt install -y apt-transport-https ca-certificates curl software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository -y "deb http://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + sudo apt update + sudo apt install -y docker-ce + else + sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + sudo yum -y install docker-ce docker-ce-cli containerd.io + fi sudo mkdir -p /etc/docker/ echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json sudo systemctl start docker @@ -243,7 +278,13 @@ name: ta-builder-install-build-tools builders: - shell: |2- - sudo yum -y install createrepo libguestfs-tools-c jq + #!/bin/bash -ex + if apt --version 2>1 >/dev/null; then + sudo apt update + sudo apt install -y createrepo libguestfs-tools jq + else + sudo yum -y install createrepo libguestfs-tools-c jq + fi sudo systemctl start libvirtd systemctl status libvirtd @@ -307,6 +348,6 @@ publishers: - lf-infra-publish - trigger-parameterized-builds: - - project: 'ta-ci-build' + - project: 'ta-ci-build-{arch}' condition: UNSTABLE_OR_BETTER trigger-with-no-params: true diff --git a/jjb/akraino-templates/akraino-ta-common-templates.yaml b/jjb/akraino-templates/akraino-ta-common-templates.yaml index 123fda6..7151f5e 100644 --- a/jjb/akraino-templates/akraino-ta-common-templates.yaml +++ b/jjb/akraino-templates/akraino-ta-common-templates.yaml @@ -18,6 +18,7 @@ project-type: freestyle submodule-recursive: true node: '{build-node}' + arch: amd64 properties: - lf-infra-properties: @@ -66,7 +67,8 @@ git-url: '{git-url}' publishers: - - ta-publisher-job-{project-type} + - ta-publisher-job-{project-type}: + arch: '{arch}' - job-template: id: akraino-project-stream-type-ta-rpmbuilder @@ -204,7 +206,7 @@ - job-template: id: akraino-project-release-ta-build - name: 'ta-ci-build' + name: '{name}' archive-artifacts: > work/**/*.spec work/**/*.repo diff --git a/jjb/shell/ta-iso-deploy.sh b/jjb/shell/ta-iso-deploy.sh index cbe86b9..c9d5559 100644 --- a/jjb/shell/ta-iso-deploy.sh +++ b/jjb/shell/ta-iso-deploy.sh @@ -27,6 +27,13 @@ nexus_repo_url="$NEXUS_URL/content/repositories/$NEXUS_REPO" mkdir -p "$upload_dir1" mkdir -p "$upload_dir2" +if [ "$(uname -m)" != 'x86_64' ]; then + # On non-x86 architecture, rename the artifacts appropiately + pushd "$WORKSPACE/work/results/images/" + rename "s/\./.$(uname -m)./" *.* + popd +fi + cp "$WORKSPACE/work/results/images/"* "$upload_dir1" cp "$WORKSPACE/work/results/images/"* "$upload_dir2" diff --git a/jjb/shell/ta-rpm-deploy.sh b/jjb/shell/ta-rpm-deploy.sh index a9c03e3..aa46e7a 100644 --- a/jjb/shell/ta-rpm-deploy.sh +++ b/jjb/shell/ta-rpm-deploy.sh @@ -20,7 +20,7 @@ NEXUS_REPO=rpm.snapshots release_path=TA/release-1 repo_dir="$WORKSPACE/work/nexus/$NEXUS_REPO" -x86_dir="$repo_dir/$release_path/rpms/x86_64" +arch_dir="$repo_dir/$release_path/rpms/$(uname -m)" sources_dir="$repo_dir/$release_path/rpms/Sources" nexus_repo_url="$ALT_NEXUS_URL/repository/$NEXUS_REPO" results_dir="$WORKSPACE/work/results" @@ -29,20 +29,20 @@ repo_name=`echo $WORKSPACE | awk -F '/' '{print $4}' | cut -d '-' -f2- | sed 's| #Creating dirs to move duplicate RPMs/SRPMs to avoid re-upload and copy the changed RPMs/SRPMs mkdir "$results_dir/repo/duplicates" mkdir "$results_dir/src_repo/duplicates" -mkdir -p "$x86_dir" +mkdir -p "$arch_dir" mkdir -p "$sources_dir" #List all RPMs available in Nexus, move the duplicates and copy the changed ones for artifact in \ `ls $results_dir/repo/*.rpm` do - if curl --head --fail $nexus_repo_url/$release_path/rpms/x86_64/$(basename $artifact) + if curl --head --fail $nexus_repo_url/$release_path/rpms/$(uname -m)/$(basename $artifact) then echo "RPM - $(basename $artifact) already available in Nexus" mv $results_dir/repo/$(basename $artifact) $results_dir/repo/duplicates/ else echo "RPM - $(basename $artifact) is not available in Nexus. Will be uploaded" - cp $results_dir/repo/$(basename $artifact) $x86_dir + cp $results_dir/repo/$(basename $artifact) $arch_dir fi done diff --git a/jjb/ta/ci-build.yaml b/jjb/ta/ci-build.yaml index 62399a5..dab1436 100644 --- a/jjb/ta/ci-build.yaml +++ b/jjb/ta/ci-build.yaml @@ -1,5 +1,5 @@ - project: - name: ta-ci-build + name: ta-ci-build-amd64 project: none project-name: none build-node: centos7-builder-8c-32g @@ -7,3 +7,13 @@ build-timeout: 120 jobs: - akraino-project-release-ta-build + +- project: + name: ta-ci-build-arm64 + project: none + project-name: none + build-node: aarch64_dev + stream: master + build-timeout: 120 + jobs: + - akraino-project-release-ta-build diff --git a/jjb/ta/python-peewee.yaml b/jjb/ta/python-peewee.yaml index 576f6aa..514e4e4 100644 --- a/jjb/ta/python-peewee.yaml +++ b/jjb/ta/python-peewee.yaml @@ -1,7 +1,7 @@ - project: - name: ta-python-peewee + name: ta-python-peewee-amd64 project: ta/python-peewee - project-name: ta-python-peewee + project-name: ta-python-peewee-amd64 project-type: - merge - verify @@ -9,3 +9,16 @@ stream: master jobs: - akraino-project-stream-type-ta + +- project: + name: ta-python-peewee-arm64 + project: ta/python-peewee + project-name: ta-python-peewee-arm64 + project-type: + - merge + - verify + build-node: aarch64_dev + arch: arm64 + stream: master + jobs: + - akraino-project-stream-type-ta