From: Saku Chydenius Date: Mon, 18 Mar 2019 07:08:45 +0000 (+0200) Subject: Add initial code X-Git-Url: https://gerrit.akraino.org/r/gitweb?p=ta%2Fbuild-tools.git;a=commitdiff_plain;h=4ded4f2a805e9447be90751d7d4fb7e11552e545 Add initial code Change-Id: I72d87e74c74defc97bd956c3b23de9a4e01acb28 Signed-off-by: Saku Chydenius --- diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..f3a5cd4 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + .tox/* diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e1d93d9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +.idea +*.pyc +.coverage +.coverage.* +.pytest-cache/ +.pytest-tmpdir/ +.tox/ +htmlcov/ +junit.xml +tmp_yum.conf diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000..0c6c889 --- /dev/null +++ b/.gitreview @@ -0,0 +1,6 @@ +[gerrit] +host=gerrit.akraino.org +port=29418 +project=ta/build-tools +defaultremote=origin + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README b/README new file mode 100644 index 0000000..4efc9eb --- /dev/null +++ b/README @@ -0,0 +1,8 @@ +# Unit tests and static analysis + +Execute with: + $ tox + +Execute only subset with verbose diff on assertion errors: + $ tox -e py27 -- -vv tools/script/process_rpmdata_test.py::test_components + diff --git a/akraino_splash.png b/akraino_splash.png new file mode 100644 index 0000000..ed3e9bd Binary files /dev/null and b/akraino_splash.png differ diff --git a/build_step_create_install_cd.sh b/build_step_create_install_cd.sh new file mode 100755 index 0000000..40de1cc --- /dev/null +++ b/build_step_create_install_cd.sh @@ -0,0 +1,133 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh +_read_manifest_vars + +tmp=$WORKTMP/install_cd +iso_build_dir=$tmp/build + +input_image="$WORKTMP/goldenimage/${GOLDEN_IMAGE_NAME}" +output_image_path="$1" +[[ $output_image_path =~ ^/ ]] || output_image_path=$(pwd)/$output_image_path +output_bootcd_path="$2" +[[ $output_bootcd_path =~ ^/ ]] || output_bootcd_path=$(pwd)/$output_bootcd_path +mkdir -p $tmp +rm -rf $iso_build_dir +mkdir -p $iso_build_dir + +reposnap_base=$(_read_build_config DEFAULT centos_reposnap_base) +release_version=$PRODUCT_RELEASE_LABEL +reposnap_base_dir="${reposnap_base}/os/x86_64/" +iso_image_label=$(_read_build_config DEFAULT iso_image_label) +cd_efi_dir="${reposnap_base_dir}/EFI" +cd_images_dir="${reposnap_base_dir}/images" +cd_isolinux_dir="${reposnap_base_dir}/isolinux" + +remove_extra_slashes_from_url() { + echo $1 | sed -re 's#([^:])//+#\1/#g' +} + +get_nexus() { + $scriptdir/nexus3_dl.sh \ + $nexus_url \ + $(basename $nexus_reposnaps) \ + ${reposnap_base#$nexus_reposnaps/}/os/x86_64 $@ +} + +wget_dir() { + local url=$1 + echo $url | grep -q /$ || _abort "wget path '$url' must end with slash for recursive wget" + # if any extra slashes within path, it messes up the --cut-dirs count + url=$(remove_extra_slashes_from_url $url) + # count cut length in case url depth changes + cut_dirs=$(echo $url | sed -re 's|.*://[^/]+/(.+)|\1|' -e 's|/$||' | grep -o / | wc -l) + wget -N -r --no-host-directories --no-verbose --cut-dirs=${cut_dirs} --reject index.html* --no-parent $url +} + +pushd $iso_build_dir + +# Get files needed for generating CD image. +if echo $reposnap_base_dir | grep -E "https?://nexus3"; then + nexus_url=$(_read_build_config DEFAULT nexus_url) + nexus_reposnaps=$(_read_build_config DEFAULT nexus_reposnaps) + get_nexus "EFI/BOOT" "EFI/BOOT/fonts" + get_nexus "images:*efiboot.img" "images/pxeboot" + get_nexus "isolinux" +else + wget_dir ${cd_efi_dir}/ + wget_dir ${cd_images_dir}/ + rm -rf images/boot.iso + sync + wget_dir ${cd_isolinux_dir}/ +fi +chmod +w -R isolinux/ EFI/ images/ + +if [ -e $scriptdir/isolinux/isolinux.cfg ]; then + cp $scriptdir/isolinux/isolinux.cfg isolinux/isolinux.cfg +else + sed -i "s/^timeout.*/timeout 100/" isolinux/isolinux.cfg + sed -i "s/^ - Press.*/Beginning the cloud installation process/" isolinux/boot.msg + sed -i "s/^#menu hidden/menu hidden/" isolinux/isolinux.cfg + sed -i "s/menu default//" isolinux/isolinux.cfg + sed -i "/^label linux/amenu default" isolinux/isolinux.cfg + sed -i "/append initrd/ s/$/ console=tty0 console=ttyS1,115200/" isolinux/isolinux.cfg +fi +cp -f $scriptdir/akraino_splash.png isolinux/splash.png + +popd + +pushd $tmp + + # Copy latest kernel and initrd-provisioning from boot dir +virt-copy-out -a $input_image /boot/ ./ +chmod u+w boot/ +rm -f $iso_build_dir/isolinux/vmlinuz $iso_build_dir/isolinux/initrd.img +KVER=`ls -lrt boot/vmlinuz-* |grep -v rescue |tail -n1 |awk -F 'boot/vmlinuz-' '{print $2}'` +cp -fp boot/vmlinuz-${KVER} $iso_build_dir/isolinux/vmlinuz +cp -fp boot/initrd-provisioning.img $iso_build_dir/isolinux/initrd.img +rm -rf boot/ + +echo "Generating boot iso" +_run_cmd genisoimage -U -r -v -T -J -joliet-long \ + -V "${release_version}" -A "${release_version}" -P ${iso_image_label} \ + -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 \ + -boot-info-table -eltorito-alt-boot -e images/efiboot.img -no-emul-boot \ + -o boot.iso $iso_build_dir +_publish_image $tmp/boot.iso $output_bootcd_path + +cp -f ${input_image} $iso_build_dir/ + +# Keep the placeholder +mkdir -p $iso_build_dir/rpms + +echo "Generating product iso" +_run_cmd genisoimage -U -r -v -T -J -joliet-long \ + -V "${release_version}" -A "${release_version}" -P ${iso_image_label} \ + -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 \ + -boot-info-table -eltorito-alt-boot -e images/efiboot.img -no-emul-boot \ + -o release.iso $iso_build_dir +_run_cmd isohybrid $tmp/release.iso +_publish_image $tmp/release.iso $output_image_path + +echo "Clean up to preserve workspace footprint" +rm -f $iso_build_dir/$(basename ${input_image}) +rm -rf $iso_build_dir/rpms + +popd diff --git a/build_step_create_localrepo.sh b/build_step_create_localrepo.sh new file mode 100755 index 0000000..4bb22a0 --- /dev/null +++ b/build_step_create_localrepo.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh + +_run_cmd_as_step _create_localrepo diff --git a/build_step_create_rpms.sh b/build_step_create_rpms.sh new file mode 100755 index 0000000..3d45388 --- /dev/null +++ b/build_step_create_rpms.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh + +_get_projects_all() { repo list -g torpm -n | sort; } + +_get_project_dirs() { + for i in $@; do + repo info -l $i | grep "^Mount path: " | awk '{print $3}' | tr '\n' ' ' + done +} + +build_rpms() +{ + local work=$1 + shift + rm -rf $work + mkdir -p $work + + CENTOS_SOURCES="$(_read_build_config rpm centos_sources)" \ + $RPM_BUILDER/makebuild.py \ + -m $RPM_BUILDER_SETTINGS \ + -w $work \ + $@ #-v --nowipe +} + +# build one or all projects +if [ -n "$1" ]; then + projects_to_build=$@ +else + projects_to_build="$MANIFEST_PATH $(_get_project_dirs $(_get_projects_all))" + _run_cmd $LIBDIR/prepare_manifest.sh +fi + +rpmwork=$WORKTMP/rpms +build_rpms $rpmwork $projects_to_build +_add_rpms_to_repos_from_workdir $rpmwork diff --git a/build_step_create_yum_repo_files.sh b/build_step_create_yum_repo_files.sh new file mode 100755 index 0000000..c08ae0c --- /dev/null +++ b/build_step_create_yum_repo_files.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh + +config_ini=${1:-$BUILD_CONFIG_INI} +output_dir=${2:-$REPO_FILES} +[ -f "$config_ini" ] || _abort "Config INI '$config_ini' not found" + +gen() { + PYTHONPATH=$LIBDIR python $LIBDIR/tools/script/generate_repo_files.py \ + --config-ini $config_ini \ + --output-dir $output_dir \ + $1 +} + +mkdir -p $output_dir + +gen localrepo +gen repositories +gen baseimage-repositories diff --git a/build_step_golden_image.sh b/build_step_golden_image.sh new file mode 100755 index 0000000..2dc1a37 --- /dev/null +++ b/build_step_golden_image.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh + +output_image_path="$1" +[[ $output_image_path =~ ^/ ]] || output_image_path=$(pwd)/$output_image_path +rpm_info_output_dir=$2 +[[ $rpm_info_output_dir =~ ^/ ]] || rpm_info_output_dir=$(pwd)/$rpm_info_output_dir + +docker_dib_image=dib:2.0 +_load_docker_image $docker_dib_image + +docker run \ + --rm \ + --privileged \ + -v /dev:/dev \ + -v $WORK:/work \ + $docker_dib_image \ + $(realpath --relative-to $WORK $scriptdir)/create_golden_image.sh + +_publish_image ${TMP_GOLDEN_IMAGE}.qcow2 $output_image_path/${GOLDEN_IMAGE_NAME} + +input_dir=$WORKTMP/rpmdata +mkdir $input_dir +cp -r \ + $BUILD_CONFIG_INI $RPMLISTS/rpm_info_installed $RPMLISTS/yum_info_installed $RPMLISTS/crypto_rpms.json $RPMLISTS/boms \ + $input_dir + +$LIBDIR/create_rpmdata_in_docker.sh \ + $input_dir \ + $RPMLISTS \ + -v + diff --git a/build_step_prepare.sh b/build_step_prepare.sh new file mode 100755 index 0000000..57d66cc --- /dev/null +++ b/build_step_prepare.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh +_read_manifest_vars + +_run_cmd_as_step _initialize_work_dirs + +_run_cmd_as_step $LIBDIR/repo_summary.sh + +_run_cmd_as_step $LIBDIR/create_mock_config.sh diff --git a/create_golden_image.sh b/create_golden_image.sh new file mode 100755 index 0000000..19dad38 --- /dev/null +++ b/create_golden_image.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e +set -o pipefail + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh + +BASE_IMAGE_URL=${BASE_IMAGE_URL:-$(_read_build_config DEFAULT base_image)} +CENTOS_SNAP=${CENTOS_SNAP:-$(_read_build_config DEFAULT centos_reposnap)} + +BASE_IMAGE_NAME=`echo $BASE_IMAGE_URL | awk -F "/" '{print $NF}'` +BASE_IMAGE_SIZE="8GiB" + +wget_args="" +[ -n "$GOLDEN_BASE_IMAGE_FETCH_USER" ] && wget_args="$wget_args --http-user=$GOLDEN_BASE_IMAGE_FETCH_USER" +[ -n "$GOLDEN_BASE_IMAGE_FETCH_PASSWORD" ] && wget_args="$wget_args --http-password=$GOLDEN_BASE_IMAGE_FETCH_PASSWORD" + +fetch_image() { + sourceurl=$1 + echo "Download $sourceurl to $WORKTMP/base-img" + mkdir -pv $WORKTMP/base-img + pushd $WORKTMP/base-img + _run_cmd wget --no-check-certificate --no-verbose -N \ + --auth-no-challenge $wget_args \ + $sourceurl + popd +} + +fetch_image $BASE_IMAGE_URL +cp $MANIFEST_PATH/packages.yaml $scriptdir/dib_elements/myproduct/package-installs.yaml + +DIB_DEBUG_TRACE=1 \ + FS_TYPE=xfs \ + PACKAGES_TO_INSTALL="$(_get_package_list install)" \ + PACKAGES_TO_UNINSTALL="$(_get_package_list uninstall)" \ + DIB_RPMLISTS=$RPMLISTS \ + DIB_CHECKSUM=$CHECKSUM_DIR \ + DIB_LOCAL_REPO=$REPO_DIR \ + DIB_DISTRIBUTION_MIRROR=$CENTOS_SNAP \ + DIB_YUM_REPO_CONF="${REPO_FILES}/repositories.repo ${REPO_FILES}/localrepo.repo" \ + DIB_LOCAL_IMAGE=$WORKTMP/base-img/$BASE_IMAGE_NAME \ + ELEMENTS_PATH=$scriptdir/dib_elements/ \ + /usr/bin/disk-image-create --root-label img-rootfs --image-size $BASE_IMAGE_SIZE vm centos7 selinux-permissive myproduct -o $TMP_GOLDEN_IMAGE + +rm -rf $WORKTMP/base-img diff --git a/create_mock_config.sh b/create_mock_config.sh new file mode 100755 index 0000000..701cbc1 --- /dev/null +++ b/create_mock_config.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh + +config_ini=${1:-$BUILD_CONFIG_INI} +output_repo_files_dir=${2:-$REPO_FILES} +output_mock_dir=${2:-$(dirname $RPM_BUILDER_SETTINGS)} +[ -f "$config_ini" ] || _abort "Config INI '$config_ini' not found" + +# Create .repo files +$LIBDIR/build_step_create_yum_repo_files.sh $config_ini $output_repo_files_dir + +# Create mock config +mkdir -p $output_mock_dir +cp $LIBDIR/mock/logging.ini $output_mock_dir/ +cp $LIBDIR/mock/site-defaults.cfg $output_mock_dir/ +mock_cfg=$output_mock_dir/mock.cfg +sed -e "/#REPOSITORIES#/r $output_repo_files_dir/repositories.repo" $LIBDIR/mock/mock.cfg.template > $mock_cfg +sed -i \ + -e "s/#RPM_PACKAGER#/\"$(_read_build_config $config_ini rpm packager)\"/" \ + -e "s/#RPM_VENDOR#/\"$(_read_build_config $config_ini rpm vendor)\"/" \ + -e "s/#RPM_LICENSE#/\"$(_read_build_config $config_ini rpm license)\"/" \ + -e "s/#RPM_RELEASE_ID#/\"$(_read_build_config $config_ini rpm release_id)\"/" \ + $mock_cfg + +docker_sock=/var/run/docker.sock +if [ -S "$docker_sock" ]; then + sed -i "/#ADDITIONAL_CONFIG_OPTS#/a config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('$docker_sock', '$docker_sock'))" $mock_cfg +fi +if [ -n "$DOCKER_HOST" ]; then + sed -i "/#ADDITIONAL_CONFIG_OPTS#/a config_opts['environment']['DOCKER_HOST'] = '${DOCKER_HOST}:2375'" $mock_cfg +fi + +# HACK ?? +# include kernels in order for yum development group to install +sed -i -e 's/exclude=kernel/#exclude=kernel/' $mock_cfg + +echo "Wrote mock configuration to: $output_mock_dir" diff --git a/create_rpmdata_in_docker.sh b/create_rpmdata_in_docker.sh new file mode 100755 index 0000000..84d2472 --- /dev/null +++ b/create_rpmdata_in_docker.sh @@ -0,0 +1,71 @@ +#!/bin/sh +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh + +docker_image=buildtools:2.0 +_load_docker_image $docker_image + +function _resolve_abs_path() { + if ! echo $1 | grep -q "^/"; then + echo $(pwd)/$1 + else + echo $1 + fi +} + +input_dir=$(_resolve_abs_path ${1:?ERROR, please give input dir as argument}) +output_dir=$(_resolve_abs_path ${2:?ERROR, please give output dir as argument}) +shift; shift + +# Run! +input_mp=/input +output_mp=/output +docker run \ + --rm \ + -e PYTHONPATH=/work \ + -e BUILD_URL -e JENKINS_USERNAME -e JENKINS_TOKEN -e WORKSPACE \ + -v $scriptdir:/work \ + -v $input_dir:$input_mp \ + -v $output_dir:$output_mp \ + -w /work \ + $docker_image \ + python tools/script/create_rpm_data.py \ + --build-config-path $input_mp/build_config.ini \ + --yum-info-path $input_mp/yum_info_installed \ + --rpm-info-path $input_mp/rpm_info_installed \ + --crypto-info-path $input_mp/crypto_rpms.json \ + --boms-path $input_mp/boms \ + --output-json $output_mp/rpmdata.json \ + --output-csv $output_mp/rpmdata.csv \ + --output-ms-csv $output_mp/rpmdata-ms.csv \ + --output-rpmlist $output_mp/rpmlist \ + $* + +docker run \ + --rm \ + -e PYTHONPATH=/work \ + -v $scriptdir:/work \ + -v $output_dir:$output_mp \ + -w /work \ + $docker_image \ + python tools/script/process_rpmdata.py \ + --rpmdata-path $output_mp/rpmdata.json \ + --output-components $output_mp/components.json \ + --output-components-csv $output_mp/components-ms.csv \ + diff --git a/dib_elements/myproduct/cleanup.d/50-copy-build-details-out b/dib_elements/myproduct/cleanup.d/50-copy-build-details-out new file mode 100755 index 0000000..7732c3c --- /dev/null +++ b/dib_elements/myproduct/cleanup.d/50-copy-build-details-out @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +[ -n "$TARGET_ROOT" ] +mv $TARGET_ROOT/root/yum_info_installed $DIB_RPMLISTS +mv $TARGET_ROOT/root/rpm_info_installed $DIB_RPMLISTS +mv $TARGET_ROOT/root/boms $DIB_RPMLISTS +mv $TARGET_ROOT/root/binary_checksum.md5 $DIB_CHECKSUM +mv $TARGET_ROOT/root/binary_checksum.sha256 $DIB_CHECKSUM +mv $TARGET_ROOT/crypto_rpms.json $DIB_RPMLISTS diff --git a/dib_elements/myproduct/extra-data.d/01-copy-extra-data b/dib_elements/myproduct/extra-data.d/01-copy-extra-data new file mode 100755 index 0000000..175af75 --- /dev/null +++ b/dib_elements/myproduct/extra-data.d/01-copy-extra-data @@ -0,0 +1,20 @@ +#!/bin/sh +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +SCRIPT_DIR="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" + +cp -rf ${SCRIPT_DIR}/* $TMP_HOOKS_PATH/ diff --git a/dib_elements/myproduct/extra-data.d/collect_ecc.py b/dib_elements/myproduct/extra-data.d/collect_ecc.py new file mode 100644 index 0000000..56f0d2a --- /dev/null +++ b/dib_elements/myproduct/extra-data.d/collect_ecc.py @@ -0,0 +1,128 @@ +#!/usr/bin/python +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +import json +import sys + +# following libs are found from openssl, libgcrypto, openssh with rpm -qR +# in the future, the libs need to be updated with exact version number + +input_capabilities = [] +output_rpms = [] +input_packages = [ + 'openssl', + 'openssh', + 'p11-kit', + 'openssl-libs', + 'libgcrypt', + 'libgcrypt.so', + 'python2-cryptography', + 'sshpass', + 'm2crypto', + 'erlang-crypto', + 'openssh-clients', + 'python2-passlib', + 'python-paramiko', + 'python-keyring', + 'python2-asn1crypto', + 'python2-cryptography', + 'python2-pyasn1', + 'xstatic-jsencrypt-common', + 'krb5-libs' + ] +crypto_capabilities = [ + 'rtld(', + 'libcrypt.so' + ] + + +def output_rpm_command(lib, command): + command.append(lib) + try: + output = subprocess.check_output(command) + except: + output = "" + return output + + +def build_capability_list_dynamically(): + map_dict = {} + command = ['rpm', + '-qa', + '--queryformat', + '[%{=NAME}-%{VERSION}-%{RELEASE}.%{ARCH} %{PROVIDES}\n]'] + output = subprocess.check_output(command) + for item in output.splitlines(): + items = item.split(' ') + if items: + if items[0] in map_dict: + capa_list = map_dict[items[0]] + capa_list.append(items[1]) + map_dict[items[0]] = capa_list + else: + map_dict[items[0]] = [items[1]] + for rpms, caps in map_dict.items(): + for cap in caps: + for item in crypto_capabilities: + if cap.startswith(item): + if cap not in input_capabilities: + input_capabilities.append(cap) + command = ['rpm', '-q', '--whatprovides'] + output = output_rpm_command(cap, command) + name = output.strip('\n') + result = filter(lambda rpm: rpm['name'] == name, output_rpms) + if not result: + output_rpms.append({'name': name, 'provides': [cap]}) + else: + sources_list = result[0]['provides'] + if cap not in sources_list: + sources_list.append(cap) + + +def capability_dependencies_with_whatrequires(): + for capability in input_capabilities: + command = ['rpm', '-q', '--whatrequires'] + output = output_rpm_command(capability, command) + if output: + for item in output.splitlines(): + result = filter(lambda rpm: rpm['name'] == item, output_rpms) + if not result: + output_rpms.append({'name': item, 'requires': [capability]}) + else: + sources_list = result[0]['requires'] + if capability not in sources_list: + sources_list.append(capability) + + +def package_dependencies_with_provides(): + for package in input_packages: + command = ['rpm', '-q'] + name = output_rpm_command(package, command) + name = name.strip('\n') + output_rpms.append({'name': name, 'requires': []}) + command = ['rpm', '-ql', '--provides'] + output = output_rpm_command(package, command) + for item in output.splitlines(): + input_capabilities.append(item) + + +if __name__ == '__main__': + build_capability_list_dynamically() + package_dependencies_with_provides() + capability_dependencies_with_whatrequires() + rpms_json = json.dumps(output_rpms, sort_keys=True, indent=4) + with open(sys.argv[1], "w") as f: + f.write(rpms_json) diff --git a/dib_elements/myproduct/finalise.d/01-remove-old-kernels b/dib_elements/myproduct/finalise.d/01-remove-old-kernels new file mode 100755 index 0000000..c207fb6 --- /dev/null +++ b/dib_elements/myproduct/finalise.d/01-remove-old-kernels @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +rpm -qa kernel | sort -V | head -n -1 | xargs rpm -e || true + diff --git a/dib_elements/myproduct/finalise.d/99-collect-rpm-info b/dib_elements/myproduct/finalise.d/99-collect-rpm-info new file mode 100755 index 0000000..f3cc6b8 --- /dev/null +++ b/dib_elements/myproduct/finalise.d/99-collect-rpm-info @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +yum info installed > /root/yum_info_installed +rpm -qai --queryformat "Obsoletes : [%{OBSOLETES},]\n" > /root/rpm_info_installed + +# Collect bills of material, RPM subcomponent lists +boms_dir=/root/boms +mkdir $boms_dir +for bom_path in $(rpm -qal | egrep "/[^/]+-bom.json$" || :); do + cp $bom_path $boms_dir/$(rpm -qf $bom_path) +done diff --git a/dib_elements/myproduct/finalise.d/99-create-bonding-soft-dep b/dib_elements/myproduct/finalise.d/99-create-bonding-soft-dep new file mode 100755 index 0000000..f1dc245 --- /dev/null +++ b/dib_elements/myproduct/finalise.d/99-create-bonding-soft-dep @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +echo "softdep bonding pre: mlx5_core ixgbe" > /etc/modprobe.d/bonding.conf diff --git a/dib_elements/myproduct/finalise.d/99-fix-grub-console b/dib_elements/myproduct/finalise.d/99-fix-grub-console new file mode 100755 index 0000000..951e39f --- /dev/null +++ b/dib_elements/myproduct/finalise.d/99-fix-grub-console @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +sed -i -e "s/console=ttyS0,115200/console=ttyS1,115200/g" /etc/default/grub +grub2-mkconfig -o /boot/grub2/grub.cfg diff --git a/dib_elements/myproduct/finalise.d/99-generate-binary-checksum b/dib_elements/myproduct/finalise.d/99-generate-binary-checksum new file mode 100755 index 0000000..1668ceb --- /dev/null +++ b/dib_elements/myproduct/finalise.d/99-generate-binary-checksum @@ -0,0 +1,63 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +md5sum $(find / ! -path "/tmp/*" -executable -type f) > /root/binary_checksum.md5 +sha256sum $(find / ! -path "/tmp/*" -executable -type f) > /root/binary_checksum.sha256 + +# Remove checksum that has been modified during deployment +changed_checksums=( "resolv.conf" "mariadb.sg" "rabbitmq-server.sg" "audit.rules" "influxdb.conf" "influxdb-1" ) +for checksum in ${changed_checksums[@]} +do + sed -i "/${checksum}/d" /root/binary_checksum.md5 + sed -i "/${checksum}/d" /root/binary_checksum.sha256 +done + +# Add checksums that will be added during deployment +checksums_to_add=(\ +"/etc/openstack-dashboard/enabled/_2200_ironic.py" \ +"/etc/openstack-dashboard/enabled/_2200_ironic.pyc" \ +"/etc/openstack-dashboard/enabled/_2200_ironic.pyo" \ +"/etc/openstack-dashboard/local_settings" \ +"/etc/openstack-dashboard/nova_policy.json" \ +"/etc/openstack-dashboard/neutron_policy.json" \ +"/etc/openstack-dashboard/keystone_policy.json" \ +"/etc/openstack-dashboard/glance_policy.json" \ +"/etc/openstack-dashboard/cinder_policy.json" \ +"/etc/openstack-dashboard/cinder_policy.d/consistencygroup.yaml" \ +"/etc/openstack-dashboard/nova_policy.d/api-extensions.yaml" \ +) +for f in ${checksums_to_add[@]}; do + if test -f ${f}; then + md5sum ${f} >> /root/binary_checksum.md5 + sha256sum ${f} >> /root/binary_checksum.sha256 + fi +done + +# Include docker images +docker_images=$( find /var/lib/crf-images/docker-images/infra/ -type f ) || : +if [ -n "${docker_images}" ]; then + md5sum ${docker_images} >> /root/binary_checksum.md5 + sha256sum ${docker_images} >> /root/binary_checksum.sha256 +fi + +# Sort the checksum files +sort -k2 -o /root/binary_checksum.md5 /root/binary_checksum.md5 +sort -k2 -o /root/binary_checksum.sha256 /root/binary_checksum.sha256 diff --git a/dib_elements/myproduct/finalise.d/99-remove-dhcp-all-interfaces-udev-rules b/dib_elements/myproduct/finalise.d/99-remove-dhcp-all-interfaces-udev-rules new file mode 100755 index 0000000..f8382d4 --- /dev/null +++ b/dib_elements/myproduct/finalise.d/99-remove-dhcp-all-interfaces-udev-rules @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +rm -rf /etc/udev/rules.d/99-dhcp-all-interfaces.rules +# below file is coming from the base image we use +rm -rf /etc/sysconfig/network-scripts/ifcfg-eth0 diff --git a/dib_elements/myproduct/finalise.d/99-set-sshd-config-defaults b/dib_elements/myproduct/finalise.d/99-set-sshd-config-defaults new file mode 100755 index 0000000..103ddc8 --- /dev/null +++ b/dib_elements/myproduct/finalise.d/99-set-sshd-config-defaults @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +sed -i 's/^#*UseDNS .*/UseDNS no/' /etc/ssh/sshd_config +sed -i 's/^#*GSSAPIAuthentication .*/GSSAPIAuthentication no/' /etc/ssh/sshd_config + diff --git a/dib_elements/myproduct/install.d/50-set-rootpasswd b/dib_elements/myproduct/install.d/50-set-rootpasswd new file mode 100755 index 0000000..4a7ea58 --- /dev/null +++ b/dib_elements/myproduct/install.d/50-set-rootpasswd @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script should be removed as soon as possible. +# It is hardcoding root password to root. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +echo "root:root" | chpasswd + +sed -i 's/.*PermitRootLogin yes/PermitRootLogin yes/' /etc/ssh/sshd_config +sed -i 's/.*PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config diff --git a/dib_elements/myproduct/post-install.d/50-remove-local-repofile b/dib_elements/myproduct/post-install.d/50-remove-local-repofile new file mode 100755 index 0000000..aad0d37 --- /dev/null +++ b/dib_elements/myproduct/post-install.d/50-remove-local-repofile @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +rm -rf /etc/yum.repos.d/localrepo.repo diff --git a/dib_elements/myproduct/post-install.d/98-collect-ecc-packages b/dib_elements/myproduct/post-install.d/98-collect-ecc-packages new file mode 100755 index 0000000..69fccd9 --- /dev/null +++ b/dib_elements/myproduct/post-install.d/98-collect-ecc-packages @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +cp -f /tmp/in_target.d/collect_ecc.py . +python collect_ecc.py crypto_rpms.json +rm -f collect_ecc.py diff --git a/dib_elements/myproduct/post-install.d/99-validate-packages-to-install b/dib_elements/myproduct/post-install.d/99-validate-packages-to-install new file mode 100755 index 0000000..028330b --- /dev/null +++ b/dib_elements/myproduct/post-install.d/99-validate-packages-to-install @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" + +rpm -q --queryformat '' $PACKAGES_TO_INSTALL + diff --git a/dib_elements/myproduct/pre-install.d/01-enable-yum-priorities b/dib_elements/myproduct/pre-install.d/01-enable-yum-priorities new file mode 100755 index 0000000..6c472d2 --- /dev/null +++ b/dib_elements/myproduct/pre-install.d/01-enable-yum-priorities @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +pkgs="yum-plugin-priorities" +yum install -y $pkgs +rpm -q --queryformat '' $pkgs diff --git a/dib_elements/myproduct/root.d/50-local-repo b/dib_elements/myproduct/root.d/50-local-repo new file mode 100755 index 0000000..bb9bd64 --- /dev/null +++ b/dib_elements/myproduct/root.d/50-local-repo @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +[ -n "$TARGET_ROOT" ] + +# Remove all the repo file content in the target root. +for file in `ls $TMP_MOUNT_PATH/etc/yum.repos.d/*.repo`; do + sudo echo > $file +done + +# mount the local repo dir on target loop device +mkdir -p $TARGET_ROOT/$DIB_LOCAL_REPO +sudo mount --bind $DIB_LOCAL_REPO $TARGET_ROOT/$DIB_LOCAL_REPO diff --git a/dib_elements/myproduct/root.d/51-rm-grub-defaults b/dib_elements/myproduct/root.d/51-rm-grub-defaults new file mode 100755 index 0000000..313bb70 --- /dev/null +++ b/dib_elements/myproduct/root.d/51-rm-grub-defaults @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +[ -n "$TARGET_ROOT" ] + +# Empty the grub defaults file coming from the base image. +# Native DIB element running in the finalise stage is appending to the original file. +# This was resulting in duplicate entries. +echo > $TMP_MOUNT_PATH/etc/default/grub diff --git a/docker-context/Dockerfile-buildtools b/docker-context/Dockerfile-buildtools new file mode 100644 index 0000000..65f4302 --- /dev/null +++ b/docker-context/Dockerfile-buildtools @@ -0,0 +1,20 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM fedora:27 + +RUN dnf install -y python wget yum-utils && \ + pip install requests +WORKDIR /work + diff --git a/docker-context/Dockerfile-dib b/docker-context/Dockerfile-dib new file mode 100644 index 0000000..fa1359f --- /dev/null +++ b/docker-context/Dockerfile-dib @@ -0,0 +1,32 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM centos:7.5.1804 +RUN \ + yum-config-manager --add-repo http://mirror.centos.org/centos/7/cloud/x86_64/openstack-pike/ && \ + yum-config-manager --add-repo https://trunk.rdoproject.org/centos7-pike/57/c7/57c7be250c919c04b51361d4d42e95818cfec5a5_15fc9723 && \ + yum install --nogpgcheck -y diskimage-builder \ + git \ + python \ + wget \ + which \ + findutils \ + systemd-udev \ + PyYAML \ + parted \ + sudo \ + e2fsprogs \ + xfsprogs + +WORKDIR /work diff --git a/docker-context/README b/docker-context/README new file mode 100644 index 0000000..a701673 --- /dev/null +++ b/docker-context/README @@ -0,0 +1,31 @@ +Rules +----- + +- CI must pull images from the intranet and not from the internet +- CI must use always versioned tags, not the "latest" + + +How to use? +----------- + +e.g. + $ curl -L http:///dib:1.2.tar | docker load + + +How to develop? +--------------- + +#. Create local image + + .. code-block:: + + $ docker build --rm -f Dockerfile-dib -t dib: . + e.g. + $ docker build --rm -f Dockerfile-dib -t dib:1.2 .``` + +#. Upload to file server + + .. code-block:: + + $ docker save dib:1.2 > dib:1.2.tar + $ scp ./dib\:1.2.tar @:/var/www// diff --git a/isolinux/isolinux.cfg b/isolinux/isolinux.cfg new file mode 100644 index 0000000..bbd96eb --- /dev/null +++ b/isolinux/isolinux.cfg @@ -0,0 +1,48 @@ +default vesamenu.c32 +timeout 100 + +display boot.msg + +# Clear the screen when exiting the menu, instead of leaving the menu displayed. +# For vesamenu, this means the graphical background is still displayed without +# the menu itself for as long as the screen remains in graphics mode. +menu clear +menu background splash.png +menu title Cloud +menu vshift 8 +menu rows 18 +menu margin 8 +menu hidden + +menu color border 0 #ffffffff #ee000000 std +menu color title 0 #ffffffff #ee000000 std +menu color disabled 0 #ffffffff #ee000000 std +menu color sel 0 #ffffffff #85000000 std +menu color unsel 0 #ffffffff #ee000000 std +menu color pwdheader 0 #ff000000 #99ffffff rev +menu color pwdborder 0 #ff000000 #99ffffff rev +menu color pwdentry 0 #ff000000 #99ffffff rev +menu color hotkey 0 #ff00ff00 #ee000000 std +menu color hotsel 0 #ffffffff #85000000 std +menu color cmdmark 0 #ffffffff #ee000000 std +menu color cmdline 0 #ffffffff #ee000000 std + +# Do not display the actual menu unless the user presses a key. All that is displayed is a timeout message. + +menu tabmsg Press Tab for full configuration options on menu items. + +menu separator # insert an empty line +menu separator # insert an empty line + +label linux +menu default + menu label ^Install Cloud + kernel vmlinuz + append initrd=initrd.img inst.stage2=hd:LABEL=CentOS\x207\x20x86_64 quiet console=tty0 console=ttyS1,115200 + +label check + menu label Test this ^media & install Cloud + kernel vmlinuz + append initrd=initrd.img inst.stage2=hd:LABEL=CentOS\x207\x20x86_64 rd.live.check quiet console=tty0 console=ttyS1,115200 + +menu end diff --git a/lib.sh b/lib.sh new file mode 100644 index 0000000..671d041 --- /dev/null +++ b/lib.sh @@ -0,0 +1,233 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o pipefail +set -e + +LIBDIR="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" + +PUBLISH_RESULTS="${PUBLISH_RESULTS:-false}" +VIRT_CUSTOMIZE_MEM="${VIRT_CUSTOMIZE_MEM:-}" +VIRT_CUSTOMIZE_SMP="${VIRT_CUSTOMIZE_SMP:-}" +PARALLEL_BUILD_TIMEOUT="${PARALLEL_BUILD_TIMEOUT:-0}" +ENABLE_GOLDEN_IMAGE_ROOT_PASSWORD="${ENABLE_GOLDEN_IMAGE_ROOT_PASSWORD:-true}" +GOLDEN_BASE_IMAGE_TAR_URL=${GOLDEN_BASE_IMAGE_TAR_URL:-} +GOLDEN_BASE_IMAGE_FETCH_USER=${GOLDEN_BASE_IMAGE_FETCH_USER:-} +GOLDEN_BASE_IMAGE_FETCH_PASSWORD=${GOLDEN_BASE_IMAGE_FETCH_PASSWORD:-} + +WORK=$(dirname $(dirname $LIBDIR)) +RPM_BUILDER=$(find $WORK -maxdepth 2 -type d -name rpmbuilder) + +WORKTMP=$WORK/tmp +WORKLOGS=$WORKTMP/logs +DURATION_LOG=$WORKLOGS/durations.log +MANIFEST_PATH=$WORK/.repo/manifests +BUILD_CONFIG_INI=$WORK/.repo/manifests/build_config.ini +GOLDEN_IMAGE_NAME=guest-image.img +TMP_GOLDEN_IMAGE=$WORKTMP/$GOLDEN_IMAGE_NAME + +WORKRESULTS=$WORK/results +REPO_FILES=$WORKRESULTS/repo_files +REPO_DIR=$WORKRESULTS/repo +SRC_REPO_DIR=$WORKRESULTS/src_repo +RPMLISTS=$WORKRESULTS/rpmlists +CHECKSUM_DIR=$WORKRESULTS/bin_checksum +RESULT_IMAGES_DIR=$WORKRESULTS/images +RPM_BUILDER_SETTINGS=$WORKTMP/mocksettings/mock.cfg + +function _read_build_config() +{ + local config_ini=$BUILD_CONFIG_INI + if [[ -f "$1" ]] && [[ $1 == *.ini ]]; then + config_ini=$1 + shift + fi + PYTHONPATH=$LIBDIR $LIBDIR/tools/script/read_build_config.py $config_ini $@ +} + +function _read_manifest_vars() +{ + PRODUCT_RELEASE_BUILD_ID="${BUILD_NUMBER:?0}" + PRODUCT_RELEASE_LABEL="$(_read_build_config DEFAULT product_release_label)" +} + +function _initialize_work_dirs() +{ + rm -rf $WORKRESULTS + mkdir -p $WORKRESULTS $REPO_FILES $REPO_DIR $RPMLISTS $CHECKSUM_DIR + # dont clear tmp, can be used for caching + mkdir -p $WORKTMP + rm -rf $WORKLOGS + mkdir -p $WORKLOGS +} + +function _log() +{ + echo "$(date) $@" +} + +function _info() +{ + _log INFO: $@ +} + +function _header() +{ + _info "##################################################################" + _info "# $@" + _info "##################################################################" +} + + +function _divider() +{ + _info "------------------------------------------------------------------" +} + + +function _step() +{ + _header "STEP START: $@" +} + + +function _abort() +{ + _header "ERROR: $@" + exit 1 +} + + +function _success() +{ + _header "STEP OK: $@" +} + + +function _run_cmd() +{ + _info "[cmd-start]: $@" + stamp_start=$(date +%s) + time $@ 2>&1 || _abort "Command failed: $@" + stamp_end=$(date +%s) + echo "$((stamp_end - stamp_start)) $@" >> $DURATION_LOG.unsorted + sort -nr $DURATION_LOG.unsorted > $DURATION_LOG + _log "[cmd-end]: $@" +} + + +function _run_cmd_as_step() +{ + if [ $# -eq 1 -a -f $1 ]; then + step="$(basename $1)" + else + step="$@" + fi + _step $step + _run_cmd $@ + _success $step +} + + +function _add_rpms_to_repo() +{ + local repo_dir=$1 + local rpm_dir=$2 + mkdir -p $repo_dir + cp -f $(repomanage --keep=1 --new $rpm_dir) $repo_dir/ +} + +function _create_localrepo() +{ + pushd $REPO_DIR + _run_cmd createrepo --workers=8 --update . + popd + pushd $SRC_REPO_DIR + _run_cmd createrepo --workers=8 --update . + popd +} + +function _add_rpms_to_repos_from_workdir() +{ + _add_rpms_to_repo $REPO_DIR $1/buildrepository/mock/rpm + _add_rpms_to_repo $SRC_REPO_DIR $1/buildrepository/mock/srpm + #find $1/ -name '*.tar.gz' | xargs rm -f + true +} + +function _publish_results() +{ + local from=$1 + local to=$2 + mkdir -p $(dirname $to) + mv -f $from $to +} + +function _publish_image() +{ + _publish_results $1 $2 + _create_checksum $2 +} + +function _create_checksum() +{ + _create_md5_checksum $1 + _create_sha256_checksum $1 +} + +function _create_sha256_checksum() +{ + pushd $(dirname $1) + time sha256sum $(basename $1) > $(basename $1).sha256 + popd +} + +function _create_md5_checksum() +{ + pushd $(dirname $1) + time md5sum $(basename $1) > $(basename $1).md5 + popd +} + +function _is_true() +{ + # e.g. for Jenkins boolean parameters + [ "$1" == "true" ] +} + +function _join_array() +{ + local IFS="$1" + shift + echo "$*" +} + +function _get_package_list() +{ + PYTHONPATH=$LIBDIR $LIBDIR/tools/script/read_package_config.py $@ +} + +function _load_docker_image() +{ + local docker_image=$1 + local docker_image_url="$(_read_build_config DEFAULT docker_images)/${docker_image}.tar" + if docker inspect ${docker_image} &> /dev/null; then + echo "Using already built ${docker_image} image" + else + echo "Loading ${docker_image} image" + curl -L $docker_image_url | docker load + fi +} + diff --git a/mock/logging.ini b/mock/logging.ini new file mode 100644 index 0000000..8186ead --- /dev/null +++ b/mock/logging.ini @@ -0,0 +1,84 @@ +[formatters] +keys: detailed,simple,unadorned,state + +[handlers] +keys: simple_console,detailed_console,unadorned_console,simple_console_warnings_only + +[loggers] +keys: root,build,state,mockbuild + +[formatter_state] +format: %(asctime)s - %(message)s + +[formatter_unadorned] +format: %(message)s + +[formatter_simple] +format: %(levelname)s: %(message)s + +;useful for debugging: +[formatter_detailed] +format: %(levelname)s %(filename)s:%(lineno)d: %(message)s + +[handler_unadorned_console] +class: StreamHandler +args: [] +formatter: unadorned +level: INFO + +[handler_simple_console] +class: StreamHandler +args: [] +formatter: simple +level: INFO + +[handler_simple_console_warnings_only] +class: StreamHandler +args: [] +formatter: simple +level: WARNING + +[handler_detailed_console] +class: StreamHandler +args: [] +formatter: detailed +level: WARNING + +; usually dont want to set a level for loggers +; this way all handlers get all messages, and messages can be filtered +; at the handler level +; +; all these loggers default to a console output handler +; +[logger_root] +level: NOTSET +handlers: simple_console + +; mockbuild logger normally has no output +; catches stuff like mockbuild.trace_decorator and mockbuild.util +; dont normally want to propagate to root logger, either +[logger_mockbuild] +level: NOTSET +handlers: +qualname: mockbuild +propagate: 1 + +[logger_state] +level: NOTSET +; unadorned_console only outputs INFO or above +handlers: unadorned_console +qualname: mockbuild.Root.state +propagate: 0 + +[logger_build] +level: NOTSET +handlers: simple_console_warnings_only +qualname: mockbuild.Root.build +propagate: 0 + +; the following is a list mock logger qualnames used within the code: +; +; qualname: mockbuild.util +; qualname: mockbuild.uid +; qualname: mockbuild.trace_decorator + diff --git a/mock/mock.cfg.template b/mock/mock.cfg.template new file mode 100644 index 0000000..e2db78b --- /dev/null +++ b/mock/mock.cfg.template @@ -0,0 +1,89 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Root name to be used for chroot and caching, must differ between products +config_opts['root'] = 'akrainolite' + +config_opts['target_arch'] = 'x86_64' +config_opts['legal_host_arches'] = ('x86_64',) +config_opts['dist'] = 'el7' # only useful for --resultdir variable subst +config_opts['chroot_setup_cmd'] = 'install createrepo yum-utils bison byacc cscope ctags cvs diffstat doxygen flex gcc gcc-c++ gcc-gfortran gettext git indent intltool libtool patch patchutils rcs redhat-rpm-config rpm-build subversion swig systemtap sudo' +config_opts['plugin_conf']['yum_cache_enable'] = False +config_opts['plugin_conf']['ccache_enable'] = False +config_opts['plugin_conf']['ccache_opts']['max_cache_size'] = '1G' +config_opts['plugin_conf']['ccache_opts']['dir'] = "/dev/shm/ccache.lcc-epel7" +config_opts['rpmbuild_networking'] = True +config_opts['cleanup_on_success'] = True +config_opts['cleanup_on_failure'] = False +config_opts['exclude_from_homedir_cleanup'] = ('build/SOURCES', '.bash_history', '.bashrc', 'build/RPMS', ) +#ADDITIONAL_CONFIG_OPTS# + +# Common RPM directive values +config_opts['macros']['%packager'] = #RPM_PACKAGER# +config_opts['macros']['%dist'] = ".el7.centos%{_platform_product}" +config_opts['macros']['%_platform_product'] = #RPM_RELEASE_ID# +config_opts['macros']['%_platform_dist'] = ".el7.centos" +config_opts['macros']['%_platform_vendor'] = #RPM_VENDOR# +config_opts['macros']['%_platform_license'] = #RPM_LICENSE# +config_opts['macros']['%_platform_licence'] = "%{_platform_license}" + +# Product specific macros +config_opts['macros']['%_playbooks_path'] = "/opt/openstack-ansible/playbooks" +config_opts['macros']['%_inventory_path'] = "/opt/openstack-ansible/inventory" +config_opts['macros']['%_roles_path'] = "/etc/ansible/roles" +config_opts['macros']['%_installation_root_path'] = "/etc/lcm/playbooks/installation" +config_opts['macros']['%_bootstrapping_path'] = "%{_installation_root_path}/bootstrapping" +config_opts['macros']['%_provisioning_path'] = "%{_installation_root_path}/provisioning" +config_opts['macros']['%_postconfig_path'] = "%{_installation_root_path}/postconfig" +config_opts['macros']['%_finalize_path'] = "%{_installation_root_path}/finalize" +config_opts['macros']['%_ansible_filter_plugins_path'] = "%{_roles_path}/plugins/filter" +config_opts['macros']['%_caas_path'] = "/var/lib/caas" +config_opts['macros']['%_caas_container_tar_path'] = "%{_caas_path}/images" +config_opts['macros']['%_caas_manifest_path'] = "%{_caas_path}/manifests" +config_opts['macros']['%_caas_chart_path'] = "${_caas_path}/chart/" +config_opts['macros']['%_platform_bin_path'] = "/usr/local/bin" +config_opts['macros']['%_platform_lib_path'] = "/usr/local/lib" +config_opts['macros']['%_platform_etc_path'] = "/etc" +config_opts['macros']['%_platform_share_path'] = "/share" +config_opts['macros']['%_platform_man_path'] = "%{_platform_share_path}/man" +config_opts['macros']['%_platform_doc_path'] = "%{_platform_share_path}/doc" +config_opts['macros']['%_platform_var_path'] = "/var" +config_opts['macros']['%_platform_python'] = "/python2.7" +config_opts['macros']['%_platform_python_site_packages_path'] = "%{_platform_lib_path}%{_platform_python}/site-packages" +config_opts['macros']['%_platform_ocf_resource_path'] = "/usr/lib/ocf/resource.d" +config_opts['macros']['%_python_site_packages_path'] = "/usr/lib/python2.7/site-packages" +config_opts['macros']['%_secrets_path'] = "/etc/required-secrets" + +# Compilation +#config_opts['macros']['%_smp_mflags'] = "-j6" +#config_opts['macros']['%_smp_ncpus_max'] = 0 + +# Yum configuration +config_opts['yum.conf'] = """ +[main] +cachedir=/var/cache/yum +keepcache=1 +debuglevel=2 +reposdir=/dev/null +logfile=/var/log/yum.log +retries=20 +obsoletes=1 +gpgcheck=0 +assumeyes=1 +syslog_ident=mock +syslog_device= + +#REPOSITORIES# + +""" diff --git a/mock/site-defaults.cfg b/mock/site-defaults.cfg new file mode 100644 index 0000000..9fff7af --- /dev/null +++ b/mock/site-defaults.cfg @@ -0,0 +1,160 @@ +# mock defaults +# vim:tw=0:ts=4:sw=4:et: +# +# This config file is for site-specific default values that apply across all +# configurations. Options specified in this config file can be overridden in +# the individual mock config files. +# +# The site-defaults.cfg delivered by default has NO options set. Only set +# options here if you want to override the defaults. +# +# Entries in this file follow the same format as other mock config files. +# config_opts['foo'] = bar + +############################################################################# +# +# Things that we recommend you set in site-defaults.cfg: +# +# config_opts['basedir'] = '/var/lib/mock/' +# config_opts['cache_topdir'] = '/var/cache/mock' +# Note: the path pointed to by basedir and cache_topdir must be owned +# by group 'mock' and must have mode: g+rws +# config_opts['rpmbuild_timeout'] = 0 +# config_opts['use_host_resolv'] = True + +# You can configure log format to pull from logging.ini formats of these names: +# config_opts['build_log_fmt_name'] = "unadorned" +# config_opts['root_log_fmt_name'] = "detailed" +# config_opts['state_log_fmt_name'] = "state" +# +# mock will normally set up a minimal chroot /dev. +# If you want to use a pre-configured /dev, disable this and use the bind-mount +# plugin to mount your special /dev +# config_opts['internal_dev_setup'] = True +# +# internal_setarch defaults to 'True' if the python 'ctypes' package is +# available. It is in the python std lib on >= python 2.5. On older versions, +# it is available as an addon. On systems w/o ctypes, it will default to 'False' +# config_opts['internal_setarch'] = False +# +# the cleanup_on_* options allow you to automatically clean and remove the +# mock build directory, but only take effect if --resultdir is used. +# config_opts provides fine-grained control. cmdline only has big hammer +# +# config_opts['cleanup_on_success'] = 1 +# config_opts['cleanup_on_failure'] = 1 + +# if you want mock to automatically run createrepo on the rpms in your +# resultdir. +# config_opts['createrepo_on_rpms'] = False +# config_opts['createrepo_command'] = '/usr/bin/createrepo -d -q -x *.src.rpm' + +# if you want mock to backup the contents of a result dir before clean +# config_opts['backup_on_clean'] = False +# config_opts('backup_base_dir'] = config_opts['basedir'] + "backup" + + +############################################################################# +# +# plugin related. Below are the defaults. Change to suit your site +# policy. site-defaults.cfg is a good place to do this. +# +# NOTE: Some of the caching options can theoretically affect build +# reproducability. Change with care. +# +# config_opts['plugin_conf']['package_state_enable'] = True +# config_opts['plugin_conf']['ccache_enable'] = True +# config_opts['plugin_conf']['ccache_opts']['max_cache_size'] = '4G' +# config_opts['plugin_conf']['ccache_opts']['compress'] = None +# config_opts['plugin_conf']['ccache_opts']['dir'] = "%(cache_topdir)s/%(root)s/ccache/" +# config_opts['plugin_conf']['yum_cache_enable'] = True +# config_opts['plugin_conf']['yum_cache_opts']['max_age_days'] = 30 +# config_opts['plugin_conf']['yum_cache_opts']['dir'] = "%(cache_topdir)s/%(root)s/yum_cache/" +# config_opts['plugin_conf']['root_cache_enable'] = True +# config_opts['plugin_conf']['root_cache_opts']['max_age_days'] = 15 +# config_opts['plugin_conf']['root_cache_opts']['dir'] = "%(cache_topdir)s/%(root)s/root_cache/" +# config_opts['plugin_conf']['root_cache_opts']['compress_program'] = "pigz" +# config_opts['plugin_conf']['root_cache_opts']['extension'] = ".gz" +# config_opts['plugin_conf']['root_cache_opts']['exclude_dirs'] = ["./proc", "./sys", "./dev", +# "./tmp/ccache", "./var/cache/yum" ] +# +# bind mount plugin is enabled by default but has no configured directories to +# mount +# config_opts['plugin_conf']['bind_mount_enable'] = True +# config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/host/path', '/bind/mount/path/in/chroot/' )) +# +# config_opts['plugin_conf']['tmpfs_enable'] = False +# config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 1024 +# config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = '512m' +# config_opts['plugin_conf']['tmpfs_opts']['mode'] = '0755' +# config_opts['plugin_conf']['chroot_scan_enable'] = False +# config_opts['plugin_conf']['chroot_scan_opts'] = [ "core(\.\d+)?", "\.log$",] + +############################################################################# +# +# environment for chroot +# +# config_opts['environment']['TERM'] = 'vt100' +# config_opts['environment']['SHELL'] = '/bin/bash' +# config_opts['environment']['HOME'] = '/builddir' +# config_opts['environment']['HOSTNAME'] = 'mock' +# config_opts['environment']['PATH'] = '/usr/bin:/bin:/usr/sbin:/sbin' +# config_opts['environment']['PROMPT_COMMAND'] = 'echo -n ""' +# config_opts['environment']['LANG'] = os.environ.setdefault('LANG', 'en_US.UTF-8') +# config_opts['environment']['TZ'] = os.environ.setdefault('TZ', 'EST5EDT') + +############################################################################# +# +# Things that you can change, but we dont recommend it: +# config_opts['chroothome'] = '/builddir' +# config_opts['clean'] = True + +############################################################################# +# +# Things that must be adjusted if SCM integration is used: +# +# config_opts['scm'] = True +# config_opts['scm_opts']['method'] = 'git' +# config_opts['scm_opts']['cvs_get'] = 'cvs -d /srv/cvs co SCM_BRN SCM_PKG' +# config_opts['scm_opts']['git_get'] = 'git clone SCM_BRN git://localhost/SCM_PKG.git SCM_PKG' +# config_opts['scm_opts']['svn_get'] = 'svn co file:///srv/svn/SCM_PKG/SCM_BRN SCM_PKG' +# config_opts['scm_opts']['spec'] = 'SCM_PKG.spec' +# config_opts['scm_opts']['ext_src_dir'] = '/dev/null' +# config_opts['scm_opts']['write_tar'] = True +# config_opts['scm_opts']['git_timestamps'] = True + +# These options are also recognized but usually defined in cmd line +# with --scm-option package= --scm-option branch= +# config_opts['scm_opts']['package'] = 'mypkg' +# config_opts['scm_opts']['branch'] = 'master' + +############################################################################# +# +# Things that are best suited for individual chroot config files: +# +# MUST SET (in individual chroot cfg file): +# config_opts['root'] = 'name-of-yum-build-dir' +# config_opts['target_arch'] = 'i386' +# config_opts['yum.conf'] = '' +# config_opts['yum_common_opts'] = [] +# +# CAN SET, defaults usually work ok: +# config_opts['chroot_setup_cmd'] = 'install buildsys-build' +# config_opts['log_config_file'] = 'logging.ini' +# config_opts['more_buildreqs']['srpm_name-version-release'] = 'dependencies' +# config_opts['macros']['%Add_your_macro_name_here'] = "add macro value here" +# config_opts['files']['path/name/no/leading/slash'] = "put file contents here." +# config_opts['chrootuid'] = os.getuid() + +# If you change chrootgid, you must also change "mock" to the correct group +# name in this line of the mock PAM config: +# auth sufficient pam_succeed_if.so user ingroup mock use_uid quiet +# config_opts['chrootgid'] = grp.getgrnam("mock")[2] + +# config_opts['useradd'] = '/usr/sbin/useradd -m -u %(uid)s -g %(gid)s -d %(home)s -n %(user)s' # Fedora/RedHat +# +# Security related +# config_opts['no_root_shells'] = False +# +# Proxy settings (https_proxy, ftp_proxy, and no_proxy can also be set) +# config_opts['http_proxy'] = 'http://localhost:3128' diff --git a/nexus3_dl.sh b/nexus3_dl.sh new file mode 100755 index 0000000..03dd7ed --- /dev/null +++ b/nexus3_dl.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +NEXUS_URL=$1 +NEXUS_REPOSITORY=$2 +NEXUS_REPOSITORY_BASE_PATH=$3 +shift;shift;shift +NEXUS_REPOSITORY_SEARCH_PATTERNS=$@ + +_abort() { + echo "ERROR: $@" + exit 1 +} + +_search_group() { + local params="" + [ -n "$1" ] && params+="&group=$1" + [ -n "${2:-}" ] && params+="&name=$2" + curl "${NEXUS_URL}/service/rest/v1/search?repository=${NEXUS_REPOSITORY}${params}" +} + +for pat in $NEXUS_REPOSITORY_SEARCH_PATTERNS; do + search_group="/$NEXUS_REPOSITORY_BASE_PATH/$(echo $pat | cut -d':' -f1)" + search_name="" + if echo $pat | grep ':'; then + search_name="$(echo $pat | cut -d':' -f2)" + fi + resp=$(_search_group $search_group $search_name) + if [ "$(echo $resp | jq -r '.continuationToken')" != "null" ]; then + _abort "Pagination not implemented" + fi + for url in $(echo $resp | jq -r '.items[].assets[].downloadUrl'); do + to=${url#$NEXUS_URL/repository/$NEXUS_REPOSITORY/$NEXUS_REPOSITORY_BASE_PATH/} + mkdir -p $(dirname $to) + echo "Fetch $url" + curl $url > $to + done +done diff --git a/prepare_manifest.sh b/prepare_manifest.sh new file mode 100755 index 0000000..e34536d --- /dev/null +++ b/prepare_manifest.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh +_read_manifest_vars + +pushd $MANIFEST_PATH +sed -i -e "s/^Version: .*/Version: $PRODUCT_RELEASE_BUILD_ID/" rpmbuild.spec +cat > product-release <?$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module +max-module-lines=1000 + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of statements in function / method body +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[IMPORTS] + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/repo_summary.sh b/repo_summary.sh new file mode 100755 index 0000000..e533389 --- /dev/null +++ b/repo_summary.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh + +_run_repo_cmd() +{ + _divider + _run_cmd $@ +} + +pushd $WORK/.repo/repo +_run_repo_cmd git rev-parse --short HEAD +popd +echo q | _run_repo_cmd repo info +_run_repo_cmd repo status -o +echo q | _run_repo_cmd repo forall -p -c git rev-parse HEAD + +# Store build manifest with project HEADS to a file +repo manifest -o $WORKRESULTS/manifest.xml +repo manifest -r -o $WORKRESULTS/manifest_revisions.xml diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..41944d1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +requests +pyyaml +pytest +pytest-cov +pytest-flakes +pytest-pep8 +mock diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/buildconfig.py b/tools/buildconfig.py new file mode 100755 index 0000000..a43fa92 --- /dev/null +++ b/tools/buildconfig.py @@ -0,0 +1,33 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ConfigParser + +from tools.statics import BUILD_CONFIG_PATH + + +class BuildConfigParser(ConfigParser.ConfigParser): + def __init__(self, ini_file=BUILD_CONFIG_PATH): + ConfigParser.ConfigParser.__init__(self) + self.ini_file = ini_file + self.optionxform = str + self.read(self.ini_file) + + def items(self, section): # pylint: disable=arguments-differ + defaults = self.defaults() + resultlist = [] + for item in ConfigParser.ConfigParser.items(self, section): + if item[0] not in defaults: + resultlist.append(item) + return resultlist diff --git a/tools/convert.py b/tools/convert.py new file mode 100644 index 0000000..31fa30d --- /dev/null +++ b/tools/convert.py @@ -0,0 +1,142 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import re + + +def to_json(data): + return json.dumps(data, sort_keys=True, indent=4) + + +class CsvConverter(object): + def __init__(self, data, preferred_field_order=None, escape_newlines=True): + self.data = data + self.preferred_field_order = preferred_field_order + self.escape_newlines = escape_newlines + self.csv_data = None + self._convert() + + def __str__(self): + return self.convert() + + def convert(self): + return self._render(CsvFormatter(self.csv_data)) + + def convert_to_ms_excel(self, text_fields=None): + """ + CSV that Microsoft Excel can read well. + + :param text_fields: list of columns to mark as text + NOTE: must not be used for fields that can contain comma(,) or + semicolon(;) as field will be split from these + :return: + """ + return self._render(CsvMSFormatter(self.csv_data, text_fields=text_fields)) + + def _convert(self): + if not isinstance(self.data, list): + raise Exception('Input data given is NOT a list') + if not self.data: + self.csv_data = [] + return + if not isinstance(self.data[0], dict): + raise Exception('First data element is NOT a dict') + headers = [] + possible_fields = list(set([key for i in self.data for key in i.keys()])) + if self.preferred_field_order is not None: + for preferred_field in self.preferred_field_order: + if preferred_field in possible_fields: + headers.append(preferred_field) + possible_fields.remove(preferred_field) + headers += sorted(possible_fields) + self.csv_data = [headers] + for obj in self.data: + row_data = [] + for header in headers: + field = obj.get(header) + if isinstance(field, (list, dict)): + x = json.dumps(field, sort_keys=True) + elif isinstance(field, unicode): + x = field.encode('utf-8') + else: + x = str(field) + row_data.append(x) + self.csv_data.append(row_data) + + def _render(self, formatter): + return formatter.format(self.escape_newlines) + + +class CsvFormatter(object): + def __init__(self, csv_data): + self.csv_data = csv_data + + def format(self, escape_newlines=True): + f_file = [] + for record in self.csv_data: + f_record = [] + for field in record: + f_field = self._field_formatter(field, escape_newlines) + f_record.append(f_field) + f_file.append(','.join(self._record_formatter(f_record))) + return '\r\n'.join(self._file_formatter(f_file)) + + @staticmethod + def _file_formatter(_file): + return _file + + @staticmethod + def _record_formatter(record): + return ['"{}"'.format(i) for i in record] + + @staticmethod + def _field_formatter(field, escape_newlines): + out = field.replace('"', '""') + if escape_newlines: + out = out.replace('\n', '\\n') + return out + + +class CsvMSFormatter(CsvFormatter): + max_cell_size = 32000 + + def __init__(self, csv_data, text_fields=None): + super(CsvMSFormatter, self).__init__(csv_data) + self.text_fields = text_fields + + def _file_formatter(self, _file): + return ['sep=,'] + super(CsvMSFormatter, self)._file_formatter(_file) + + def _record_formatter(self, record): + record = super(CsvMSFormatter, self)._record_formatter(record) + if self.text_fields: + formatted_record = [] + for index, field in enumerate(record): + heading = self.csv_data[0][index] + if heading in self.text_fields: + formatted_field = '=' + field + else: + formatted_field = field + formatted_record.append(formatted_field) + record = formatted_record + return record + + def _field_formatter(self, field, escape_newlines): + field = super(CsvMSFormatter, self)._field_formatter(field, escape_newlines) + if len(field) > self.max_cell_size: + field = field[:self.max_cell_size / 2] + "..." + field[-self.max_cell_size / 2:] + if not re.match(r'^-\d+$', field) and re.match(r'^-.*$', field): + return r'\{}'.format(field) + return field diff --git a/tools/convert_test.py b/tools/convert_test.py new file mode 100755 index 0000000..cdfe7dc --- /dev/null +++ b/tools/convert_test.py @@ -0,0 +1,165 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=invalid-name +from collections import OrderedDict + +import pytest + +from tools.convert import to_json, CsvConverter + + +class TestJson(object): + @staticmethod + def test_json(): + _input = dict(b=1, a=2) + expected = '\n'.join(['{', + ' "a": 2, ', + ' "b": 1', + '}']) + assert to_json(_input) == expected + + +class TestCsv(object): + @staticmethod + @pytest.mark.parametrize('_input, expected', [ + ([], ''), + ([{}], '\r\n'), + ([dict(a=1)], '\r\n'.join(['"a"', + '"1"'])), + ([dict(a=1), dict(a=2)], '\r\n'.join(['"a"', + '"1"', + '"2"'])), + ([dict(a=1, b=2), dict(a=3, b=4)], '\r\n'.join(['"a","b"', + '"1","2"', + '"3","4"'])), + ([dict(x=OrderedDict({'b': 1, 'a': 2})), + dict(x={'a': 2, 'b': 1})], '\r\n'.join(['"x"', + '"{""a"": 2, ""b"": 1}"', + '"{""a"": 2, ""b"": 1}"'])) + + ]) + def test_csv(_input, expected): + assert str(CsvConverter(_input)) == expected + + @staticmethod + def test_str(): + _input = [dict(a=1)] + assert str(CsvConverter(_input)) == CsvConverter(_input).convert() + + @staticmethod + @pytest.mark.parametrize('_input, exception_re', [ + ('boom', r'NOT a list'), + ([['boom']], r'NOT a dict') + ]) + def test_to_csv_fail(_input, exception_re): + with pytest.raises(Exception, match=exception_re): + str(CsvConverter(_input)) + + @staticmethod + def test_newlines_are_escaped(): + """ + ...in order to get properly formatted file when written + """ + _input = [dict(a='1"2')] + expected = '\r\n'.join(['"a"', + '"1""2"']) + assert CsvConverter(_input).convert() == expected + + @staticmethod + def test_missing_fields_are_filled_with_none(): + _input = [dict(a='1"2')] + expected = '\r\n'.join(['"a"', + '"1""2"']) + assert CsvConverter(_input).convert() == expected + + @staticmethod + def test_double_quote_is_escaped_with_double_quote(): + """ + RFC 4180 + """ + _input = [dict(a='1"2')] + expected = '\r\n'.join(['"a"', + '"1""2"']) + assert CsvConverter(_input).convert() == expected + + @staticmethod + @pytest.mark.parametrize('_input, expected', [ + ([dict(a=1, b=2)], '\r\n'.join(['"b","a"', '"2","1"'])), + ([dict(a=1, c=2)], '\r\n'.join(['"a","c"', '"1","2"'])) + ]) + def test_csv_preferred_order(_input, expected): + assert CsvConverter(_input, preferred_field_order=['b', 'a']).convert() == expected + + @staticmethod + @pytest.mark.parametrize('_input, expected', [ + ([dict(a='Copyright \xc2\xa9 2014')], '\r\n'.join(['"a"', + '"Copyright \xc2\xa9 2014"'])), + ([dict(a=u'Copyright \xa9 2014')], '\r\n'.join(['"a"', + '"Copyright \xc2\xa9 2014"'])), + ]) + def test_unicode_input(_input, expected): + assert str(CsvConverter(_input)) == expected + + +class TestCsvMSExcel(object): + @staticmethod + def test_ms_excel_format(): + """ + MS Excel treats CSV files with 'sep=,' as the first line to get automatically columnized + """ + _input = [dict(a=1, b=2)] + expected = '\r\n'.join(['sep=,', + '"a","b"', + '"1","2"']) + assert CsvConverter(_input).convert_to_ms_excel() == expected + + @staticmethod + def test_text_fields(): + """ + MS Excel CSV fields prefixed with '=' will be treated as equations to string. + This makes it possible to e.g. to have all RPM version strings treated equally instead + of making some of them treated as generic and some of them as integers. + """ + _input = [dict(a=1, b=2)] + expected = '\r\n'.join(['sep=,', + '"a",="b"', + '"1",="2"']) + assert CsvConverter(_input).convert_to_ms_excel(text_fields=['b']) == expected + + @staticmethod + def test_field_with_beginning_minus_is_prefixed(): + """ + MS Excel CSV fields beginning with '-' are treated as an equation even they would be + essentially just strings. Make sure to escape the beginning signs with something in order + not to get field braking equation. + """ + _input = [dict(a=-1), dict(a="-2a")] + expected = '\r\n'.join(['sep=,', + '"a"', + '"-1"', + r'"\-2a"']) + assert CsvConverter(_input).convert_to_ms_excel() == expected + + @staticmethod + def test_too_big_cell_is_truncated(): + """ + MS Excel has ~32k character limit per cell. BOM information can easily exceed this. + """ + _input = [dict(a='1' * 32000), dict(a='2' * 32001)] + expected = '\r\n'.join(['sep=,', + '"a"', + '"{}"'.format('1' * 32000), + '"{}...{}"'.format('2' * 16000, '2' * 16000)]) + assert CsvConverter(_input).convert_to_ms_excel() == expected diff --git a/tools/executor.py b/tools/executor.py new file mode 100755 index 0000000..f2a428c --- /dev/null +++ b/tools/executor.py @@ -0,0 +1,107 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import logging +import shlex +import subprocess + + +class ExecutionError(Exception): + def __init__(self, msg, result): + super(ExecutionError, self).__init__(msg) + self.result = result + + +class Result(object): + def __init__(self, status, stdout, stderr): + self.status = status + self.stdout = stdout + self.stderr = stderr + + @property + def str_status(self): + return 'Status:"{}"'.format(self.status) + + @property + def str_stdout(self): + return 'Stdout:"{}"'.format(self.stdout) + + @property + def str_stderr(self): + return 'Stderr:"{}"'.format(self.stderr) + + def __str__(self): + return '\n'.join([self.str_status, self.str_stdout, self.str_stderr]) + + +def run(*args, **kwargs): + return Executor().run(*args, **kwargs) + + +class Executor(object): + def __init__(self, shell=False, chdir=None): + self.shell = shell + self.chdir = chdir + + def run(self, cmd, raise_on_error=True, raise_on_stderr=True, retries=0): + result = self._run(cmd, retries) + if raise_on_error and result.status != 0: + raise ExecutionError('Command "{}" execution status NOT zero: {}'.format( + cmd, str(result)), result.status) + if raise_on_stderr and result.stderr: + raise ExecutionError('Command "{}" execution stderr not empty: {}'.format( + cmd, str(result)), result.status) + return result + + @staticmethod + def _log_result(result): + logging.debug('Result: %s', str(result)) + + def _run(self, cmd, retries): + logstr = 'Executing command: "{}"'.format(cmd) + cwd = os.getcwd() + if self.chdir is not None: + os.chdir(self.chdir) + logstr += ' in dir {}'.format(self.chdir) + logging.debug(logstr) + result = self._run_with_retries(cmd, retries) + if self.chdir is not None: + os.chdir(cwd) + return result + + def _run_with_retries(self, cmd, retries): + result = self._execute(cmd) + while result.status != 0 and retries > 0: + logging.debug('Retrying, retries left: %s', retries) + retries -= 1 + result = self._execute(cmd) + self._log_result(result) + if result.status == 0: + break + return result + + def _execute(self, cmd): + if not self.shell: + if isinstance(cmd, list): + cmd_args = cmd[:] + else: + cmd_args = shlex.split(cmd) + else: + cmd_args = cmd + process = subprocess.Popen(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=self.shell) + stdout, stderr = process.communicate() + result = Result(process.returncode, stdout, stderr) + return result diff --git a/tools/executor_test.py b/tools/executor_test.py new file mode 100755 index 0000000..5b389c7 --- /dev/null +++ b/tools/executor_test.py @@ -0,0 +1,81 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import mock + +from tools.executor import Executor, ExecutionError +from tools.executor import Result + + +@mock.patch.object(Executor, '_execute') +def test_success(mocky): + mocky.side_effect = [Result(0, 'fake-stdout', '')] + Executor().run('test-cmd') + mocky.assert_called_once_with('test-cmd') + + +@mock.patch.object(Executor, '_execute') +def test_fail_returncode(mocky): + mocky.side_effect = [Result(1, 'fake-stdout', '')] + with pytest.raises(ExecutionError, match=r'execution status NOT zero'): + Executor().run('test-cmd') + mocky.assert_called_once_with('test-cmd') + + +@mock.patch.object(Executor, '_execute') +def test_fail_stderr(mocky): + mocky.side_effect = [Result(0, 'fake-stdout', 'fake-stderr')] + with pytest.raises(ExecutionError, match=r'stderr not empty'): + Executor().run('test-cmd') + mocky.assert_called_once_with('test-cmd') + + +@mock.patch.object(Executor, '_execute') +def test_success_ignore_returncode(mocky): + mocky.side_effect = [Result(1, 'fake-stdout', '')] + Executor().run('test-cmd', raise_on_error=False) + mocky.assert_called_once_with('test-cmd') + + +@mock.patch.object(Executor, '_execute') +def test_success_ignore_stderr(mocky): + mocky.side_effect = [Result(0, 'fake-stdout', 'fake-stderr')] + Executor().run('test-cmd', raise_on_stderr=False) + mocky.assert_called_once_with('test-cmd') + + +@mock.patch.object(Executor, '_execute') +def test_no_retry_on_success(mocky): + mocky.side_effect = [Result(0, 'fake-stdout', '')] + Executor().run('test-cmd', retries=1) + mocky.assert_called_once_with('test-cmd') + + +@mock.patch.object(Executor, '_execute') +def test_retry_on_fail(mocky): + mocky.side_effect = [Result(1, 'fake-stdout', ''), + Result(0, 'fake-stdout', '')] + Executor().run('test-cmd', retries=1) + expected = [mock.call('test-cmd'), + mock.call('test-cmd')] + assert mocky.mock_calls == expected + + +@mock.patch.object(Executor, '_execute') +def test_error_on_retry_exceeded(mocky): + mocky.side_effect = [Result(1, 'fake-stdout', ''), + Result(1, 'fake-stdout', '')] + with pytest.raises(ExecutionError): + Executor().run('test-cmd', retries=1) diff --git a/tools/io.py b/tools/io.py new file mode 100755 index 0000000..c09f9ce --- /dev/null +++ b/tools/io.py @@ -0,0 +1,37 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging + +from tools.convert import to_json + + +def write_to(file_path, data): + with open(file_path, 'w') as f: + f.write(data) + logging.debug('Wrote: {}'.format(file_path)) + + +def read_from(file_path): + with open(file_path, 'r') as f: + return f.read() + + +def write_json(file_path, data): + write_to(file_path, to_json(data)) + + +def read_json(file_path): + return json.loads(read_from(file_path)) diff --git a/tools/log.py b/tools/log.py new file mode 100755 index 0000000..42ce383 --- /dev/null +++ b/tools/log.py @@ -0,0 +1,29 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys + + +def set_logging(debug=True, timestamps=False): + _format = '{}%(levelname)s:%(message)s' + if timestamps: + _format = _format.format('%(asctime)-15s:') + else: + _format = _format.format('') + if debug: + level = logging.DEBUG + else: + level = logging.INFO + logging.basicConfig(stream=sys.stdout, level=level, format=_format) diff --git a/tools/package.py b/tools/package.py new file mode 100755 index 0000000..1a42af9 --- /dev/null +++ b/tools/package.py @@ -0,0 +1,53 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml + +from tools.statics import PACKAGES_CONFIG_PATH + + +class PackageConfigReader(object): + def __init__(self, config_file=PACKAGES_CONFIG_PATH): + self.config_file = config_file + self.config = None + self._read_file() + + def _read_file(self): + with open(self.config_file, 'r') as fp: + self.config = yaml.load(fp) + + def get(self, package_operation_type): + return self._get_packages_by_type(package_operation_type) + + def _get_packages_by_type(self, package_operation_type): + if self.config is None: + return [] + if package_operation_type == 'install': + return self._get_installed_packages() + elif package_operation_type == 'uninstall': + return self._get_uninstalled_packages() + raise Exception('Never here') + + def _get_uninstalled_packages(self): + return sorted([p for p in self.config if not self._is_install_package(p)]) + + def _get_installed_packages(self): + return sorted([p for p in self.config if self._is_install_package(p)]) + + def _is_install_package(self, package): + if self.config[package] is None: + return True + if self.config[package].get('uninstall') is True: + return False + return True diff --git a/tools/package_test.py b/tools/package_test.py new file mode 100755 index 0000000..2225d53 --- /dev/null +++ b/tools/package_test.py @@ -0,0 +1,44 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +from tools.package import PackageConfigReader + + +def test_packages(): + with mock.patch('tools.package.open', mock.mock_open(read_data=FAKE_PACKAGES_YAML)) as _: + config = PackageConfigReader('test-config') + assert config.get('install') == ['install-pkg-1', + 'install-pkg-2'] + assert config.get('uninstall') == ['remove-pkg-1', + 'remove-pkg-2'] + + +def test_empty(): + with mock.patch('tools.package.open', mock.mock_open(read_data="")) as _: + config = PackageConfigReader('test-config') + assert config.get('install') == [] + + +FAKE_PACKAGES_YAML = """ +remove-pkg-1: + uninstall: True +remove-pkg-2: + uninstall: True + +install-pkg-1: + uninstall: False +install-pkg-2: +""" diff --git a/tools/releasereader.py b/tools/releasereader.py new file mode 100755 index 0000000..982504a --- /dev/null +++ b/tools/releasereader.py @@ -0,0 +1,31 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +from tools.executor import Executor + + +class ReleaseReader(object): + def __init__(self, manifest_dir): + self.manifest_dir = manifest_dir + + def read_current_release(self): + return Executor(chdir=self.manifest_dir).run( + ["git", "describe", "--tags", "--abbrev=0"]).stdout + + def get_next_release_label(self): + current = self.read_current_release() + match = re.search(r'^(.+[\.-])(\d+)$', current) + return '{}{}'.format(match.group(1), (int(match.group(2)) + 1)) diff --git a/tools/releasereader_test.py b/tools/releasereader_test.py new file mode 100755 index 0000000..273b680 --- /dev/null +++ b/tools/releasereader_test.py @@ -0,0 +1,34 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import mock + +from tools.releasereader import ReleaseReader +from tools.executor import Result + + +@pytest.mark.parametrize('curr_release, next_release', [ + ('ABC_D19-0', 'ABC_D19-1'), + ('ABC_D19-19', 'ABC_D19-20'), + ('ABC_D19A-1', 'ABC_D19A-2'), + ('ABC_D19A-1.0', 'ABC_D19A-1.1'), + ('ABC_D20-0', 'ABC_D20-1') +]) +@mock.patch('tools.releasereader.Executor') +def test_next_release(mock_exec, curr_release, next_release): + mock_exec.return_value.run.return_value = Result(0, curr_release + '\n', '') + assert ReleaseReader('test-path').get_next_release_label() == next_release + mock_exec.assert_called_once_with(chdir='test-path') + mock_exec.return_value.run.assert_called_once_with(['git', 'describe', '--tags', '--abbrev=0']) diff --git a/tools/repository.py b/tools/repository.py new file mode 100755 index 0000000..19000e9 --- /dev/null +++ b/tools/repository.py @@ -0,0 +1,53 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from tools.buildconfig import BuildConfigParser +from tools.statics import BUILD_CONFIG_PATH +from tools.utils import validate_environ + + +class RepositoryConfig(object): + def __init__(self, ini_file=BUILD_CONFIG_PATH): + self.config = BuildConfigParser(ini_file=ini_file) + + def read_section(self, section): + repositories = [] + for repo_name, repo_value in self.config.items(section): + parts = repo_value.split('#') + repodata = dict(name=repo_name, baseurl=parts[0]) + for p in parts[1:]: + key, value = p.split('=', 1) + repodata[key] = value + repositories.append(repodata) + return repositories + + def read_sections(self, sections): + repositories = [] + for s in sections: + repositories += self.read_section(s) + return repositories + + @classmethod + def get_localrepo(cls, remote=False): + dirname = 'repo' + if remote: + validate_environ(['BUILD_URL']) + baseurl = os.path.join(os.environ['BUILD_URL'], 'artifact/results', dirname) + else: + validate_environ(['WORKSPACE']) + baseurl = 'file://' + \ + os.path.join(os.environ['WORKSPACE'], 'results', dirname) + return dict(name='localrepo', baseurl=baseurl) diff --git a/tools/rpm.py b/tools/rpm.py new file mode 100755 index 0000000..efd7365 --- /dev/null +++ b/tools/rpm.py @@ -0,0 +1,99 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + + +class RpmData(dict): + + @property + def name(self): + return self['Name'] + + @property + def epoch(self): + return self.get('Epoch', '0') + + @property + def version(self): + return self['Version'] + + @property + def release(self): + return self['Release'] + + @property + def arch(self): + return self['Architecture'] + + @property + def vendor(self): + return self.get('Vendor', '') + + @property + def license(self): + return self.get('License', '') + + def __str__(self): + return '{}-{}-{}.{}'.format(self.name, + self.version, + self.release, + self.arch) + + def is_same_package_as(self, other): + for attr in ['name', 'epoch', 'version', 'release', 'arch']: + if getattr(self, attr) != getattr(other, attr): + return False + return True + + +class RpmInfoParser(object): + """ + Parse 'rpm -qi' output + """ + + def parse_file(self, rpm_info_installed_file_path): + with open(rpm_info_installed_file_path, 'r') as f: + return self.parse_multiple(f.read()) + + def parse_multiple(self, rpm_info_output_multiple): + packages = [] + package_index = -1 + for line in rpm_info_output_multiple.splitlines(): + if re.match(r'^Name\s+:.*', line): + packages.append(line) + package_index += 1 + else: + packages[package_index] += '\n' + line + return [self.parse_package(pkg) for pkg in packages] + + @staticmethod + def parse_package(rpm_info_output): + result = RpmData() + current_key = None + colon_location = rpm_info_output.splitlines()[0].find(':') + matcher = re.compile(r'^([A-Z][A-Za-z0-9 ]{{{}}}):( ?| .+)$'.format(colon_location - 1)) + for line in rpm_info_output.splitlines(): + match = matcher.match(line) + if match: + parsed_key = match.group(1).rstrip() + parsed_value = match.group(2).strip() + result[parsed_key] = parsed_value + current_key = parsed_key + else: + if not result[current_key]: + result[current_key] = line + else: + result[current_key] = result[current_key] + '\n' + line + return result diff --git a/tools/rpm_info_installed.sample b/tools/rpm_info_installed.sample new file mode 100644 index 0000000..a6e2c63 --- /dev/null +++ b/tools/rpm_info_installed.sample @@ -0,0 +1,144 @@ +Name : python-futures +Version : 3.0.3 +Release : 1.el7 +Architecture: noarch +Install Date: Wed Feb 7 13:49:03 2018 +Group : Development/Libraries +Size : 88366 +License : BSD +Signature : RSA/SHA1, Fri Sep 1 15:07:30 2017, Key ID f9b9fee7764429e6 +Source RPM : python-futures-3.0.3-1.el7.src.rpm +Build Date : Tue Jul 28 12:27:11 2015 +Build Host : c1bj.rdu2.centos.org +Relocations : (not relocatable) +Packager : CBS +Vendor : CentOS +URL : https://github.com/agronholm/pythonfutures +Summary : Backport of the concurrent.futures package from Python 3.2 +Description : +The concurrent.futures module provides a high-level interface for +asynchronously executing callables. +Name : libgnome-keyring +Version : 3.8.0 +Release : 3.el7 +Architecture: x86_64 +Install Date: Wed Feb 7 13:50:17 2018 +Group : System Environment/Libraries +Size : 303457 +License : GPLv2+ and LGPLv2+ +Signature : RSA/SHA256, Fri Jul 4 02:48:34 2014, Key ID 24c6a8a7f4a80eb5 +Source RPM : libgnome-keyring-3.8.0-3.el7.src.rpm +Build Date : Tue Jun 10 08:58:52 2014 +Build Host : worker1.bsys.centos.org +Relocations : (not relocatable) +Packager : CentOS BuildSystem +Vendor : CentOS +URL : http://live.gnome.org/GnomeKeyring +Summary : Framework for managing passwords and other secrets +Description : +gnome-keyring is a program that keep password and other secrets for +users. The library libgnome-keyring is used by applications to integrate +with the gnome-keyring system. +Name : python-ldap3 +Version : 0.9.8.6 +Release : 2.el7 +Architecture: noarch +Install Date: Wed Feb 7 13:52:25 2018 +Group : Unspecified +Size : 3120182 +License : LGPLv2+ +Signature : RSA/SHA1, Fri Sep 1 15:07:38 2017, Key ID f9b9fee7764429e6 +Source RPM : python-ldap3-0.9.8.6-2.el7.src.rpm +Build Date : Fri Nov 20 16:28:01 2015 +Build Host : c1bk.rdu2.centos.org +Relocations : (not relocatable) +Packager : CBS +Vendor : CentOS +URL : https://pypi.python.org/pypi/ldap3/0.9.8.6 +Summary : Strictly RFC 4511 conforming LDAP V3 pure Python client +Description : +python-ldap3 is a strictly RFC 4511 conforming LDAP V3 pure Python client. +The same codebase works with Python, Python 3, PyPy and PyPy3. +Name : python-dracclient +Version : 1.3.1 +Release : 0.20170926083913.8f368ed.el7.centos +Architecture: noarch +Install Date: Wed Feb 7 13:49:30 2018 +Group : Unspecified +Size : 1325637 +License : ASL 2.0 +Signature : (none) +Source RPM : python-dracclient-1.3.1-0.20170926083913.8f368ed.el7.centos.src.rpm +Build Date : Tue Sep 26 08:39:32 2017 +Build Host : trunk-primary.rdoproject.org.rdocloud +Relocations : (not relocatable) +URL : http://github.com/openstack/python-dracclient +Summary : Library for managing machines with Dell iDRAC cards. +Description : +Library for managing machines with Dell iDRAC cards. +Name : dialog +Version : 1.2 +Release : 4.20130523.el7 +Architecture: x86_64 +Install Date: Wed Feb 7 13:49:48 2018 +Group : Applications/System +Size : 517263 +License : LGPLv2 +Signature : RSA/SHA256, Fri Jul 4 01:08:16 2014, Key ID 24c6a8a7f4a80eb5 +Source RPM : dialog-1.2-4.20130523.el7.src.rpm +Build Date : Tue Jun 10 01:38:26 2014 +Build Host : worker1.bsys.centos.org +Relocations : (not relocatable) +Packager : CentOS BuildSystem +Vendor : CentOS +URL : http://invisible-island.net/dialog/dialog.html +Summary : A utility for creating TTY dialog boxes +Description : +Dialog is a utility that allows you to show dialog boxes (containing +questions or messages) in TTY (text mode) interfaces. Dialog is called +from within a shell script. The following dialog boxes are implemented: +yes/no, menu, input, message, text, info, checklist, radiolist, and +gauge. + +Install dialog if you would like to create TTY dialog boxes. +Name : filesystem +Version : 3.2 +Release : 21.el7 +Architecture: x86_64 +Install Date: Wed Feb 7 13:44:24 2018 +Group : System Environment/Base +Size : 0 +License : Public Domain +Signature : RSA/SHA256, Sun Nov 20 17:43:00 2016, Key ID 24c6a8a7f4a80eb5 +Source RPM : filesystem-3.2-21.el7.src.rpm +Build Date : Sat Nov 5 15:39:14 2016 +Build Host : worker1.bsys.centos.org +Relocations : (not relocatable) +Packager : CentOS BuildSystem +Vendor : CentOS +URL : https://fedorahosted.org/filesystem +Summary : The basic directory layout for a Linux system +Description : +The filesystem package is one of the basic packages that is installed +on a Linux system. Filesystem contains the basic directory layout +for a Linux operating system, including the correct permissions for +the directories. +Name : python2-tenacity +Version : 4.4.0 +Release : 1.el7 +Architecture: noarch +Install Date: Wed Feb 7 13:49:04 2018 +Group : Unspecified +Size : 159111 +License : ASL 2.0 +Signature : RSA/SHA1, Fri Sep 1 15:11:29 2017, Key ID f9b9fee7764429e6 +Source RPM : python-tenacity-4.4.0-1.el7.src.rpm +Build Date : Sat Aug 5 13:47:41 2017 +Build Host : c1bd.rdu2.centos.org +Relocations : (not relocatable) +Packager : CBS +Vendor : CentOS +URL : https://github.com/jd/tenacity +Summary : Tenacity is a general purpose retrying library +Description : + Tenacity is a general purpose retrying library \ No newline at end of file diff --git a/tools/rpm_test.py b/tools/rpm_test.py new file mode 100755 index 0000000..d875df3 --- /dev/null +++ b/tools/rpm_test.py @@ -0,0 +1,50 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +import pytest + +from tools.rpm import RpmInfoParser +from tools.rpm_test_data import \ + bash_expected, conntrack_tools_expected, cpp_expected, usbredir_expected, perl_compress_expected +from tools.test_data_rpm import \ + bash_rpm_info, cpp_rpm_info, conntrack_tools_rpm_info, usbredir_rpm_info, perl_compress_rpm_info + + +@pytest.mark.parametrize('rpm_info, expected_output', [ + (bash_rpm_info, bash_expected), + (conntrack_tools_rpm_info, conntrack_tools_expected), + (cpp_rpm_info, cpp_expected), + (usbredir_rpm_info, usbredir_expected), + (perl_compress_rpm_info, perl_compress_expected) +]) +def test_parse_package(rpm_info, expected_output): + parsed = RpmInfoParser().parse_package(rpm_info) + assert parsed == expected_output + + +def test_parse_multiple(): + parsed = RpmInfoParser().parse_multiple('\n'.join([bash_rpm_info, conntrack_tools_rpm_info])) + assert parsed == [bash_expected, conntrack_tools_expected] + + +def test_parse_file(): + test_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'rpm_info_installed.sample') + parsed = RpmInfoParser().parse_file(test_file) + with open(test_file, 'r') as f: + expected_rpms = re.findall(r'^Name\s+:.*$', f.read(), re.MULTILINE) + assert len(parsed) == len(expected_rpms) diff --git a/tools/rpm_test_data.py b/tools/rpm_test_data.py new file mode 100755 index 0000000..d89daca --- /dev/null +++ b/tools/rpm_test_data.py @@ -0,0 +1,179 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=invalid-name,line-too-long + + +bash_expected = { + 'Name': 'bash', + 'Version': '4.2.46', + 'Release': '21.el7_3', + 'Architecture': 'x86_64', + 'Install Date': 'Thu 11 Jan 2018 12:32:51 PM EET', + 'Group': 'System Environment/Shells', + 'Size': '3663714', + 'License': 'GPLv3+', + 'Signature': 'RSA/SHA256, Wed 07 Dec 2016 02:11:28 AM EET, Key ID 24c6a8a7f4a80eb5', + 'Source RPM': 'bash-4.2.46-21.el7_3.src.rpm', + 'Build Date': 'Wed 07 Dec 2016 01:21:54 AM EET', + 'Build Host': 'c1bm.rdu2.centos.org', + 'Relocations': '(not relocatable)', + 'Packager': 'CentOS BuildSystem ', + 'Vendor': 'CentOS', + 'URL': 'http://www.gnu.org/software/bash', + 'Summary': 'The GNU Bourne Again shell', + 'Description': '\n'.join( + ['The GNU Bourne Again shell (Bash) is a shell or command language', + 'interpreter that is compatible with the Bourne shell (sh). Bash', + 'incorporates useful features from the Korn shell (ksh) and the C shell', + '(csh). Most sh scripts can be run by bash without modification.']) +} + +conntrack_tools_expected = { + 'Name': 'conntrack-tools', + 'Version': '1.4.4', + 'Release': '3.el7_3', + 'Architecture': 'x86_64', + 'Install Date': 'Thu 11 Jan 2018 12:39:20 PM EET', + 'Group': 'System Environment/Base', + 'Size': '562826', + 'License': 'GPLv2', + 'Signature': 'RSA/SHA256, Thu 29 Jun 2017 03:36:05 PM EEST, Key ID 24c6a8a7f4a80eb5', + 'Source RPM': 'conntrack-tools-1.4.4-3.el7_3.src.rpm', + 'Build Date': 'Thu 29 Jun 2017 03:18:42 AM EEST', + 'Build Host': 'c1bm.rdu2.centos.org', + 'Relocations': '(not relocatable)', + 'Packager': 'CentOS BuildSystem ', + 'Vendor': 'CentOS', + 'URL': 'http://netfilter.org', + 'Summary': 'Manipulate netfilter connection tracking table and run High Availability', + 'Description': '\n'.join([ + 'With conntrack-tools you can setup a High Availability cluster and', + 'synchronize conntrack state between multiple firewalls.', + '', + 'The conntrack-tools package contains two programs:', + '- conntrack: the command line interface to interact with the connection', + ' tracking system.', + '- conntrackd: the connection tracking userspace daemon that can be used to', + ' deploy highly available GNU/Linux firewalls and collect', + ' statistics of the firewall use.', + '', + 'conntrack is used to search, list, inspect and maintain the netfilter', + 'connection tracking subsystem of the Linux kernel.', + 'Using conntrack, you can dump a list of all (or a filtered selection of)', + 'currently tracked connections, delete connections from the state table,', + 'and even add new ones.', + 'In addition, you can also monitor connection tracking events, e.g.', + 'show an event message (one line) per newly established connection.' + ]) +} + +cpp_expected = { + 'Name': 'cpp', + 'Version': '4.8.5', + 'Release': '11.el7', + 'Architecture': 'x86_64', + 'Install Date': 'Thu 11 Jan 2018 12:37:55 PM EET', + 'Group': 'Development/Languages', + 'Size': '15632501', + 'License': 'GPLv3+ and GPLv3+ with exceptions and GPLv2+ with exceptions and LGPLv2+ and BSD', + 'Signature': 'RSA/SHA256, Sun 20 Nov 2016 07:27:00 PM EET, Key ID 24c6a8a7f4a80eb5', + 'Source RPM': 'gcc-4.8.5-11.el7.src.rpm', + 'Build Date': 'Fri 04 Nov 2016 06:01:22 PM EET', + 'Build Host': 'worker1.bsys.centos.org', + 'Relocations': '(not relocatable)', + 'Packager': 'CentOS BuildSystem ', + 'Vendor': 'CentOS', + 'URL': 'http://gcc.gnu.org', + 'Summary': 'The C Preprocessor', + 'Description': + '\n'.join(['Cpp is the GNU C-Compatible Compiler Preprocessor.', + 'Cpp is a macro processor which is used automatically', + 'by the C compiler to transform your program before actual', + 'compilation. It is called a macro processor because it allows', + 'you to define macros, abbreviations for longer', + 'constructs.', + '', + 'The C preprocessor provides four separate functionalities: the', + 'inclusion of header files (files of declarations that can be', + 'substituted into your program); macro expansion (you can define macros,', + 'and the C preprocessor will replace the macros with their definitions', + 'throughout the program); conditional compilation (using special', + 'preprocessing directives, you can include or exclude parts of the', + 'program according to various conditions); and line control (if you use', + 'a program to combine or rearrange source files into an intermediate', + 'file which is then compiled, you can use line control to inform the', + 'compiler about where each source line originated).', + '', + 'You should install this package if you are a C programmer and you use', + 'macros.']), +} + +usbredir_expected = { + 'Name': 'usbredir', + 'Version': '0.7.1', + 'Release': '1.el7', + 'Architecture': 'x86_64', + 'Install Date': 'Wed Feb 7 13:49:24 2018', + 'Group': 'System Environment/Libraries', + 'Size': '108319', + 'License': 'LGPLv2+', + 'Signature': 'RSA/SHA256, Sun Nov 20 20:56:49 2016, Key ID 24c6a8a7f4a80eb5', + 'Source RPM': 'usbredir-0.7.1-1.el7.src.rpm', + 'Build Date': 'Sat Nov 5 18:33:15 2016', + 'Build Host': 'worker1.bsys.centos.org', + 'Relocations': '(not relocatable)', + 'Packager': 'CentOS BuildSystem ', + 'Vendor': 'CentOS', + 'URL': 'http://spice-space.org/page/UsbRedir', + 'Summary': 'USB network redirection protocol libraries', + 'Description': '\n'.join([ + 'The usbredir libraries allow USB devices to be used on remote and/or virtual', + 'hosts over TCP. The following libraries are provided:', + '', + 'usbredirparser:', + 'A library containing the parser for the usbredir protocol', + '', + 'usbredirhost:', + 'A library implementing the USB host side of a usbredir connection.', + 'All that an application wishing to implement a USB host needs to do is:', + '* Provide a libusb device handle for the device', + '* Provide write and read callbacks for the actual transport of usbredir data', + '* Monitor for usbredir and libusb read/write events and call their handlers']) +} + +perl_compress_expected = { + 'Name': 'perl-Compress-Raw-Zlib', + 'Epoch': '1', + 'Version': '2.061', + 'Release': '4.el7', + 'Architecture': 'x86_64', + 'Install Date': 'Sat Jan 26 20:05:50 2019', + 'Group': 'Development/Libraries', + 'Size': '139803', + 'License': 'GPL+ or Artistic', + 'Signature': 'RSA/SHA256, Fri Jul 4 04:15:33 2014, Key ID 24c6a8a7f4a80eb5', + 'Source RPM': 'perl-Compress-Raw-Zlib-2.061-4.el7.src.rpm', + 'Build Date': 'Tue Jun 10 01:12:08 2014', + 'Build Host': 'worker1.bsys.centos.org', + 'Relocations': '(not relocatable)', + 'Packager': 'CentOS BuildSystem ', + 'Vendor': 'CentOS', + 'URL': 'http://search.cpan.org/dist/Compress-Raw-Zlib/', + 'Summary': 'Low-level interface to the zlib compression library', + 'Description': '\n'.join([ + 'The Compress::Raw::Zlib module provides a Perl interface to the zlib', + 'compression library, which is used by IO::Compress::Zlib.']), + 'Obsoletes': '' +} diff --git a/tools/script/__init__.py b/tools/script/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/script/ci_build_diff.py b/tools/script/ci_build_diff.py new file mode 100755 index 0000000..c9dc567 --- /dev/null +++ b/tools/script/ci_build_diff.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys +import argparse +from operator import itemgetter +from pprint import pformat + +from tools.convert import CsvConverter +from tools.io import read_json, write_to, write_json +from tools.log import set_logging + + +class SwComponent(dict): + + @property + def name(self): + return self['Name'] + + @property + def version(self): + return self['Version'] + + @property + def foss_id(self): + return self.name, self.version, self.get('Release') + + def __str__(self): + return '{}:{}({})'.format(self.name, + self.version, + self['Source RPM']) + + +class BuildDiffReader(object): + + def __init__(self): + self.changes = {} + self.summary = {} + + @staticmethod + def get_component_names(data): + return [i['Name'] for i in data] + + @staticmethod + def get_components(name, components): + out = [] + for component in components: + if component['Name'] == name: + out.append(SwComponent(component)) + return sorted(out, key=itemgetter('Name', 'Version', 'Source RPM')) + + def read(self, json_old, json_new): + old_build = read_json(json_old) + new_build = read_json(json_new) + self.summary['input'] = {} + self.summary['input']['from'] = len(old_build) + self.summary['input']['to'] = len(new_build) + self.changes = self.read_build_diff(old_build, new_build) + self.summary['output'] = self._generate_summary(self.changes) + + @staticmethod + def _generate_summary(changes): + summary = {} + summary['counts'] = changes['counts'] + summary['added'] = {name: [str(c) for c in compos] for name, compos in + changes['added'].items()} + summary['removed'] = {name: [str(c) for c in compos] for name, compos in + changes['removed'].items()} + summary['changed'] = {name: {'old': [str(c) for c in change['old']], + 'new': [str(c) for c in change['new']]} for name, change in + changes['changed'].items()} + return summary + + def read_build_diff(self, old_build, new_build): + old_names = self.get_component_names(old_build) + logging.debug('Old names: {}'.format(old_names)) + new_names = self.get_component_names(new_build) + logging.debug('New names: {}'.format(new_names)) + added = {n: self.get_components(n, new_build) for n in set(new_names) - set(old_names)} + self._mark('[MARK] added', [j for i in added.values() for j in i]) + removed = {n: self.get_components(n, old_build) for n in set(old_names) - set(new_names)} + self._mark('[MARK] removed', [j for i in removed.values() for j in i]) + changed = {} + for n in set(old_names) & set(new_names): + old_components = self.get_components(n, old_build) + new_components = self.get_components(n, new_build) + if sorted([i.foss_id for i in old_components]) != \ + sorted([i.foss_id for i in new_components]): + changed[n] = {'old': old_components, 'new': new_components} + self._mark('[MARK] changed old', changed[n]['old']) + self._mark('[MARK] changed new', changed[n]['new']) + return dict(counts=dict(added=len(added), + changed=len(changed), + removed=len(removed)), + added=added, + removed=removed, + changed=changed) + + @staticmethod + def _mark(title, components): + logging.debug( + '[MARK] {}: {}'.format(title, pformat([i.foss_id for i in components]))) + + @staticmethod + def _get_csv_cells(name, old_components, new_components): + cells = dict(name=name) + if old_components: + cells.update(dict(old_components='\n'.join([str(i) for i in old_components]), + old_srpms='\n'.join([i['Source RPM'] for i in old_components]), + old_licenses='\n'.join( + [i.get('License', 'Unknown') for i in old_components]))) + if new_components: + cells.update(dict(new_components='\n'.join([str(i) for i in new_components]), + new_srpms='\n'.join([i['Source RPM'] for i in new_components]), + new_licenses='\n'.join( + [i.get('License', 'Unknown') for i in new_components]))) + return cells + + def write_csv(self, path): + data = [] + for name, components in self.changes['added'].items(): + data += [self._get_csv_cells(name, [], components)] + + for name, components in self.changes['removed'].items(): + data += [self._get_csv_cells(name, components, [])] + + for name, components in self.changes['changed'].items(): + data += [self._get_csv_cells(name, components['old'], components['new'])] + + csv = CsvConverter(sorted(data, key=itemgetter('name')), + preferred_field_order=['name', + 'old_components', 'old_srpms', 'old_licenses', + 'new_components', 'new_srpms', 'new_licenses'], + escape_newlines=False) + write_to(path, csv.convert_to_ms_excel()) + + +def parse(args): + parser = argparse.ArgumentParser(description='Outputs RPM changes between two CI builds') + parser.add_argument('--verbose', '-v', action='store_true', + help='More verbose logging') + parser.add_argument('components_json_1', + help='Components json file path (CI build artifact)') + parser.add_argument('components_json_2', + help='Components json file path (CI build artifact)') + parser.add_argument('--output-json', + help='output to json file') + parser.add_argument('--output-csv', + help='output to $MS csv file') + return parser.parse_args(args) + + +def main(input_args): + args = parse(input_args) + set_logging(debug=args.verbose) + x = BuildDiffReader() + x.read(args.components_json_1, args.components_json_2) + logging.info('----- SUMMARY ------\n{}'.format(pformat(x.summary))) + if args.output_json: + write_json(args.output_json, x.changes) + if args.output_csv: + x.write_csv(args.output_csv) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/tools/script/ci_build_diff_test.py b/tools/script/ci_build_diff_test.py new file mode 100755 index 0000000..81a1e76 --- /dev/null +++ b/tools/script/ci_build_diff_test.py @@ -0,0 +1,249 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long,invalid-name +import json + +from tools.script.ci_build_diff import main +from tools.script.ci_build_diff_test_data import component_added, component_changed_old, \ + component_changed_new, component_removed, caas_grafana1, caas_grafana2, caas_grafana2_sub, \ + caas_grafana1_sub, caas_grafana1_v2, caas_grafana1_v2sub, caas_grafana1_r2, \ + caas_grafana1_r2sub, grafana_v1, grafana_v2, caas_grafana3, caas_grafana3_sub, caas_abc1_sub, \ + caas_abc1, caas_abc1_r2, caas_abc1_sub_r2, abc1_v2, abc1, abc2, abc3, \ + caas_grafana1_sub_new_field + + +def test_no_output_args(tmpdir): + input_old, input_new = _gen_input_json(tmpdir) + main([str(input_old), str(input_new)]) + + +def test_no_changes(tmpdir): + input_old, _ = _gen_input_json(tmpdir) + output_csv = tmpdir.join('diff.csv') + output_json = tmpdir.join('diff.json') + main([str(input_old), str(input_old), + '--output-csv', str(output_csv), + '--output-json', str(output_json)]) + assert output_csv.read() == 'sep=,' + assert json.loads(output_json.read()) == {"added": {}, + "changed": {}, + "removed": {}, + "counts": { + "added": 0, + "changed": 0, + "removed": 0 + }} + + +def test_no_changes_if_json_field_is_added(tmpdir): + """ + Make sure diff is ok e.g. when new build has new field in json + """ + _assert_json_out(tmpdir, + [caas_grafana1_sub], + [caas_grafana1_sub_new_field], + {"added": {}, + "changed": {}, + "removed": {}, + "counts": { + "added": 0, + "changed": 0, + "removed": 0 + }}) + + +def test_multiple_rpms(tmpdir): + _assert_json_out(tmpdir, + [component_changed_old, component_removed], + [component_changed_new, component_added], + {"added": {component_added['Name']: [component_added]}, + "changed": {component_changed_old['Name']: {'old': [component_changed_old], + 'new': [component_changed_new]}}, + "removed": {component_removed['Name']: [component_removed]}, + "counts": { + "added": 1, + "changed": 1, + "removed": 1 + }}) + + +def test_component_name_change(tmpdir): + """ + Sub-component does not change although RPM containing it changes + """ + _assert_json_out(tmpdir, + [caas_grafana1, caas_grafana1_sub], + [caas_grafana2, caas_grafana2_sub], + {"added": {caas_grafana2['Name']: [caas_grafana2]}, + "changed": {}, + "removed": {caas_grafana1['Name']: [caas_grafana1]}, + "counts": { + "added": 1, + "changed": 0, + "removed": 1 + }}) + + +def test_component_version_change(tmpdir): + _assert_json_out(tmpdir, + [caas_grafana1, caas_grafana1_sub], + [caas_grafana1_v2, caas_grafana1_v2sub], + {"added": {}, + "changed": {caas_grafana1['Name']: {'old': [caas_grafana1], + 'new': [caas_grafana1_v2]}, + caas_grafana1_sub['Name']: {'old': [caas_grafana1_sub], + 'new': [caas_grafana1_v2sub]}}, + "removed": {}, + "counts": { + "added": 0, + "changed": 2, + "removed": 0 + }}) + + +def test_component_release_change(tmpdir): + _assert_json_out(tmpdir, + [caas_grafana1, caas_grafana1_sub], + [caas_grafana1_r2, caas_grafana1_r2sub], + {"added": {}, + "changed": {caas_grafana1['Name']: {'old': [caas_grafana1], + 'new': [caas_grafana1_r2]}}, + "removed": {}, + "counts": { + "added": 0, + "changed": 1, + "removed": 0 + }}) + + +def test_same_name_component_added(tmpdir): + _assert_json_out(tmpdir, + [caas_grafana1, caas_grafana1_sub], + [caas_grafana1, caas_grafana1_sub, grafana_v1], + {"added": {}, + "changed": {grafana_v1['Name']: {'old': [caas_grafana1_sub], + 'new': [caas_grafana1_sub, grafana_v1]}}, + "removed": {}, + "counts": { + "added": 0, + "changed": 1, + "removed": 0 + }}) + + +def test_same_name_component_removed(tmpdir): + _assert_json_out(tmpdir, + [caas_grafana1, caas_grafana1_sub, grafana_v1], + [caas_grafana1, caas_grafana1_sub], + {"added": {}, + "changed": {grafana_v1['Name']: {'old': [caas_grafana1_sub, grafana_v1], + 'new': [caas_grafana1_sub]}}, + "removed": {}, + "counts": { + "added": 0, + "changed": 1, + "removed": 0 + }}) + + +def test_same_name_component_changed(tmpdir): + _assert_json_out(tmpdir, + [caas_grafana1, caas_grafana1_sub, grafana_v1], + [caas_grafana1, caas_grafana1_sub, grafana_v2], + {"added": {}, + "changed": {grafana_v1['Name']: {'old': [caas_grafana1_sub, grafana_v1], + 'new': [caas_grafana1_sub, grafana_v2]}}, + "removed": {}, + "counts": { + "added": 0, + "changed": 1, + "removed": 0 + }}) + + +def test_epic(tmpdir): + _assert_json_out(tmpdir, + [caas_grafana1, caas_grafana1_sub, + caas_grafana2, caas_grafana2_sub, + grafana_v1], + [caas_grafana1_r2, caas_grafana1_r2sub, + grafana_v2, + caas_grafana3, caas_grafana3_sub], + {"added": {caas_grafana3['Name']: [caas_grafana3]}, + "changed": {caas_grafana1['Name']: {'old': [caas_grafana1], + 'new': [caas_grafana1_r2]}, + grafana_v1['Name']: { + 'old': [caas_grafana1_sub, caas_grafana2_sub, grafana_v1], + 'new': [caas_grafana1_r2sub, caas_grafana3_sub, grafana_v2]}}, + "removed": {caas_grafana2['Name']: [caas_grafana2]}, + "counts": { + "added": 1, + "changed": 2, + "removed": 1 + }}) + + +def _assert_json_out(tmpdir, from_build, to_build, expected_output): + input_old, input_new = _gen_input_json(tmpdir, from_build, to_build) + output_json = tmpdir.join('diff.json') + main([str(input_old), str(input_new), '--output-json', str(output_json)]) + assert json.loads(output_json.read()) == expected_output + + +def test_csv(tmpdir): + input_old, input_new = _gen_input_json(tmpdir, + [caas_abc1, caas_abc1_sub, abc1, abc2], + [caas_abc1_r2, caas_abc1_sub_r2, abc1_v2, abc3]) + output_csv = tmpdir.join('diff.csv') + main([str(input_old), str(input_new), '--output-csv', str(output_csv)]) + rows = [['name', + 'old_components', 'old_srpms', 'old_licenses', + 'new_components', 'new_srpms', 'new_licenses'], + ['abc', + 'abc:v1(abc-v1-r1.src.rpm)\nabc:v1(caas-abc-v1-r1.src.rpm)', + 'abc-v1-r1.src.rpm\ncaas-abc-v1-r1.src.rpm', + 'GPL\nUnknown', + 'abc:v1(caas-abc-v1-r2.src.rpm)\nabc:v2(abc-v2-r1.src.rpm)', + 'caas-abc-v1-r2.src.rpm\nabc-v2-r1.src.rpm', + 'Unknown\nGPL'], + ['abc2', + 'abc2:v1(abc2-v1-r1.src.rpm)', 'abc2-v1-r1.src.rpm', 'GPL', + 'None', 'None', 'None'], + ['abc3', + 'None', 'None', 'None', + 'abc3:v1(abc3-v1-r1.src.rpm)', 'abc3-v1-r1.src.rpm', 'GPL'], + ['caas-abc', + 'caas-abc:v1(caas-abc-v1-r1.src.rpm)', 'caas-abc-v1-r1.src.rpm', 'Commercial', + 'caas-abc:v1(caas-abc-v1-r2.src.rpm)', 'caas-abc-v1-r2.src.rpm', 'Commercial']] + expected_csv = '\r\n'.join(['sep=,'] + [_get_csv_row(row) for row in rows]) + assert output_csv.read() == expected_csv + + +def _get_csv_row(items): + return ','.join(['"{}"'.format(i) for i in items]) + + +def _gen_input_json(tmpdir, _from=None, _to=None): + if _from is None: + _from = [component_changed_old, component_removed] + if _to is None: + _to = [component_changed_new, component_added] + input_old = tmpdir.join('input_old.json') + input_new = tmpdir.join('input_new.json') + json_old = json.dumps(_from) + json_new = json.dumps(_to) + input_old.write(json_old) + input_new.write(json_new) + return input_old, input_new diff --git a/tools/script/ci_build_diff_test_data.py b/tools/script/ci_build_diff_test_data.py new file mode 100755 index 0000000..e3df7a3 --- /dev/null +++ b/tools/script/ci_build_diff_test_data.py @@ -0,0 +1,281 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long,invalid-name +import re + +from copy import deepcopy + + +def _copy(d, pattern, repl): + x = deepcopy(d) + for k, v in x.items(): + if isinstance(v, str): + x[k] = re.sub(pattern, repl, v) + return x + + +caas_grafana1 = { + "Architecture": "noarch", + "Build Date": "Wed May 30 11:39:05 2018", + "Build Host": "crf-dev", + "Crypto capable": False, + "Description": "This is grafana, this is SPARTAAAAA!", + "FOSS": "No", + "From repo": "caas-artifactory", + "Group": "Unspecified", + "Install Date": "Sun Nov 18 22:15:41 2018", + "Is sane": True, + "License": "Commercial", + "Name": "caas.grafana", + "Obsoletes": "", + "Release": "1.el7.centos", + "Relocations": "(not relocatable)", + "Repo": "installed", + "Repo data": { + "baseurl": "http://files/20181023", + "name": "caas-artifactory" + }, + "Signature": "(none)", + "Size": "109683450", + "Source RPM": "caas.grafana-4.4.1.9-1.el7.centos.src.rpm", + "Source repo data": { + "baseurl": "http://files/20181023", + "name": "caas-artifactory" + }, + "Source to be delivered": "No", + "Summary": "caasgrafana", + "Vendor": "Something", + "Version": "4.4.1.9" +} + +caas_grafana1_sub = { + "Name": "grafana", + "Version": "4.4.1.9", + "Source RPM": "caas.grafana-4.4.1.9-1.el7.centos.src.rpm", + "Source URL": "https://some/grafana/url", + "FOSS": "yes" +} + +caas_grafana1_sub_new_field = deepcopy(caas_grafana1_sub) +caas_grafana1_sub_new_field['ABC'] = True + +caas_grafana2 = _copy(caas_grafana1, 'caas.grafana', 'caas.grafana2') +caas_grafana2_sub = _copy(caas_grafana1_sub, 'caas.grafana', 'caas.grafana2') + +caas_grafana3 = _copy(caas_grafana1, 'caas.grafana', 'caas.grafana3') +caas_grafana3_sub = _copy(caas_grafana1_sub, 'caas.grafana', 'caas.grafana3') + +caas_grafana1_v2 = _copy(caas_grafana1, '4.4.1.9', '4.4.1.10') +caas_grafana1_v2sub = _copy(caas_grafana1_sub, '4.4.1.9', '4.4.1.10') + +caas_grafana1_r2 = _copy(caas_grafana1, '1.el7.centos', '2.el7.centos') +caas_grafana1_r2sub = _copy(caas_grafana1_sub, '1.el7.centos', '2.el7.centos') + +caas_abc1 = { + "License": "Commercial", + "Name": "caas-abc", + "Version": "v1", + "Release": "r1", + "Source RPM": "caas-abc-v1-r1.src.rpm", +} + +caas_abc1_sub = { + "Name": "abc", + "Version": "v1", + "Source RPM": "caas-abc-v1-r1.src.rpm", +} +caas_abc1_r2 = _copy(caas_abc1, 'r1', 'r2') +caas_abc1_sub_r2 = _copy(caas_abc1_sub, 'r1', 'r2') + +abc1 = { + "License": "GPL", + "Name": "abc", + "Version": "v1", + "Release": "r1", + "Source RPM": "abc-v1-r1.src.rpm", +} +abc1_v2 = _copy(abc1, 'v1', 'v2') +abc2 = _copy(abc1, 'abc', 'abc2') +abc3 = _copy(abc1, 'abc', 'abc3') + +component_added = { + 'Architecture': 'noarch', + 'Build Date': 'Sun Nov 11 12:54:39 2018', + 'Build Host': 'build-7.novalocal', + 'Description': 'This RPM contains configuration management openstack configuration override ' + 'plugin', + 'FOSS': 'No', + 'From repo': 'localrepo', + 'Group': 'Unspecified', + 'Install Date': 'Tue Nov 13 19:14:29 2018', + 'Is sane': True, + 'License': 'Commercial', + 'Name': 'openstack-config-overrides-validator', + 'Obsoletes': '', + 'Packager': 'Something', + 'Release': '1.el7.centos', + 'Relocations': '(not relocatable)', + 'Repo': 'installed', + 'Repo data': { + 'baseurl': 'https://jenkins/ci-build/2490/artifact/results/repo', + 'name': 'localrepo'}, + 'Signature': '(none)', + 'Size': '3097', + 'Source RPM': 'openstack-config-overrides-validator-c2.gd1b7aec-1.el7.centos.src.rpm', + 'Source repo data': { + 'baseurl': 'https://jenkins/ci-build/2490/artifact/results/src_repo', + 'name': 'localrepo'}, + 'Source to be delivered': 'No', + 'Summary': 'Openstack configuration override CM validator plugin.', + 'Vendor': 'Something', + 'Version': 'c2.gd1b7aec' +} + +component_removed = { + 'Architecture': 'x86_64', + 'Build Date': 'Thu Aug 16 14:46:11 2018', + 'Build Host': 'x86-01.bsys.centos.org', + 'Description': 'The fence-agents-ibmblade package contains a fence agent for IBM BladeCenter ' + 'devices that are accessed via the SNMP protocol.', + 'FOSS': 'Undefined', + 'From repo': 'purkki-centos-updates', + 'Group': 'System Environment/Base', + 'Install Date': 'Wed Nov 7 21:20:01 2018', + 'Is sane': False, + 'License': 'GPLv2+ and LGPLv2+', + 'Name': 'fence-agents-ibmblade', + 'Obsoletes': 'fence-agents,', + 'Packager': 'CentOS BuildSystem ', + 'Release': '86.el7_5.3', + 'Relocations': '(not relocatable)', + 'Repo': 'installed', + 'Repo data': { + 'baseurl': 'http://purkki/mirror/centos/snapshot/20181024/7/updates/x86_64/', + 'exclude': 'libgudev1 httpd httpd-devel systemd-libs.i686 resource-agents ' + 'dhcp-libs dhclient dhcp-common php-fpm php-common php-cli php', + 'name': 'purkki-centos-updates'}, + 'Signature': 'RSA/SHA256, Mon Aug 20 14:15:17 2018, Key ID 24c6a8a7f4a80eb5', + 'Size': '3898', + 'Source RPM': 'fence-agents-4.0.11-86.el7_5.3.src.rpm', + 'Source repo data': { + 'baseurl': 'http://purkki/mirror/centos/snapshot/20181024/7/updates/Source/', + 'name': 'purkki-centos-updates'}, + 'Source to be delivered': 'Undefined', + 'Summary': 'Fence agent for IBM BladeCenter', + 'URL': 'https://github.com/ClusterLabs/fence-agents', + 'Vendor': 'CentOS', + 'Version': '4.0.11' +} +component_changed_old = { + 'Architecture': 'noarch', + 'Build Date': 'Fri Oct 19 00:22:00 2018', + 'Build Host': 'f32725a719ce4c82a53b44644dfd2718', + 'Description': 'This RPM contains source code for the Authentication, Authorization and ' + 'Accounting cli', + 'FOSS': 'No', + 'From repo': 'localrepo', + 'Group': 'Unspecified', + 'Install Date': 'Wed Nov 7 21:22:14 2018', + 'Is sane': True, + 'License': 'Commercial', + 'Name': 'aaacli', + 'Obsoletes': '', + 'Packager': 'Something', + 'Release': '2.el7.centos', + 'Relocations': '(not relocatable)', + 'Repo': 'installed', + 'Repo data': { + 'baseurl': 'https://jenkins/ci-build/2432/artifact/results/repo', + 'name': 'localrepo'}, + 'Signature': '(none)', + 'Size': '43968', + 'Source RPM': 'aaacli-c12.gc62d348-2.el7.centos.src.rpm', + 'Source repo data': { + 'baseurl': 'https://jenkins/ci-build/2432/artifact/results/src_repo', + 'name': 'localrepo'}, + 'Source to be delivered': 'No', + 'Summary': 'Authentication, Authorization and Accounting Command Line Interface', + 'Vendor': 'Something', + 'Version': 'c12.gc62d348' +} + +component_changed_new = { + 'Architecture': 'noarch', + 'Build Date': 'Thu Nov 8 05:05:07 2018', + 'Build Host': 'build-3.novalocal', + 'Description': 'This RPM contains source code for the Authentication, ' + 'Authorization and Accounting cli', + 'FOSS': 'No', + 'From repo': 'localrepo', + 'Group': 'Unspecified', + 'Install Date': 'Tue Nov 13 19:09:30 2018', + 'Is sane': True, + 'License': 'Commercial', + 'Name': 'aaacli', + 'Obsoletes': '', + 'Packager': 'Something', + 'Release': '1.el7.centos', + 'Relocations': '(not relocatable)', + 'Repo': 'installed', + 'Repo data': { + 'baseurl': 'https://jenkins/ci-build/2490/artifact/results/repo', + 'name': 'localrepo'}, + 'Signature': '(none)', + 'Size': '44034', + 'Source RPM': 'aaacli-c13.gcb6b490-1.el7.centos.src.rpm', + 'Source repo data': { + 'baseurl': 'https://jenkins/ci-build/2490/artifact/results/src_repo', + 'name': 'localrepo'}, + 'Source to be delivered': 'No', + 'Summary': 'Authentication, Authorization and Accounting Command Line Interface', + 'Vendor': 'Something', + 'Version': 'c13.gcb6b490' +} + +grafana_v1 = { + 'Architecture': 'x86_64', + 'Build Date': 'Mon Sep 17 14:30:25 2018', + 'Build Host': 'd8fb00edf57f4254bb45073a941929ff', + 'Description': 'Grafana is an open source, feature rich metrics dashboard and graph ' + 'editor for\nGraphite, InfluxDB & OpenTSDB.', + 'FOSS': 'Modified', + 'From repo': 'localrepo', + 'Group': 'Unspecified', + 'Install Date': 'Fri Nov 16 02:23:02 2018', + 'Is sane': True, + 'License': 'Commercial and ASL 2.0 and others', + 'Name': 'grafana', + 'Obsoletes': '', + 'Packager': 'Something', + 'Release': '1.el7.centos.1', + 'Relocations': '(not relocatable)', + 'Repo': 'installed', + 'Repo data': { + 'baseurl': 'https://jenkins/ci-build/2506/artifact/results/repo', + 'name': 'localrepo'}, + 'Signature': '(none)', + 'Size': '93957067', + 'Source RPM': 'grafana-5.2.3-1.el7.centos.1.src.rpm', + 'Source repo data': { + 'baseurl': 'https://jenkins/ci-build/2506/artifact/results/src_repo', + 'name': 'localrepo'}, + 'Source to be delivered': 'Upstream', + 'Summary': 'Grafana is an open source, feature rich metrics dashboard and graph editor', + 'URL': 'https://github.com/grafana/grafana', + 'Vendor': 'Something and Grafana modified', + 'Version': '5.2.3' +} + +grafana_v2 = _copy(grafana_v1, '1.el7.centos.1', '1.el7.centos.2') diff --git a/tools/script/create_rpm_data.py b/tools/script/create_rpm_data.py new file mode 100755 index 0000000..9c4b747 --- /dev/null +++ b/tools/script/create_rpm_data.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=too-many-instance-attributes,too-many-arguments + +import argparse +import copy +import sys +import logging +import re +import json +from pprint import pformat + +import os + +from tools.rpm import RpmInfoParser +from tools.utils import apply_jenkins_auth +from tools.yum import Yum, YumInfoParser +from tools.repository import RepositoryConfig +from tools.log import set_logging +from tools.io import read_from, write_to, read_json +from tools.convert import to_json, CsvConverter + + +class RpmDataBuilder(object): + def __init__(self, build_config, yum_info_installed, rpm_info_installed, + crypto_info_installed, boms, remote=False): + self.remote = remote + self.yum_info_installed = yum_info_installed + self.rpm_info_installed = rpm_info_installed + self.crypto_info_installed = json.loads(crypto_info_installed) + self.boms = boms + logging.debug('BOMS: {}'.format(pformat(self.boms))) + self.repoconfig = RepositoryConfig(build_config) + self.installed_rpms = None + self.repos = None + + def run(self): + self.installed_rpms = self.read_installed_rpms() + srpms = set([rpm['Source RPM'] for rpm in self.installed_rpms]) + logging.info('Installed RPMs:{} SRPMs:{}'.format(len(self.installed_rpms), len(srpms))) + self.repos = self._read_configured_repos() + logging.info('Configured repos: {}'.format(len(self.repos))) + available_rpms = self._read_available_rpms(self.repos) + logging.info('Found {} available RPMs in binary repos'.format( + len([rpm for repo_rpms in available_rpms.values() for rpm in repo_rpms]))) + for i_rpm in self.installed_rpms: + i_rpm_repo_name = self._get_rpm_available_in(i_rpm, available_rpms) + i_rpm['Repo data'] = self._get_repo(i_rpm_repo_name) + i_rpm['Obsoletes'] = self._resolve_obsoletes(i_rpm) + i_rpm['Crypto capable'] = self._resolve_ecc(str(i_rpm)) + i_rpm['BOM'] = self._resolve_bom(i_rpm) + self._log_repo_rpm_statistics() + self._log_rpm_statistics() + return self.installed_rpms + + @staticmethod + def _resolve_obsoletes(rpm): + if 'Obsoletes' not in rpm: + return 'N/A' + elif rpm['Obsoletes'] == '(none)': + return 'N/A' + return rpm['Obsoletes'] + + def _resolve_ecc(self, rpm): + for item in self.crypto_info_installed: + if item['name'] == rpm: + return True + return False + + def _resolve_bom(self, rpm): + bom_content = self.boms.get(str(rpm)) + if bom_content is None: + return '' + self._validate_bom(str(rpm), bom_content) + return bom_content['bom'] + + @staticmethod + def _validate_bom(rpm_name, bom_content): + try: + if 'bom' not in bom_content: + raise Exception('BOM base object "bom" missing') + bom = bom_content['bom'] + for material in bom: + for key in ['name', 'version', 'source-url', 'foss']: + if key not in material: + raise Exception('Key "{}" not found in BOM'.format(key)) + if material['foss'].lower() not in ['yes', 'no', 'modified']: + raise Exception('BOM foss value not valid') + missing_crypto_count = len([material for material in bom if + 'crypto-capable' not in material]) + if missing_crypto_count != 0: + logging.warning( + 'crypto-capable missing from %s materials in RPM %s', + missing_crypto_count, rpm_name) + except Exception as e: + correct_format = {'bom': [ + {'name': '', + 'version': '', + 'source-url': '', + 'foss': '', + 'crypto-capable': ''}]} + msg_fmt = 'BOM for {rpm} is not correct format. {error}:\n{correct_format}' + raise Exception(msg_fmt.format(rpm=rpm_name, + error=str(e), + correct_format=pformat(correct_format))) + + def _get_repo(self, name): + for r in self.repos: + if r['name'] == name: + return r + raise Exception('No repository found with name: {}'.format(name)) + + def read_installed_rpms(self): + installed_rpms = [] + yum_rpms = YumInfoParser().parse_installed(self.yum_info_installed) + rpm_rpms = RpmInfoParser().parse_multiple(self.rpm_info_installed) + self._validate_rpm_lists_identical(yum_rpms, rpm_rpms) + yum_rpms_dict = {rpm['Name']: rpm for rpm in yum_rpms} + for rpm_data in rpm_rpms: + yum_data = yum_rpms_dict[rpm_data['Name']] + combined_data = self._combine_rpm_data(rpm_data, yum_data) + installed_rpms.append(combined_data) + logging.debug('One parsed RPM data as example:\n{}'.format(pformat(installed_rpms[0]))) + return installed_rpms + + def _combine_rpm_data(self, rpm_data, yum_data): + combined_data = copy.deepcopy(rpm_data) + fields_known_to_differ = ['Description', # May contain deffering newline and indentation + 'Size'] # Bytes in RPM, humanreadable in yum + yum2rpm_field_name_map = {'Arch': 'Architecture'} + for yum_key in yum_data: + if yum_key in yum2rpm_field_name_map: + rpm_key = yum2rpm_field_name_map[yum_key] + else: + rpm_key = yum_key + if rpm_key in combined_data: + yum_comparable_rpm_string = self._rpm_info_str_to_yum_info_str( + combined_data[rpm_key]) + if yum_comparable_rpm_string != yum_data[yum_key]: + if rpm_key in fields_known_to_differ: + continue + raise Exception( + 'RPM data in "{}" not match in rpm "{}" vs yum "{}" for package {}'.format( + rpm_key, + repr(combined_data[rpm_key]), + repr(yum_data[yum_key]), + combined_data)) + else: + combined_data[rpm_key] = yum_data[yum_key] + return combined_data + + @staticmethod + def _rpm_info_str_to_yum_info_str(string): + try: + string.decode() + except (UnicodeEncodeError, UnicodeDecodeError): + return re.sub(r'[^\x00-\x7F]+', '?', string) + except Exception as e: + logging.error('{}: for string {}'.format(str(e), repr(string))) + raise + return string + + @staticmethod + def _validate_rpm_lists_identical(yum_rpms, rpm_rpms): + yum_rpms_dict = {rpm['Name']: rpm for rpm in yum_rpms} + rpm_rpms_dict = {rpm['Name']: rpm for rpm in rpm_rpms} + if len(yum_rpms) != len(rpm_rpms): + raise Exception( + 'Given RPM lists are unequal: yum RPM count {} != rpm RPM count {}'.format( + len(yum_rpms), len(rpm_rpms))) + assert sorted(yum_rpms_dict.keys()) == sorted(rpm_rpms_dict.keys()) + for name in yum_rpms_dict.keys(): + if not yum_rpms_dict[name].is_same_package_as(rpm_rpms_dict[name]): + raise Exception( + 'Packages are not same: yum {} != rpm {}'.format(yum_rpms_dict[name], + rpm_rpms_dict[name])) + + def _read_configured_repos(self): + repos = self.repoconfig.read_sections( + ['baseimage-repositories', 'repositories']) + repos.append(self.repoconfig.get_localrepo(remote=True)) + logging.debug('Configured repos: {}'.format(pformat(repos))) + return repos + + def _read_available_rpms(self, repos): + Yum.clean_and_remove_cache() + yum = Yum() + for repo in repos: + name = repo['name'] + if name == 'localrepo': + if self.remote: + url = self.repoconfig.get_localrepo(remote=True)['baseurl'] + yum.add_repo(name, apply_jenkins_auth(url)) + else: + url = self.repoconfig.get_localrepo(remote=False)['baseurl'] + yum.add_repo(name, url) + else: + yum.add_repo(name, repo['baseurl']) + yum_available_output = yum.read_all_packages() + available_rpms = YumInfoParser().parse_available(yum_available_output) + rpms_per_repo = {} + for rpm in available_rpms: + repo = rpm.get('Repo') + if repo not in rpms_per_repo: + rpms_per_repo[repo] = [] + rpms_per_repo[repo].append(rpm) + return rpms_per_repo + + def _log_repo_rpm_statistics(self): + logging.info('--- RPM repo statistics ---') + for repo in self.repos: + name = repo['name'] + repo_url = repo['baseurl'] + if name in [r['name'] for r in self._get_nonerepos()]: + expected_from_repo = None + else: + expected_from_repo = name + repo_installed_rpm_count = len([rpm for rpm in self.installed_rpms if + rpm['Repo data']['baseurl'] == repo_url and rpm.get( + 'From repo') == expected_from_repo]) + logging.info( + 'RPMs installed from repo "{}": {}'.format(name, repo_installed_rpm_count)) + if repo_installed_rpm_count is 0: + logging.warning( + 'Repository configured but no RPMs installed: {}={}'.format(name, repo_url)) + + return self.installed_rpms + + def _log_rpm_statistics(self): + def _get_count(func): + return len([rpm for rpm in self.installed_rpms if func(rpm)]) + + logging.info('----- RPMs per type -----') + logging.info(' => Total: %s', len(self.installed_rpms)) + logging.info('----- RPMs per attribute -----') + logging.info(' * Crypto capable: %s', _get_count(lambda rpm: rpm['Crypto capable'])) + logging.info(' * Complex (BOM): %s', _get_count(lambda rpm: rpm['BOM'])) + + def _get_rpm_available_in(self, rpm, available_rpms): + if 'From repo' in rpm.keys(): + if rpm['From repo'] == 'localrepo': + return 'localrepo' + available_repo_rpms = available_rpms[rpm['From repo']] + for a_rpm in available_repo_rpms: + if self._is_same_rpm(a_rpm, rpm): + return rpm['From repo'] + rpms_in_matching_repo = [str(a_rpm) for a_rpm in available_repo_rpms] + rpms_with_matching_name = [str(a_rpm) for a_rpm in available_repo_rpms if + rpm['Name'] == a_rpm['Name']] + if len(rpms_in_matching_repo) <= 1000: + logging.debug( + 'Available RPMs in {}: {}'.format(rpm['From repo'], rpms_in_matching_repo)) + error_str = 'RPM "{}" is not available in configured repo: {}, ' \ + 'RPMs with correct name: {}'.format(str(rpm), rpm['From repo'], + rpms_with_matching_name) + raise Exception(error_str) + else: + none_repos = self._get_nonerepos() + for repo in [r['name'] for r in none_repos]: + for a_rpm in available_rpms[repo]: + if self._is_same_rpm(a_rpm, rpm): + return repo + msg = 'RPM "{}" is not available in any configured "none*" repos: {}'.format( + rpm['Name'], none_repos) + raise Exception(msg) + + def _get_nonerepos(self): + return [repo for repo in self.repos if re.match(r'^none\d+$', repo['name'])] + + @staticmethod + def _is_same_rpm(rpm1, rpm2): + return rpm1['Name'] == rpm2['Name'] and \ + rpm1['Version'] == rpm2['Version'] and \ + rpm1['Release'] == rpm2['Release'] and \ + rpm1['Arch'] == rpm2['Architecture'] + + +def parse(args): + p = argparse.ArgumentParser( + description='Generate package info', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + p.add_argument('--verbose', '-v', action='store_true', + help='More verbose logging') + p.add_argument('--yum-info-path', required=True, + help='"yum info all" output as file') + p.add_argument('--rpm-info-path', required=True, + help='"rpm -qai" output as file') + p.add_argument('--crypto-info-path', + help='Dir from where to find ECC file') + p.add_argument('--boms-path', + help='Dir from where to find RPM bill of material files') + p.add_argument('--output-rpmlist', + help='output as rpm list like "rpm-qa"') + p.add_argument('--output-json', + help='output json file path') + p.add_argument('--output-csv', + help='output csv file path') + p.add_argument('--output-ms-csv', + help='output Microsoft Excel compatible csv file path') + p.add_argument('--build-config-path', required=True, + help='Build configuration ini path') + p.add_argument('--remote', action='store_true', + help='Read localrepo from remote defined by BUILD_URL, ' + 'otherwise use localrepo from WORKSPACE') + args = p.parse_args(args) + return args + + +def read_files(boms_dir): + boms = {} + for f in os.listdir(boms_dir): + boms[f] = read_json(boms_dir + '/' + f) + return boms + + +def main(input_args): + args = parse(input_args) + if args.verbose: + set_logging(debug=True, timestamps=True) + else: + set_logging(debug=False) + rpmdata = RpmDataBuilder(args.build_config_path, + read_from(args.yum_info_path), + read_from(args.rpm_info_path), + read_from(args.crypto_info_path), + read_files(args.boms_path), + remote=args.remote).run() + if args.output_rpmlist: + write_to(args.output_rpmlist, '\n'.join(sorted([str(rpm) for rpm in rpmdata]))) + if args.output_json: + write_to(args.output_json, to_json(rpmdata)) + csv = CsvConverter(rpmdata, preferred_field_order=['Name', 'Version', 'Release', + 'License', 'Vendor', 'From repo', + 'Source RPM']) + if args.output_csv: + write_to(args.output_csv, str(csv)) + if args.output_ms_csv: + write_to(args.output_ms_csv, + csv.convert_to_ms_excel(text_fields=['Version', 'Size', 'Release'])) + if not args.output_json and not args.output_csv: + print(rpmdata) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/tools/script/create_rpm_data_test.py b/tools/script/create_rpm_data_test.py new file mode 100755 index 0000000..1e47e89 --- /dev/null +++ b/tools/script/create_rpm_data_test.py @@ -0,0 +1,84 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=invalid-name +import os +import sys +import logging +import pytest +import mock + +from tools.script.create_rpm_data import RpmDataBuilder +from tools.repository import BuildConfigParser +from tools.executor import Result +import tools.repository +from tools.script.create_rpm_data_test_data import \ + yum_installed_output, \ + rpm_info_output, \ + expected_output, \ + basesystem_combined, \ + cpp_combined, \ + centos_logos_combined, \ + dejavu_fonts_common_combined, yum_available_output, \ + crypto_rpms_json, boms_output + +from tools.test_data_rpm import basesystem_rpm_info, cpp_rpm_info, centos_logos_rpm_info, \ + dejavu_fonts_common_rpm_info +from tools.test_data_yum import basesystem_yum_info, cpp_yum_info, centos_logos_yum_info, \ + dejavu_fonts_common_yum_info +from tools.yum import YumConfig + + +@mock.patch.object(YumConfig, 'write') +@mock.patch.object(tools.yum, 'run') +@mock.patch.object(BuildConfigParser, 'items') +def test_complete_parse(mock_config, mock_reporeader, _): + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + mock_config.side_effect = [ + [('base', 'test-url-for-base'), + ('none1', 'test-url-for-none1'), + ('none2', 'test-url-for-none2')], + [('purkki-3rdparty', 'test-url-for-purkki-3rdparty#test-extra-option=test-value')], + [('base', 'src-test-url-for-base'), + ('none1', 'src-test-url-for-none1'), + ('none2', 'src-test-url-for-none2')], + [('purkki-3rdparty', 'src-test-url-for-purkki-3rdparty#test-extra-option2=test-value2')] + ] + mock_reporeader.side_effect = [Result(0, '', ''), # yum clean all + Result(0, '', ''), # rm -rf /var/yum/cache + Result(0, yum_available_output, '')] + os.environ['BUILD_URL'] = 'test-url/' + os.environ['WORKSPACE'] = '/foo/path' + result = RpmDataBuilder('fake_build_config_path', + yum_installed_output, + rpm_info_output, + crypto_rpms_json, + boms_output).run() + assert mock_reporeader.call_count == 3 + assert result == expected_output + + +@pytest.mark.parametrize('yum_info, rpm_info, expected_combined', [ + (basesystem_yum_info, basesystem_rpm_info, basesystem_combined), + (cpp_yum_info, cpp_rpm_info, cpp_combined), + (centos_logos_yum_info, centos_logos_rpm_info, centos_logos_combined), + (dejavu_fonts_common_yum_info, dejavu_fonts_common_rpm_info, dejavu_fonts_common_combined), +]) +def test_combine_rpm_data(yum_info, rpm_info, expected_combined): + result = RpmDataBuilder('fake_build_config_path', + yum_info, + rpm_info, + '[]', + {}).read_installed_rpms() + assert result[0] == expected_combined diff --git a/tools/script/create_rpm_data_test_data.py b/tools/script/create_rpm_data_test_data.py new file mode 100755 index 0000000..e7d38f3 --- /dev/null +++ b/tools/script/create_rpm_data_test_data.py @@ -0,0 +1,303 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=invalid-name +from tools.rpm_test_data import cpp_expected + +yum_installed_output = """Loaded plugins: fastestmirror, priorities +Loading mirror speeds from cached hostfile +Installed Packages +Name : non-repo-pkg-1 +Arch : x86_64 +Version : 1 +Release : 1 +Repo : installed + +Name : non-repo-pkg-2 +Arch : noarch +Version : 2 +Release : 2 +Repo : installed +Obsoletes : (none) + +Name : base-image-pkg +Arch : x86_64 +Version : 3 +Release : 3 +Repo : installed +From repo : base + +Name : internal-pkg +Arch : noarch +Version : 4 +Release : 4 +Repo : installed +From repo : localrepo + +Name : 3rdparty-pkg +Arch : x86_64 +Version : 5 +Release : 5 +Repo : installed +From repo : purkki-3rdparty +Obsoletes : 2ndparty-pkg + +""" + +rpm_info_output = """Name : non-repo-pkg-1 +Version : 1 +Release : 1 +Architecture: x86_64 +Source RPM : non-repo-pkg-1-1-1.src.rpm +Name : non-repo-pkg-2 +Version : 2 +Release : 2 +Architecture: noarch +Source RPM : non-repo-pkg-2-2-2.src.rpm +Name : base-image-pkg +Version : 3 +Release : 3 +Architecture: x86_64 +Source RPM : base-image-pkg-3-3.src.rpm +Name : internal-pkg +Version : 4 +Release : 4 +Architecture: noarch +Source RPM : internal-pkg-4-4.src.rpm +Name : 3rdparty-pkg +Version : 5 +Release : 5 +Architecture: x86_64 +Source RPM : 3rdparty-pkg-5-5.src.rpm +""" # noqa, PEP-8 disabled because of example output has trailing spaces + +yum_available_output_header = """Added tmprepo repo from http://url1/ +Available Packages +""" + +yum_available_output_base = """ +Name : base-image-pkg +Arch : x86_64 +Epoch : 0 +Version : 3 +Release : 3 +Size : 195 k +Repo : base +""" + +yum_available_output_none2 = """ +Name : non-repo-pkg-1 +Arch : x86_64 +Epoch : 0 +Version : 1 +Release : 1 +Size : 195 k +Repo : none2 +""" + +yum_available_output_none1 = """ +Name : non-repo-pkg-2 +Arch : noarch +Version : 2 +Release : 2 +Size : 195 k +Repo : none1 +""" + +yum_available_output_localrepo = """ +Name : internal-pkg +Arch : x86_64 +Epoch : 0 +Version : 4 +Release : 4 +Size : 195 k +Repo : localrepo +""" + +yum_available_output_purkki_3rdparty = """ +Name : 3rdparty-pkg +Arch : x86_64 +Epoch : 0 +Version : 5 +Release : 5 +Size : 195 k +Repo : purkki-3rdparty +""" + +yum_available_output = yum_available_output_header + yum_available_output_base + \ + yum_available_output_none1 + yum_available_output_none2 + \ + yum_available_output_localrepo + yum_available_output_purkki_3rdparty + +internal_pkg_bom = [{'name': '@types/d3-axis', + 'version': '1.0.10', + 'foss': 'yes', + 'source-url': 'http://some.url/1', + 'crypto-capable': True}, + {'name': '@types/d3-array@*', + 'version': '1.2.1', + 'foss': 'Yes', + 'source-url': 'http://some.url/2'}] +boms_output = {'internal-pkg-4-4.noarch': {"bom": internal_pkg_bom}} + +expected_output = [ + { + 'Name': 'non-repo-pkg-1', + 'Architecture': 'x86_64', + 'Version': '1', + 'Release': '1', + 'Repo': 'installed', + 'Repo data': {'baseurl': 'test-url-for-none2', 'name': 'none2'}, + 'Obsoletes': 'N/A', + 'Source RPM': 'non-repo-pkg-1-1-1.src.rpm', + 'Crypto capable': False, + 'BOM': '', + }, { + 'Name': 'non-repo-pkg-2', + 'Architecture': 'noarch', + 'Version': '2', + 'Release': '2', + 'Repo': 'installed', + 'Repo data': {'baseurl': 'test-url-for-none1', 'name': 'none1'}, + 'Obsoletes': 'N/A', + 'Source RPM': 'non-repo-pkg-2-2-2.src.rpm', + 'Crypto capable': False, + 'BOM': '', + }, { + 'Name': 'base-image-pkg', + 'Architecture': 'x86_64', + 'Version': '3', + 'Release': '3', + 'Repo': 'installed', + 'From repo': 'base', + 'Repo data': {'baseurl': 'test-url-for-base', 'name': 'base'}, + 'Obsoletes': 'N/A', + 'Source RPM': 'base-image-pkg-3-3.src.rpm', + 'Crypto capable': False, + 'BOM': '', + }, { + 'Name': 'internal-pkg', + 'Architecture': 'noarch', + 'Version': '4', + 'Release': '4', + 'Repo': 'installed', + 'From repo': 'localrepo', + 'Repo data': {'baseurl': 'test-url/artifact/results/repo', + 'name': 'localrepo'}, + 'Obsoletes': 'N/A', + 'Source RPM': 'internal-pkg-4-4.src.rpm', + 'Crypto capable': True, + 'BOM': internal_pkg_bom, + }, { + 'Name': '3rdparty-pkg', + 'Architecture': 'x86_64', + 'Version': '5', + 'Release': '5', + 'Repo': 'installed', + 'From repo': 'purkki-3rdparty', + 'Repo data': {'baseurl': 'test-url-for-purkki-3rdparty', 'name': 'purkki-3rdparty', + 'test-extra-option': 'test-value'}, + 'Obsoletes': '2ndparty-pkg', + 'Source RPM': '3rdparty-pkg-5-5.src.rpm', + 'Crypto capable': False, + 'BOM': '', + }] + +basesystem_combined = { + # From RPM info + 'Name': 'basesystem', + 'Version': '10.0', + 'Release': '7.el7.centos', + 'Architecture': 'noarch', + 'Install Date': 'Fri 01 Apr 2016 11:47:25 AM EEST', + 'Group': 'System Environment/Base', + 'Size': '0', + 'License': 'Public Domain', + 'Signature': 'RSA/SHA256, Fri 04 Jul 2014 03:46:57 AM EEST, Key ID 24c6a8a7f4a80eb5', + 'Source RPM': 'basesystem-10.0-7.el7.centos.src.rpm', + 'Build Date': 'Fri 27 Jun 2014 01:37:10 PM EEST', + 'Build Host': 'worker1.bsys.centos.org', + 'Relocations': '(not relocatable)', + 'Packager': 'CentOS BuildSystem ', + 'Vendor': 'CentOS', + 'Summary': 'The skeleton package which defines a simple CentOS Linux system', + 'Description': '\n'.join( + ['Basesystem defines the components of a basic CentOS Linux', + 'system (for example, the package installation order to use during', + 'bootstrapping). Basesystem should be in every installation of a system,', + 'and it should never be removed.']), + # From yum info + 'Repo': 'installed', +} + +centos_logos_combined = { + 'Name': 'centos-logos', + 'Version': '70.0.6', + 'Release': '3.el7.centos', + 'Architecture': 'noarch', + 'License': u'Copyright © 2014 The CentOS Project. All rights reserved.', +} + +cpp_combined = cpp_expected.copy() +cpp_combined.update({'Repo': 'installed', + 'From repo': 'purkki-centos-base'}) + +dejavu_fonts_common_combined = { + 'Name': 'dejavu-fonts-common', + 'Version': '2.33', + 'Release': '6.el7', + 'Architecture': 'noarch', + 'Install Date': 'Wed Feb 7 13:49:27 2018', + 'Group': 'User Interface/X', + 'Size': '130455', + 'License': 'Bitstream Vera and Public Domain', + 'Signature': 'RSA/SHA256, Fri Jul 4 01:06:50 2014, Key ID 24c6a8a7f4a80eb5', + 'Source RPM': 'dejavu-fonts-2.33-6.el7.src.rpm', + 'Build Date': 'Mon Jun 9 21:34:30 2014', + 'Build Host': 'worker1.bsys.centos.org', + 'Relocations': '(not relocatable)', + 'Packager': 'CentOS BuildSystem ', + 'Vendor': 'CentOS', + 'URL': 'http://dejavu-fonts.org/', + 'Summary': 'Common files for the Dejavu font set', + 'Description': '\n'.join( + ['The DejaVu font set is based on the “Bitstream Vera” fonts, release 1.10. Its', + 'purpose is to provide a wider range of characters, while maintaining the', + 'original style, using an open collaborative development process.', + '', + 'This package consists of files used by other DejaVu packages.']), + 'Repo': 'installed', + 'From repo': 'purkki-centos-base' +} + +crypto_rpms_json = """ +[ + { + "name": "internal-pkg-4-4.noarch", + "requires": [ + "libgssapi_krb5.so.2()(64bit)", + "libk5crypto.so.3()(64bit)", + "libkrb5.so.3()(64bit)", + "libcrypto.so.10()(64bit)", + "libcrypto.so.10(OPENSSL_1.0.1_EC)(64bit)", + "libcrypto.so.10(OPENSSL_1.0.2)(64bit)", + "libcrypto.so.10(libcrypto.so.10)(64bit)", + "libssl.so.10()(64bit)", + "libssl.so.10(libssl.so.10)(64bit)", + "openssl-libs(x86-64)", + "rtld(GNU_HASH)" + ] + } +] +""" diff --git a/tools/script/generate_repo_files.py b/tools/script/generate_repo_files.py new file mode 100755 index 0000000..07fdbb3 --- /dev/null +++ b/tools/script/generate_repo_files.py @@ -0,0 +1,74 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import argparse +import logging + +from tools.buildconfig import BuildConfigParser + + +def _parse(args): + parser = argparse.ArgumentParser(description='Generate repo files from config ini sections') + parser.add_argument('config_sections', metavar='config_section', nargs='+', + help='Config ini section') + parser.add_argument('--output-dir', '-d', required=True, + help='Directory to output the repo files') + parser.add_argument('--config-ini', '-c', required=True, + help='Path to the config ini to read') + args = parser.parse_args(args) + return args + + +class RepoGen(object): + + def __init__(self, output_dir, config_ini, config_sections): + self.output_dir = output_dir + self.config_sections = config_sections + self.config = BuildConfigParser(ini_file=config_ini) + + def run(self): + for section in self.config_sections: + repo_file_path = os.path.join(self.output_dir, section + '.repo') + self._write_repo_file(section, repo_file_path) + + def _write_repo_file(self, section, repo_file_path): + with open(repo_file_path, 'w') as f: + for repo in self.config.items(section): + name = repo[0] + parts = repo[1].split('#') + url = parts[0] + f.write('[%s]\n' % name) + f.write('name=%s\n' % name) + f.write('baseurl=%s\n' % url) + f.write('enabled=1\n') + f.write('gpgcheck=0\n') + for part in parts[1:]: + f.write('%s\n' % part) + f.write('\n') + if os.path.getsize(repo_file_path) == 0: + logging.error('Zero size output: {}'.format(repo_file_path)) + sys.exit(1) + logging.info('Wrote repo: {}'.format(repo_file_path)) + + +def main(input_args): + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + args = _parse(input_args) + RepoGen(args.output_dir, args.config_ini, args.config_sections).run() + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/tools/script/process_rpmdata.py b/tools/script/process_rpmdata.py new file mode 100755 index 0000000..bfab70b --- /dev/null +++ b/tools/script/process_rpmdata.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import sys + +from tools.convert import to_json, CsvConverter +from tools.log import set_logging +from tools.io import write_to, read_json + + +class RpmDataProcesser(object): + def __init__(self, rpmdata): + self.components = self._get_components(rpmdata) + + @staticmethod + def _get_components(rpmdata): + components = [] + bom_field_map = {'name': 'Name', + 'version': 'Version', + 'foss': 'FOSS', + 'source-url': 'Source URL', + 'crypto-capable': 'Crypto capable'} + for rpm in rpmdata: + components.append(unicode_recursively(rpm)) + bom = rpm['BOM'] + if bom: + for material in bom: + component = {'Source RPM': rpm['Source RPM']} + for field in material: + component[bom_field_map[field]] = material[field] + components.append(unicode_recursively(component)) + return components + + def gen_components(self, path): + write_to(path, to_json(self.components)) + + def gen_components_csv(self, path): + csv = CsvConverter(self.components, + preferred_field_order=['Name', 'Version', 'Release', 'Source RPM']) + write_to(path, csv.convert_to_ms_excel(text_fields=['Version'])) + + +def unicode_recursively(something): + if isinstance(something, dict): + return {unicode_recursively(key): unicode_recursively(value) for key, value in + something.iteritems()} + elif isinstance(something, list): + return [unicode_recursively(element) for element in something] + elif isinstance(something, unicode): + return something.encode('utf-8') + return something + + +def parse(args): + p = argparse.ArgumentParser( + description='Process rpmdata for multitude of tools', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + p.add_argument('--verbose', '-v', action='store_true', + help='More verbose logging') + p.add_argument('--rpmdata-path', required=True, + help='RPM data json file path') + p.add_argument('--output-components', + help='Component list that includes also RPM sub-components') + p.add_argument('--output-components-csv', + help='Component list that includes also RPM sub-components as CSV') + args = p.parse_args(args) + return args + + +def main(input_args): + args = parse(input_args) + if args.verbose: + set_logging(debug=True) + else: + set_logging(debug=False) + p = RpmDataProcesser(read_json(args.rpmdata_path)) + if args.output_components: + p.gen_components(args.output_components) + if args.output_components_csv: + p.gen_components_csv(args.output_components_csv) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/tools/script/process_rpmdata_test.py b/tools/script/process_rpmdata_test.py new file mode 100755 index 0000000..d704bf7 --- /dev/null +++ b/tools/script/process_rpmdata_test.py @@ -0,0 +1,58 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +from tools.convert import CsvConverter +from tools.script.create_rpm_data_test_data import expected_output +from tools.script.process_rpmdata import main + + +def gen_input(tmpdir, rpmdata): + p = tmpdir.join('rpmdata.json') + p.write(rpmdata) + return str(p) + + +def test_no_output(tmpdir): + main(['--rpmdata-path', gen_input(tmpdir, {})]) + + +def test_components(tmpdir): + input_content = [expected_output[0], + expected_output[3]] + input_ = json.dumps(input_content) + output_json = tmpdir.join('components.json') + output_csv = tmpdir.join('components.csv') + main(['--rpmdata-path', gen_input(tmpdir, input_), + '--output-components', str(output_json), + '--output-components-csv', str(output_csv)]) + additions = [{'FOSS': u'yes', + 'Source RPM': 'internal-pkg-4-4.src.rpm', + 'Name': '@types/d3-axis', + 'Source URL': 'http://some.url/1', + 'Version': '1.0.10', + 'Crypto capable': True}, + {'FOSS': u'Yes', + 'Source RPM': 'internal-pkg-4-4.src.rpm', + 'Name': '@types/d3-array@*', + 'Source URL': 'http://some.url/2', + 'Version': '1.2.1'}] + assert json.loads(output_json.read()) == input_content + additions + assert output_csv.read() == _gen_components_csv(input_content + additions) + + +def _gen_components_csv(_list): + csv = CsvConverter(_list, preferred_field_order=['Name', 'Version', 'Release', 'Source RPM']) + return csv.convert_to_ms_excel(text_fields=['Version']) diff --git a/tools/script/read_build_config.py b/tools/script/read_build_config.py new file mode 100755 index 0000000..1e6400a --- /dev/null +++ b/tools/script/read_build_config.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +from tools.buildconfig import BuildConfigParser + + +def main(): + config = BuildConfigParser(sys.argv[1]) + if len(sys.argv) == 3: + print(config.items(sys.argv[2])) + elif len(sys.argv) == 4: + print(config.get(sys.argv[2], sys.argv[3])) + else: + raise Exception('Invalid parameter count: {}'.format(len(sys.argv))) + + +if __name__ == "__main__": + main() diff --git a/tools/script/read_package_config.py b/tools/script/read_package_config.py new file mode 100755 index 0000000..fc0b8b2 --- /dev/null +++ b/tools/script/read_package_config.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import sys + +from tools.package import PackageConfigReader + + +def parse(args): + p = argparse.ArgumentParser( + description='Read package configuration', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + p.add_argument('--config', required=False, help='Package YAML config file path') + p.add_argument('--separator', required=False, default=' ', + help='Separator for the resulting stdout list') + p.add_argument('package_operation_type', choices=['install', 'uninstall'], + help='list packages by operation type') + args = p.parse_args(args) + return args + + +def main(input_args): + args = parse(input_args) + kwargs = {} + if args.config is not None: + kwargs['config_file'] = args.config + print(args.separator.join(PackageConfigReader(**kwargs).get(args.package_operation_type))) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/tools/statics.py b/tools/statics.py new file mode 100755 index 0000000..f1ebd9c --- /dev/null +++ b/tools/statics.py @@ -0,0 +1,26 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +WORK_ROOT = os.path.dirname( + os.path.dirname( + os.path.dirname( + os.path.dirname( + os.path.realpath(__file__))))) + +MAINFEST_PATH = os.path.join(WORK_ROOT, '.repo/manifests') + +BUILD_CONFIG_PATH = os.path.join(MAINFEST_PATH, 'build_config.ini') +PACKAGES_CONFIG_PATH = os.path.join(MAINFEST_PATH, 'packages.yaml') diff --git a/tools/test_data_rpm.py b/tools/test_data_rpm.py new file mode 100755 index 0000000..440a2bf --- /dev/null +++ b/tools/test_data_rpm.py @@ -0,0 +1,230 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=invalid-name,line-too-long + +bash_rpm_info = """Name : bash +Version : 4.2.46 +Release : 21.el7_3 +Architecture: x86_64 +Install Date: Thu 11 Jan 2018 12:32:51 PM EET +Group : System Environment/Shells +Size : 3663714 +License : GPLv3+ +Signature : RSA/SHA256, Wed 07 Dec 2016 02:11:28 AM EET, Key ID 24c6a8a7f4a80eb5 +Source RPM : bash-4.2.46-21.el7_3.src.rpm +Build Date : Wed 07 Dec 2016 01:21:54 AM EET +Build Host : c1bm.rdu2.centos.org +Relocations : (not relocatable) +Packager : CentOS BuildSystem +Vendor : CentOS +URL : http://www.gnu.org/software/bash +Summary : The GNU Bourne Again shell +Description : +The GNU Bourne Again shell (Bash) is a shell or command language +interpreter that is compatible with the Bourne shell (sh). Bash +incorporates useful features from the Korn shell (ksh) and the C shell +(csh). Most sh scripts can be run by bash without modification. +""" + +basesystem_rpm_info = """Name : basesystem +Version : 10.0 +Release : 7.el7.centos +Architecture: noarch +Install Date: Fri 01 Apr 2016 11:47:25 AM EEST +Group : System Environment/Base +Size : 0 +License : Public Domain +Signature : RSA/SHA256, Fri 04 Jul 2014 03:46:57 AM EEST, Key ID 24c6a8a7f4a80eb5 +Source RPM : basesystem-10.0-7.el7.centos.src.rpm +Build Date : Fri 27 Jun 2014 01:37:10 PM EEST +Build Host : worker1.bsys.centos.org +Relocations : (not relocatable) +Packager : CentOS BuildSystem +Vendor : CentOS +Summary : The skeleton package which defines a simple CentOS Linux system +Description : +Basesystem defines the components of a basic CentOS Linux +system (for example, the package installation order to use during +bootstrapping). Basesystem should be in every installation of a system, +and it should never be removed. +""" + +centos_logos_rpm_info = u"""Name : centos-logos +Version : 70.0.6 +Release : 3.el7.centos +Architecture: noarch +License : Copyright © 2014 The CentOS Project. All rights reserved. + +""" + +conntrack_tools_rpm_info = """Name : conntrack-tools +Version : 1.4.4 +Release : 3.el7_3 +Architecture: x86_64 +Install Date: Thu 11 Jan 2018 12:39:20 PM EET +Group : System Environment/Base +Size : 562826 +License : GPLv2 +Signature : RSA/SHA256, Thu 29 Jun 2017 03:36:05 PM EEST, Key ID 24c6a8a7f4a80eb5 +Source RPM : conntrack-tools-1.4.4-3.el7_3.src.rpm +Build Date : Thu 29 Jun 2017 03:18:42 AM EEST +Build Host : c1bm.rdu2.centos.org +Relocations : (not relocatable) +Packager : CentOS BuildSystem +Vendor : CentOS +URL : http://netfilter.org +Summary : Manipulate netfilter connection tracking table and run High Availability +Description : +With conntrack-tools you can setup a High Availability cluster and +synchronize conntrack state between multiple firewalls. + +The conntrack-tools package contains two programs: +- conntrack: the command line interface to interact with the connection + tracking system. +- conntrackd: the connection tracking userspace daemon that can be used to + deploy highly available GNU/Linux firewalls and collect + statistics of the firewall use. + +conntrack is used to search, list, inspect and maintain the netfilter +connection tracking subsystem of the Linux kernel. +Using conntrack, you can dump a list of all (or a filtered selection of) +currently tracked connections, delete connections from the state table, +and even add new ones. +In addition, you can also monitor connection tracking events, e.g. +show an event message (one line) per newly established connection. +""" + +cpp_rpm_info = """Name : cpp +Version : 4.8.5 +Release : 11.el7 +Architecture: x86_64 +Install Date: Thu 11 Jan 2018 12:37:55 PM EET +Group : Development/Languages +Size : 15632501 +License : GPLv3+ and GPLv3+ with exceptions and GPLv2+ with exceptions and LGPLv2+ and BSD +Signature : RSA/SHA256, Sun 20 Nov 2016 07:27:00 PM EET, Key ID 24c6a8a7f4a80eb5 +Source RPM : gcc-4.8.5-11.el7.src.rpm +Build Date : Fri 04 Nov 2016 06:01:22 PM EET +Build Host : worker1.bsys.centos.org +Relocations : (not relocatable) +Packager : CentOS BuildSystem +Vendor : CentOS +URL : http://gcc.gnu.org +Summary : The C Preprocessor +Description : +Cpp is the GNU C-Compatible Compiler Preprocessor. +Cpp is a macro processor which is used automatically +by the C compiler to transform your program before actual +compilation. It is called a macro processor because it allows +you to define macros, abbreviations for longer +constructs. + +The C preprocessor provides four separate functionalities: the +inclusion of header files (files of declarations that can be +substituted into your program); macro expansion (you can define macros, +and the C preprocessor will replace the macros with their definitions +throughout the program); conditional compilation (using special +preprocessing directives, you can include or exclude parts of the +program according to various conditions); and line control (if you use +a program to combine or rearrange source files into an intermediate +file which is then compiled, you can use line control to inform the +compiler about where each source line originated). + +You should install this package if you are a C programmer and you use +macros. +""" # noqa, PEP-8 disabled because of example output has trailing spaces + +dejavu_fonts_common_rpm_info = """Name : dejavu-fonts-common +Version : 2.33 +Release : 6.el7 +Architecture: noarch +Install Date: Wed Feb 7 13:49:27 2018 +Group : User Interface/X +Size : 130455 +License : Bitstream Vera and Public Domain +Signature : RSA/SHA256, Fri Jul 4 01:06:50 2014, Key ID 24c6a8a7f4a80eb5 +Source RPM : dejavu-fonts-2.33-6.el7.src.rpm +Build Date : Mon Jun 9 21:34:30 2014 +Build Host : worker1.bsys.centos.org +Relocations : (not relocatable) +Packager : CentOS BuildSystem +Vendor : CentOS +URL : http://dejavu-fonts.org/ +Summary : Common files for the Dejavu font set +Description : + +The DejaVu font set is based on the “Bitstream Vera” fonts, release 1.10. Its +purpose is to provide a wider range of characters, while maintaining the +original style, using an open collaborative development process. + +This package consists of files used by other DejaVu packages. +""" + +usbredir_rpm_info = """Name : usbredir +Version : 0.7.1 +Release : 1.el7 +Architecture: x86_64 +Install Date: Wed Feb 7 13:49:24 2018 +Group : System Environment/Libraries +Size : 108319 +License : LGPLv2+ +Signature : RSA/SHA256, Sun Nov 20 20:56:49 2016, Key ID 24c6a8a7f4a80eb5 +Source RPM : usbredir-0.7.1-1.el7.src.rpm +Build Date : Sat Nov 5 18:33:15 2016 +Build Host : worker1.bsys.centos.org +Relocations : (not relocatable) +Packager : CentOS BuildSystem +Vendor : CentOS +URL : http://spice-space.org/page/UsbRedir +Summary : USB network redirection protocol libraries +Description : +The usbredir libraries allow USB devices to be used on remote and/or virtual +hosts over TCP. The following libraries are provided: + +usbredirparser: +A library containing the parser for the usbredir protocol + +usbredirhost: +A library implementing the USB host side of a usbredir connection. +All that an application wishing to implement a USB host needs to do is: +* Provide a libusb device handle for the device +* Provide write and read callbacks for the actual transport of usbredir data +* Monitor for usbredir and libusb read/write events and call their handlers +""" + +perl_compress_rpm_info = """Name : perl-Compress-Raw-Zlib +Epoch : 1 +Version : 2.061 +Release : 4.el7 +Architecture: x86_64 +Install Date: Sat Jan 26 20:05:50 2019 +Group : Development/Libraries +Size : 139803 +License : GPL+ or Artistic +Signature : RSA/SHA256, Fri Jul 4 04:15:33 2014, Key ID 24c6a8a7f4a80eb5 +Source RPM : perl-Compress-Raw-Zlib-2.061-4.el7.src.rpm +Build Date : Tue Jun 10 01:12:08 2014 +Build Host : worker1.bsys.centos.org +Relocations : (not relocatable) +Packager : CentOS BuildSystem +Vendor : CentOS +URL : http://search.cpan.org/dist/Compress-Raw-Zlib/ +Summary : Low-level interface to the zlib compression library +Description : +The Compress::Raw::Zlib module provides a Perl interface to the zlib +compression library, which is used by IO::Compress::Zlib. +Obsoletes : +""" # noqa, PEP-8 disabled because of example output has trailing spaces diff --git a/tools/test_data_yum.py b/tools/test_data_yum.py new file mode 100755 index 0000000..6655be4 --- /dev/null +++ b/tools/test_data_yum.py @@ -0,0 +1,176 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=invalid-name + + +basesystem_yum_info = """Loaded plugins: fastestmirror, priorities +Loading mirror speeds from cached hostfile +Installed Packages +Name : basesystem +Arch : noarch +Version : 10.0 +Release : 7.el7.centos +Size : 0.0 +Repo : installed +Summary : The skeleton package which defines a simple CentOS Linux system +License : Public Domain +Description : Basesystem defines the components of a basic CentOS Linux + : system (for example, the package installation order to use during + : bootstrapping). Basesystem should be in every installation of a + : system, and it should never be removed. + +""" # noqa, PEP-8 disabled because of example output has trailing spaces + +bash_yum_info = """Name : bash +Arch : x86_64 +Version : 4.2.46 +Release : 21.el7_3 +Size : 3.5 M +Repo : installed +From repo : updates +Summary : The GNU Bourne Again shell +URL : http://www.gnu.org/software/bash +License : GPLv3+ +Description : The GNU Bourne Again shell (Bash) is a shell or command language + : interpreter that is compatible with the Bourne shell (sh). Bash + : incorporates useful features from the Korn shell (ksh) and the C + : shell (csh). Most sh scripts can be run by bash without + : modification. +""" + +centos_logos_yum_info = """ +Installed Packages +Name : centos-logos +Arch : noarch +Version : 70.0.6 +Release : 3.el7.centos +License : Copyright ? 2014 The CentOS Project. All rights reserved. + +""" + +conntrack_tools_yum_info = """Name : conntrack-tools +Arch : x86_64 +Version : 1.4.4 +Release : 3.el7_3 +Size : 550 k +Repo : installed +From repo : centos-updates +Summary : Manipulate netfilter connection tracking table and run High + : Availability +URL : http://netfilter.org +License : GPLv2 +Description : With conntrack-tools you can setup a High Availability cluster and + : synchronize conntrack state between multiple firewalls. + : + : The conntrack-tools package contains two programs: + : - conntrack: the command line interface to interact with the + : connection tracking system. + : - conntrackd: the connection tracking userspace daemon that can be + : used to deploy highly available GNU/Linux firewalls and collect + : statistics of the firewall use. + : + : conntrack is used to search, list, inspect and maintain the + : netfilter connection tracking subsystem of the Linux kernel. + : Using conntrack, you can dump a list of all (or a filtered + : selection of) currently tracked connections, delete connections + : from the state table, and even add new ones. + : In addition, you can also monitor connection tracking events, e.g. + : show an event message (one line) per newly established connection. +""" # noqa, PEP-8 disabled because of example output has trailing spaces + +cpp_yum_info = """ +Installed Packages +Name : cpp +Arch : x86_64 +Version : 4.8.5 +Release : 11.el7 +Size : 15 M +Repo : installed +From repo : purkki-centos-base +Summary : The C Preprocessor +URL : http://gcc.gnu.org +License : GPLv3+ and GPLv3+ with exceptions and GPLv2+ with exceptions and + : LGPLv2+ and BSD +Description : Cpp is the GNU C-Compatible Compiler Preprocessor. + : Cpp is a macro processor which is used automatically + : by the C compiler to transform your program before actual + : compilation. It is called a macro processor because it allows + : you to define macros, abbreviations for longer + : constructs. + : + : The C preprocessor provides four separate functionalities: the + : inclusion of header files (files of declarations that can be + : substituted into your program); macro expansion (you can define + : macros, and the C preprocessor will replace the macros with their + : definitions throughout the program); conditional compilation + : (using special preprocessing directives, you can include or + : exclude parts of the program according to various conditions); and + : line control (if you use a program to combine or rearrange source + : files into an intermediate file which is then compiled, you can + : use line control to inform the compiler about where each source + : line originated). + : + : You should install this package if you are a C programmer and you + : use macros. + +""" # noqa, PEP-8 disabled because of example output has trailing spaces + +dejavu_fonts_common_yum_info = """ +Installed Packages +Name : dejavu-fonts-common +Arch : noarch +Version : 2.33 +Release : 6.el7 +Size : 127 k +Repo : installed +From repo : purkki-centos-base +Summary : Common files for the Dejavu font set +URL : http://dejavu-fonts.org/ +License : Bitstream Vera and Public Domain +Description : + : The DejaVu font set is based on the ?Bitstream Vera? fonts, + : release 1.10. Its purpose is to provide a wider range of + : characters, while maintaining the original style, using an open + : collaborative development process. + : + : This package consists of files used by other DejaVu packages. + +""" # noqa, PEP-8 disabled because of example output has trailing spaces + +pacemaker_yum_info = """ +Name : pacemaker +Arch : x86_64 +Version : 1.1.15 +Release : 11.el7_3.5 +Size : 1.1 M +Repo : installed +From repo : purkki-centos-updates +Summary : Scalable High-Availability cluster resource manager +URL : http://www.clusterlabs.org +License : GPLv2+ and LGPLv2+ +Description : Pacemaker is an advanced, scalable High-Availability cluster + : resource manager for Corosync, CMAN and/or Linux-HA. + : + : It supports more than 16 node clusters with significant + : capabilities for managing resources and dependencies. + : + : It will run scripts at initialization, when machines go up or + : down, when related resources fail and can be configured to + : periodically check resource health. + : + : Available rpmbuild rebuild options: + : --with(out) : cman stonithd doc coverage profiling pre_release + : hardening +""" # noqa, PEP-8 disabled because of example output has trailing spaces diff --git a/tools/utils.py b/tools/utils.py new file mode 100755 index 0000000..caea09f --- /dev/null +++ b/tools/utils.py @@ -0,0 +1,31 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +def apply_jenkins_auth(url): + validate_environ(['JENKINS_USERNAME', 'JENKINS_TOKEN']) + protocol, address = url.split('://') + url = '{protocol}://{user}:{token}@{address}'.format(**dict(protocol=protocol, + user=os.environ['JENKINS_USERNAME'], + token=os.environ['JENKINS_TOKEN'], + address=address)) + return url + + +def validate_environ(expected_vars): + for env in expected_vars: + if env not in os.environ: + raise Exception('{} must be defined in environment'.format(env)) diff --git a/tools/yum.py b/tools/yum.py new file mode 100755 index 0000000..573da4c --- /dev/null +++ b/tools/yum.py @@ -0,0 +1,215 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import logging + +from tools.executor import run +from tools.rpm import RpmData + + +class YumRpm(RpmData): + @property + def arch(self): + return self['Arch'] + + +class YumInfoParserException(BaseException): + pass + + +class YumDownloaderException(BaseException): + def __init__(self, msg, failed_packages): + super(YumDownloaderException, self).__init__() + self.msg = msg + self.failed_packages = failed_packages + + def __str__(self): + return self.msg + + +class YumInfoParser(object): + """ + Parse 'yum info' output + """ + + def parse_file(self, yum_info_installed_file_path): + with open(yum_info_installed_file_path, 'r') as f: + return self.parse_installed(f.read()) + + def parse_installed(self, yum_info_installed): + return self._parse_rpms_with_regexp(yum_info_installed, r'\nInstalled Packages\n') + + def parse_available(self, yum_info_installed): + return self._parse_rpms_with_regexp(yum_info_installed, r'Available Packages\n') + + def _parse_rpms_with_regexp(self, yum_output, regexp): + parsed_output = self._split_yum_output_with(yum_output, regexp) + return [self.parse_package(pkg) for pkg in parsed_output[1].split('\n\n') if pkg] + + @staticmethod + def _split_yum_output_with(output, regexp): + parsed_output = re.split(regexp, output) + if len(parsed_output) != 2: + raise YumInfoParserException( + '{} not found from output: {}'.format(repr(regexp), output[:1000])) + return parsed_output + + @staticmethod + def parse_package(yum_info_output): + result = YumRpm() + current_key = None + for line in re.findall(r'^(.+?) : (.*)$', yum_info_output, re.MULTILINE): + parsed_key = line[0].strip() + parsed_value = line[1].rstrip(' ') + if parsed_key: + result[parsed_key] = parsed_value + current_key = parsed_key + elif current_key in ['License', 'Summary']: + result[current_key] = result[current_key] + ' ' + parsed_value + else: + result[current_key] = result[current_key] + '\n' + parsed_value + return result + + +class Yum(object): + def __init__(self): + self.config = YumConfig(filename='tmp_yum.conf') + + @classmethod + def clean_and_remove_cache(cls): + cls.clean_all() + cls.remove_cache_dir() + + @classmethod + def clean_all(cls): + run(['yum', 'clean', 'all'], raise_on_stderr=False) + + @classmethod + def remove_cache_dir(cls): + run(['rm', '-rf', '/var/cache/yum'], raise_on_stderr=True) + + @classmethod + def read_available_pkgs(cls, name, url): + filename = 'tmp_yum.conf' + yum_tmp_conf = YumConfig() + yum_tmp_conf.add_repository(name, url) + with open(filename, 'w') as f: + f.write(yum_tmp_conf.render()) + cmd = ['yum', + '--config={}'.format(filename), + '--showduplicates', + '--setopt=keepcache=0', + '--disablerepo=*', + '--enablerepo={}'.format(name), + 'info', + 'available'] + return run(cmd, raise_on_stderr=False).stdout + + def add_repo(self, name, url): + self.config.add_repository(name, url) + + def read_all_packages(self): + self.config.write() + logging.debug('Yum config:\n{}'.format(self.config)) + cmd = ['yum', + '--config={}'.format(self.config.filename), + '--showduplicates', + '--setopt=keepcache=0', + '--enablerepo=*', + 'info', + 'available'] + return run(cmd, raise_on_stderr=False).stdout + + +class YumDownloader(object): + def download(self, rpms, repositories, to_dir=None, source=False): + logging.debug('Downloading {} RPMs from repositories: {}'.format(len(rpms), + [r['name'] for r in + repositories])) + result = self._download(rpms, repositories, to_dir=to_dir, source=source) + downloaded_rpms = [rpm.rstrip('.rpm') for rpm in os.listdir(to_dir)] + not_downloaded_rpms = [rpm for rpm in rpms if rpm not in downloaded_rpms] + if len(rpms) != len(downloaded_rpms): + logging.debug('Downloaded {}/{} RPMs: {}'.format(len(downloaded_rpms), len(rpms), + downloaded_rpms)) + # Not precise way to list not downloaded RPMs - RPM name may not match the metadata + # We should read "rpm -qip" for each rpm and parse it + raise YumDownloaderException( + 'Failed to download {}/{} RPMs: {}.\nYumdownloader result: {}'.format( + len(not_downloaded_rpms), len(rpms), not_downloaded_rpms, str(result)), + not_downloaded_rpms + ) + + @staticmethod + def _download(rpms, repositories, to_dir=None, source=False): + filename = 'tmp_yum.conf' + yum_tmp_conf = YumConfig() + for r in repositories: + yum_tmp_conf.add_repository(r['name'], r['baseurl']) + with open(filename, 'w') as f: + f.write(yum_tmp_conf.render()) + logging.debug('Downloading RPMs: {}'.format(rpms)) + cmd = ['yumdownloader'] + if to_dir is not None: + cmd += ['--destdir={}'.format(to_dir)] + cmd += ['--config={}'.format(filename), + '--disablerepo=*', + '--enablerepo={}'.format(','.join([r['name'] for r in repositories])), + '--showduplicates'] + if source: + cmd.append('--source') + cmd += rpms + return run(cmd, raise_on_stderr=False) + + +class YumConfig(object): + def __init__(self, filename=None): + self.filename = filename + self.config = ['[main]', + '#cachedir=/var/cache/yum/$basearch/$releasever', + 'reposdir=/foo/bar/xyz', + 'keepcache=0', + 'debuglevel=2', + '#logfile=/var/log/yum.log', + 'exactarch=1', + 'obsoletes=1', + 'gpgcheck=0', + 'plugins=1', + 'installonly_limit=5', + 'override_install_langs=en_US.UTF-8', + 'tsflags=nodocs'] + self.repositories = [] + + def add_repository(self, name, url, exclude=None): + repo = ['[{}]'.format(name), + 'name = ' + name, + 'baseurl = ' + url, + 'enabled = 1', + 'gpgcheck = 0'] + if exclude is not None: + repo.append('exclude = ' + exclude) + self.repositories.append(repo) + + def __str__(self): + return self.render() + + def render(self): + blocks = ['\n'.join(self.config)] + ['\n'.join(repo) for repo in self.repositories] + return '\n\n'.join(blocks) + + def write(self): + with open(self.filename, 'w') as f: + f.write(self.render()) diff --git a/tools/yum_info_installed.sample b/tools/yum_info_installed.sample new file mode 100644 index 0000000..fb6f64f --- /dev/null +++ b/tools/yum_info_installed.sample @@ -0,0 +1,223 @@ +Loaded plugins: fastestmirror, priorities +Installed Packages +Name : GeoIP +Arch : x86_64 +Version : 1.5.0 +Release : 11.el7 +Size : 2.8 M +Repo : installed +From repo : purkki-centos-base +Summary : Library for country/city/organization to IP address or hostname + : mapping +URL : http://www.maxmind.com/app/c +License : LGPLv2+ and GPLv2+ and CC-BY-SA +Description : GeoIP is a C library that enables the user to find the country + : that any IP address or hostname originates from. It uses a file + : based database that is accurate as of June 2007 and can optionally + : be updated on a weekly basis by installing the GeoIP-update + : package. This database simply contains IP blocks as keys, and + : countries as values. This database should be more complete and + : accurate than using reverse DNS lookups. + : + : This package includes GeoLite data created by MaxMind, available + : from http://www.maxmind.com/ + +Name : MySQL-python +Arch : x86_64 +Version : 1.2.5 +Release : 1.el7 +Size : 284 k +Repo : installed +From repo : purkki-centos-base +Summary : An interface to MySQL +URL : https://github.com/farcepest/MySQLdb1 +License : GPLv2+ +Description : Python interface to MySQL + : + : MySQLdb is an interface to the popular MySQL database server for + : Python. The design goals are: + : + : - Compliance with Python database API version 2.0 + : - Thread-safety + : - Thread-friendliness (threads will not block each other) + : - Compatibility with MySQL 3.23 and up + : + : This module should be mostly compatible with an older interface + : written by Joe Skinner and others. However, the older version is + : a) not thread-friendly, b) written for MySQL 3.21, c) apparently + : not actively maintained. No code from that version is used in + : MySQLdb. + +Name : OpenIPMI-libs +Arch : x86_64 +Version : 2.0.19 +Release : 15.el7 +Size : 1.7 M +Repo : installed +From repo : purkki-centos-base +Summary : The OpenIPMI runtime libraries +URL : http://sourceforge.net/projects/openipmi/ +License : LGPLv2+ and GPLv2+ or BSD +Description : The OpenIPMI-libs package contains the runtime libraries for + : shared binaries and applications. + +Name : OpenIPMI-modalias +Arch : x86_64 +Version : 2.0.19 +Release : 15.el7 +Size : 210 +Repo : installed +From repo : purkki-centos-base +Summary : Module aliases for IPMI subsystem +URL : http://sourceforge.net/projects/openipmi/ +License : LGPLv2+ and GPLv2+ or BSD +Description : The OpenIPMI-modalias provides configuration file with module + : aliases of ACPI and PNP wildcards. + +Name : PyPAM +Arch : x86_64 +Version : 0.5.0 +Release : 19.el7 +Size : 51 k +Repo : installed +From repo : purkki-centos-base +Summary : PAM bindings for Python +URL : http://www.pangalactic.org/PyPAM +License : LGPLv2 +Description : PAM (Pluggable Authentication Module) bindings for Python. + +Name : PyYAML +Arch : x86_64 +Version : 3.10 +Release : 11.el7 +Size : 630 k +Repo : installed +Summary : YAML parser and emitter for Python +URL : http://pyyaml.org/ +License : MIT +Description : YAML is a data serialization format designed for human readability + : and interaction with scripting languages. PyYAML is a YAML parser + : and emitter for Python. + : + : PyYAML features a complete YAML 1.1 parser, Unicode support, + : pickle support, capable extension API, and sensible error + : messages. PyYAML supports standard YAML tags and provides + : Python-specific tags that allow to represent an arbitrary Python + : object. + : + : PyYAML is applicable for a broad range of tasks from complex + : configuration files to object serialization and persistance. + +Name : SDL +Arch : x86_64 +Version : 1.2.15 +Release : 14.el7 +Size : 467 k +Repo : installed +From repo : purkki-centos-base +Summary : A cross-platform multimedia library +URL : http://www.libsdl.org/ +License : LGPLv2+ +Description : Simple DirectMedia Layer (SDL) is a cross-platform multimedia + : library designed to provide fast access to the graphics frame + : buffer and audio device. + +Name : XStatic-Angular-common +Arch : noarch +Epoch : 1 +Version : 1.5.8.0 +Release : 1.el7 +Size : 3.1 M +Repo : installed +From repo : purkki-delorean +Summary : Common files for XStatic-Angular (XStatic packaging standard) +URL : http://angularjs.org +License : MIT +Description : Common files for XStatic-Angular (XStatic packaging standard) + +Name : acl +Arch : x86_64 +Version : 2.2.51 +Release : 12.el7 +Size : 196 k +Repo : installed +Summary : Access control list utilities +URL : http://acl.bestbits.at/ +License : GPLv2+ +Description : This package contains the getfacl and setfacl utilities needed for + : manipulating access control lists. + +Name : adwaita-cursor-theme +Arch : noarch +Version : 3.14.1 +Release : 1.el7 +Size : 1.9 M +Repo : installed +From repo : purkki-centos-base +Summary : Adwaita cursor theme +URL : http://www.gnome.org +License : LGPLv3+ or CC-BY-SA +Description : The adwaita-cursor-theme package contains a modern set of cursors + : originally designed for the GNOME desktop. + +Name : adwaita-icon-theme +Arch : noarch +Version : 3.14.1 +Release : 1.el7 +Size : 13 M +Repo : installed +From repo : purkki-centos-base +Summary : Adwaita icon theme +URL : http://www.gnome.org +License : LGPLv3+ or CC-BY-SA +Description : This package contains the Adwaita icon theme used by the GNOME + : desktop. + +Name : agg +Arch : x86_64 +Version : 2.5 +Release : 18.el7 +Size : 410 k +Repo : installed +From repo : purkki-centos-base +Summary : Anti-Grain Geometry graphical rendering engine +URL : http://www.antigrain.com +License : GPLv2+ +Description : A High Quality Rendering Engine for C++. + +Name : alsa-lib +Arch : x86_64 +Version : 1.1.1 +Release : 1.el7 +Size : 1.2 M +Repo : installed +From repo : purkki-centos-base +Summary : The Advanced Linux Sound Architecture (ALSA) library +URL : http://www.alsa-project.org/ +License : LGPLv2+ +Description : The Advanced Linux Sound Architecture (ALSA) provides audio and + : MIDI functionality to the Linux operating system. + : + : This package includes the ALSA runtime libraries to simplify + : application programming and provide higher level functionality as + : well as support for the older OSS API, providing binary + : compatibility for most OSS programs. + +Name : ansible +Arch : noarch +Version : 2.3.0.0 +Release : 3.el7 +Size : 27 M +Repo : installed +From repo : purkki-delorean +Summary : SSH-based configuration management, deployment, and task execution + : system +URL : http://ansible.com +License : GPLv3+ +Description : + : Ansible is a radically simple model-driven configuration + : management, multi-node deployment, and remote task execution + : system. Ansible works over SSH and does not require any software + : or daemons to be installed on remote nodes. Extension modules can + : be written in any language and are transferred to managed machines + : automatically. diff --git a/tools/yum_test.py b/tools/yum_test.py new file mode 100755 index 0000000..1a3dca6 --- /dev/null +++ b/tools/yum_test.py @@ -0,0 +1,67 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=wildcard-import +import os +import pytest + +from tools.test_data_yum import bash_yum_info, \ + conntrack_tools_yum_info, \ + pacemaker_yum_info +from tools.yum import YumInfoParser +from tools.yum_test_data import bash_expected, pacemaker_expected, \ + yum_info_installed_header, \ + yum_info_available_header, \ + yum_info_available_header2 +from tools.yum_test_data import conntrack_tools_expected + + +@pytest.mark.parametrize('yum_info, expected_output', [ + (bash_yum_info, bash_expected), + (conntrack_tools_yum_info, conntrack_tools_expected), + (pacemaker_yum_info, pacemaker_expected) +]) +def test_parse_package(yum_info, expected_output): + parsed = YumInfoParser().parse_package(yum_info) + expected = expected_output + assert parsed == expected + + +def test_parse_installed(): + fake_out = '\n'.join([yum_info_installed_header, + bash_yum_info, + conntrack_tools_yum_info]) + parsed = YumInfoParser().parse_installed(fake_out) + expected = [bash_expected, conntrack_tools_expected] + assert parsed == expected + + +@pytest.mark.parametrize('available_header', [ + yum_info_available_header, + yum_info_available_header2 +]) +def test_parse_available(available_header): + fake_out = '\n'.join([available_header, + bash_yum_info, + conntrack_tools_yum_info]) + parsed = YumInfoParser().parse_available(fake_out) + expected = [bash_expected, conntrack_tools_expected] + assert parsed == expected + + +def test_parse_file(): + test_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'yum_info_installed.sample') + parsed = YumInfoParser().parse_file(test_file) + assert len(parsed) == 14 diff --git a/tools/yum_test_data.py b/tools/yum_test_data.py new file mode 100755 index 0000000..d5d6a8b --- /dev/null +++ b/tools/yum_test_data.py @@ -0,0 +1,106 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=invalid-name,line-too-long + +yum_info_installed_header = """Loaded plugins: fastestmirror, priorities +Loading mirror speeds from cached hostfile +Installed Packages +""" + +yum_info_available_header = """Added tmprepo repo from http://purkki/mirror/centos/snapshot/20170705-2/7/os/x86_64/ +Available Packages +""" # noqa + +yum_info_available_header2 = """Available Packages +""" + +bash_expected = { + 'Name': 'bash', + 'Arch': 'x86_64', + 'Version': '4.2.46', + 'Release': '21.el7_3', + 'Size': '3.5 M', + 'Repo': 'installed', + 'From repo': 'updates', + 'Summary': 'The GNU Bourne Again shell', + 'URL': 'http://www.gnu.org/software/bash', + 'License': 'GPLv3+', + 'Description': '\n'.join( + ['The GNU Bourne Again shell (Bash) is a shell or command language', + 'interpreter that is compatible with the Bourne shell (sh). Bash', + 'incorporates useful features from the Korn shell (ksh) and the C', + 'shell (csh). Most sh scripts can be run by bash without', + 'modification.']) +} + +conntrack_tools_expected = { + 'Name': 'conntrack-tools', + 'Arch': 'x86_64', + 'Version': '1.4.4', + 'Release': '3.el7_3', + 'Size': '550 k', + 'Repo': 'installed', + 'From repo': 'centos-updates', + 'Summary': ' '.join( + ['Manipulate netfilter connection tracking table and run High', + 'Availability']), + 'URL': 'http://netfilter.org', + 'License': 'GPLv2', + 'Description': '\n'.join( + ['With conntrack-tools you can setup a High Availability cluster and', + 'synchronize conntrack state between multiple firewalls.', + '', + 'The conntrack-tools package contains two programs:', + '- conntrack: the command line interface to interact with the', + ' connection tracking system.', + '- conntrackd: the connection tracking userspace daemon that can be', + ' used to deploy highly available GNU/Linux firewalls and collect', + ' statistics of the firewall use.', + '', + 'conntrack is used to search, list, inspect and maintain the', + 'netfilter connection tracking subsystem of the Linux kernel.', + 'Using conntrack, you can dump a list of all (or a filtered', + 'selection of) currently tracked connections, delete connections', + 'from the state table, and even add new ones.', + 'In addition, you can also monitor connection tracking events, e.g.', + 'show an event message (one line) per newly established connection.']) +} + +pacemaker_expected = { + 'Name': 'pacemaker', + 'Arch': 'x86_64', + 'Version': '1.1.15', + 'Release': '11.el7_3.5', + 'Size': '1.1 M', + 'Repo': 'installed', + 'From repo': 'purkki-centos-updates', + 'Summary': 'Scalable High-Availability cluster resource manager', + 'URL': 'http://www.clusterlabs.org', + 'License': 'GPLv2+ and LGPLv2+', + 'Description': '\n'.join( + ['Pacemaker is an advanced, scalable High-Availability cluster', + 'resource manager for Corosync, CMAN and/or Linux-HA.', + '', + 'It supports more than 16 node clusters with significant', + 'capabilities for managing resources and dependencies.', + '', + 'It will run scripts at initialization, when machines go up or', + 'down, when related resources fail and can be configured to', + 'periodically check resource health.', + '', + 'Available rpmbuild rebuild options:', + ' --with(out) : cman stonithd doc coverage profiling pre_release', + 'hardening']) +} diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..80cf235 --- /dev/null +++ b/tox.ini @@ -0,0 +1,35 @@ +[tox] +envlist = py27, pylint +skipsdist=true + +[testenv] +commands = py.test -v \ + --basetemp={toxinidir}/.pytest-tmpdir \ + --junitxml=junit.xml \ + --pep8 \ + --cov-config {toxinidir}/.coveragerc \ + --cov-branch \ + --cov-report term-missing \ + --cov-report html:htmlcov \ + --cov=. \ + {posargs:.} + +setenv = + PYTHONPATH = {toxinidir}/tests/mocked_dependencies:{toxinidir}/src +deps=-rrequirements.txt + +[pytest] +cache_dir = .pytest-cache +pep8maxlinelength = 100 + +[testenv:pylint] +basepython = python2.7 +commands = pylint --rcfile={toxinidir}/pylintrc {posargs:tools/} + +deps=pylint < 2.0 + -rrequirements.txt + +[testenv:clean] +deps= +whitelist_externals = rm +commands = rm -rf .coverage .pytest-cache .pytest-tmpdir junit.xml htmlcov