--- /dev/null
+[run]
+omit =
+ .tox/*
--- /dev/null
+.idea
+*.pyc
+.coverage
+.coverage.*
+.pytest-cache/
+.pytest-tmpdir/
+.tox/
+htmlcov/
+junit.xml
+tmp_yum.conf
--- /dev/null
+[gerrit]
+host=gerrit.akraino.org
+port=29418
+project=ta/build-tools
+defaultremote=origin
+
--- /dev/null
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
--- /dev/null
+# Unit tests and static analysis
+
+Execute with:
+ $ tox
+
+Execute only subset with verbose diff on assertion errors:
+ $ tox -e py27 -- -vv tools/script/process_rpmdata_test.py::test_components
+
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+set -e
+
+scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
+source $scriptdir/lib.sh
+_read_manifest_vars
+
+tmp=$WORKTMP/install_cd
+iso_build_dir=$tmp/build
+
+input_image="$WORKTMP/goldenimage/${GOLDEN_IMAGE_NAME}"
+output_image_path="$1"
+[[ $output_image_path =~ ^/ ]] || output_image_path=$(pwd)/$output_image_path
+output_bootcd_path="$2"
+[[ $output_bootcd_path =~ ^/ ]] || output_bootcd_path=$(pwd)/$output_bootcd_path
+mkdir -p $tmp
+rm -rf $iso_build_dir
+mkdir -p $iso_build_dir
+
+reposnap_base=$(_read_build_config DEFAULT centos_reposnap_base)
+release_version=$PRODUCT_RELEASE_LABEL
+reposnap_base_dir="${reposnap_base}/os/x86_64/"
+iso_image_label=$(_read_build_config DEFAULT iso_image_label)
+cd_efi_dir="${reposnap_base_dir}/EFI"
+cd_images_dir="${reposnap_base_dir}/images"
+cd_isolinux_dir="${reposnap_base_dir}/isolinux"
+
+remove_extra_slashes_from_url() {
+ echo $1 | sed -re 's#([^:])//+#\1/#g'
+}
+
+get_nexus() {
+ $scriptdir/nexus3_dl.sh \
+ $nexus_url \
+ $(basename $nexus_reposnaps) \
+ ${reposnap_base#$nexus_reposnaps/}/os/x86_64 $@
+}
+
+wget_dir() {
+ local url=$1
+ echo $url | grep -q /$ || _abort "wget path '$url' must end with slash for recursive wget"
+ # if any extra slashes within path, it messes up the --cut-dirs count
+ url=$(remove_extra_slashes_from_url $url)
+ # count cut length in case url depth changes
+ cut_dirs=$(echo $url | sed -re 's|.*://[^/]+/(.+)|\1|' -e 's|/$||' | grep -o / | wc -l)
+ wget -N -r --no-host-directories --no-verbose --cut-dirs=${cut_dirs} --reject index.html* --no-parent $url
+}
+
+pushd $iso_build_dir
+
+# Get files needed for generating CD image.
+if echo $reposnap_base_dir | grep -E "https?://nexus3"; then
+ nexus_url=$(_read_build_config DEFAULT nexus_url)
+ nexus_reposnaps=$(_read_build_config DEFAULT nexus_reposnaps)
+ get_nexus "EFI/BOOT" "EFI/BOOT/fonts"
+ get_nexus "images:*efiboot.img" "images/pxeboot"
+ get_nexus "isolinux"
+else
+ wget_dir ${cd_efi_dir}/
+ wget_dir ${cd_images_dir}/
+ rm -rf images/boot.iso
+ sync
+ wget_dir ${cd_isolinux_dir}/
+fi
+chmod +w -R isolinux/ EFI/ images/
+
+if [ -e $scriptdir/isolinux/isolinux.cfg ]; then
+ cp $scriptdir/isolinux/isolinux.cfg isolinux/isolinux.cfg
+else
+ sed -i "s/^timeout.*/timeout 100/" isolinux/isolinux.cfg
+ sed -i "s/^ - Press.*/Beginning the cloud installation process/" isolinux/boot.msg
+ sed -i "s/^#menu hidden/menu hidden/" isolinux/isolinux.cfg
+ sed -i "s/menu default//" isolinux/isolinux.cfg
+ sed -i "/^label linux/amenu default" isolinux/isolinux.cfg
+ sed -i "/append initrd/ s/$/ console=tty0 console=ttyS1,115200/" isolinux/isolinux.cfg
+fi
+cp -f $scriptdir/akraino_splash.png isolinux/splash.png
+
+popd
+
+pushd $tmp
+
+ # Copy latest kernel and initrd-provisioning from boot dir
+virt-copy-out -a $input_image /boot/ ./
+chmod u+w boot/
+rm -f $iso_build_dir/isolinux/vmlinuz $iso_build_dir/isolinux/initrd.img
+KVER=`ls -lrt boot/vmlinuz-* |grep -v rescue |tail -n1 |awk -F 'boot/vmlinuz-' '{print $2}'`
+cp -fp boot/vmlinuz-${KVER} $iso_build_dir/isolinux/vmlinuz
+cp -fp boot/initrd-provisioning.img $iso_build_dir/isolinux/initrd.img
+rm -rf boot/
+
+echo "Generating boot iso"
+_run_cmd genisoimage -U -r -v -T -J -joliet-long \
+ -V "${release_version}" -A "${release_version}" -P ${iso_image_label} \
+ -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 \
+ -boot-info-table -eltorito-alt-boot -e images/efiboot.img -no-emul-boot \
+ -o boot.iso $iso_build_dir
+_publish_image $tmp/boot.iso $output_bootcd_path
+
+cp -f ${input_image} $iso_build_dir/
+
+# Keep the placeholder
+mkdir -p $iso_build_dir/rpms
+
+echo "Generating product iso"
+_run_cmd genisoimage -U -r -v -T -J -joliet-long \
+ -V "${release_version}" -A "${release_version}" -P ${iso_image_label} \
+ -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 \
+ -boot-info-table -eltorito-alt-boot -e images/efiboot.img -no-emul-boot \
+ -o release.iso $iso_build_dir
+_run_cmd isohybrid $tmp/release.iso
+_publish_image $tmp/release.iso $output_image_path
+
+echo "Clean up to preserve workspace footprint"
+rm -f $iso_build_dir/$(basename ${input_image})
+rm -rf $iso_build_dir/rpms
+
+popd
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+set -e
+
+scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
+source $scriptdir/lib.sh
+
+_run_cmd_as_step _create_localrepo
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+set -e
+
+scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
+source $scriptdir/lib.sh
+
+_get_projects_all() { repo list -g torpm -n | sort; }
+
+_get_project_dirs() {
+ for i in $@; do
+ repo info -l $i | grep "^Mount path: " | awk '{print $3}' | tr '\n' ' '
+ done
+}
+
+build_rpms()
+{
+ local work=$1
+ shift
+ rm -rf $work
+ mkdir -p $work
+
+ CENTOS_SOURCES="$(_read_build_config rpm centos_sources)" \
+ $RPM_BUILDER/makebuild.py \
+ -m $RPM_BUILDER_SETTINGS \
+ -w $work \
+ $@ #-v --nowipe
+}
+
+# build one or all projects
+if [ -n "$1" ]; then
+ projects_to_build=$@
+else
+ projects_to_build="$MANIFEST_PATH $(_get_project_dirs $(_get_projects_all))"
+ _run_cmd $LIBDIR/prepare_manifest.sh
+fi
+
+rpmwork=$WORKTMP/rpms
+build_rpms $rpmwork $projects_to_build
+_add_rpms_to_repos_from_workdir $rpmwork
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
+source $scriptdir/lib.sh
+
+config_ini=${1:-$BUILD_CONFIG_INI}
+output_dir=${2:-$REPO_FILES}
+[ -f "$config_ini" ] || _abort "Config INI '$config_ini' not found"
+
+gen() {
+ PYTHONPATH=$LIBDIR python $LIBDIR/tools/script/generate_repo_files.py \
+ --config-ini $config_ini \
+ --output-dir $output_dir \
+ $1
+}
+
+mkdir -p $output_dir
+
+gen localrepo
+gen repositories
+gen baseimage-repositories
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+set -e
+
+scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
+source $scriptdir/lib.sh
+
+output_image_path="$1"
+[[ $output_image_path =~ ^/ ]] || output_image_path=$(pwd)/$output_image_path
+rpm_info_output_dir=$2
+[[ $rpm_info_output_dir =~ ^/ ]] || rpm_info_output_dir=$(pwd)/$rpm_info_output_dir
+
+docker_dib_image=dib:2.0
+_load_docker_image $docker_dib_image
+
+docker run \
+ --rm \
+ --privileged \
+ -v /dev:/dev \
+ -v $WORK:/work \
+ $docker_dib_image \
+ $(realpath --relative-to $WORK $scriptdir)/create_golden_image.sh
+
+_publish_image ${TMP_GOLDEN_IMAGE}.qcow2 $output_image_path/${GOLDEN_IMAGE_NAME}
+
+input_dir=$WORKTMP/rpmdata
+mkdir $input_dir
+cp -r \
+ $BUILD_CONFIG_INI $RPMLISTS/rpm_info_installed $RPMLISTS/yum_info_installed $RPMLISTS/crypto_rpms.json $RPMLISTS/boms \
+ $input_dir
+
+$LIBDIR/create_rpmdata_in_docker.sh \
+ $input_dir \
+ $RPMLISTS \
+ -v
+
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+set -e
+
+scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
+source $scriptdir/lib.sh
+_read_manifest_vars
+
+_run_cmd_as_step _initialize_work_dirs
+
+_run_cmd_as_step $LIBDIR/repo_summary.sh
+
+_run_cmd_as_step $LIBDIR/create_mock_config.sh
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+set -e
+set -o pipefail
+
+scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
+source $scriptdir/lib.sh
+
+BASE_IMAGE_URL=${BASE_IMAGE_URL:-$(_read_build_config DEFAULT base_image)}
+CENTOS_SNAP=${CENTOS_SNAP:-$(_read_build_config DEFAULT centos_reposnap)}
+
+BASE_IMAGE_NAME=`echo $BASE_IMAGE_URL | awk -F "/" '{print $NF}'`
+BASE_IMAGE_SIZE="8GiB"
+
+wget_args=""
+[ -n "$GOLDEN_BASE_IMAGE_FETCH_USER" ] && wget_args="$wget_args --http-user=$GOLDEN_BASE_IMAGE_FETCH_USER"
+[ -n "$GOLDEN_BASE_IMAGE_FETCH_PASSWORD" ] && wget_args="$wget_args --http-password=$GOLDEN_BASE_IMAGE_FETCH_PASSWORD"
+
+fetch_image() {
+ sourceurl=$1
+ echo "Download $sourceurl to $WORKTMP/base-img"
+ mkdir -pv $WORKTMP/base-img
+ pushd $WORKTMP/base-img
+ _run_cmd wget --no-check-certificate --no-verbose -N \
+ --auth-no-challenge $wget_args \
+ $sourceurl
+ popd
+}
+
+fetch_image $BASE_IMAGE_URL
+cp $MANIFEST_PATH/packages.yaml $scriptdir/dib_elements/myproduct/package-installs.yaml
+
+DIB_DEBUG_TRACE=1 \
+ FS_TYPE=xfs \
+ PACKAGES_TO_INSTALL="$(_get_package_list install)" \
+ PACKAGES_TO_UNINSTALL="$(_get_package_list uninstall)" \
+ DIB_RPMLISTS=$RPMLISTS \
+ DIB_CHECKSUM=$CHECKSUM_DIR \
+ DIB_LOCAL_REPO=$REPO_DIR \
+ DIB_DISTRIBUTION_MIRROR=$CENTOS_SNAP \
+ DIB_YUM_REPO_CONF="${REPO_FILES}/repositories.repo ${REPO_FILES}/localrepo.repo" \
+ DIB_LOCAL_IMAGE=$WORKTMP/base-img/$BASE_IMAGE_NAME \
+ ELEMENTS_PATH=$scriptdir/dib_elements/ \
+ /usr/bin/disk-image-create --root-label img-rootfs --image-size $BASE_IMAGE_SIZE vm centos7 selinux-permissive myproduct -o $TMP_GOLDEN_IMAGE
+
+rm -rf $WORKTMP/base-img
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
+source $scriptdir/lib.sh
+
+config_ini=${1:-$BUILD_CONFIG_INI}
+output_repo_files_dir=${2:-$REPO_FILES}
+output_mock_dir=${2:-$(dirname $RPM_BUILDER_SETTINGS)}
+[ -f "$config_ini" ] || _abort "Config INI '$config_ini' not found"
+
+# Create .repo files
+$LIBDIR/build_step_create_yum_repo_files.sh $config_ini $output_repo_files_dir
+
+# Create mock config
+mkdir -p $output_mock_dir
+cp $LIBDIR/mock/logging.ini $output_mock_dir/
+cp $LIBDIR/mock/site-defaults.cfg $output_mock_dir/
+mock_cfg=$output_mock_dir/mock.cfg
+sed -e "/#REPOSITORIES#/r $output_repo_files_dir/repositories.repo" $LIBDIR/mock/mock.cfg.template > $mock_cfg
+sed -i \
+ -e "s/#RPM_PACKAGER#/\"$(_read_build_config $config_ini rpm packager)\"/" \
+ -e "s/#RPM_VENDOR#/\"$(_read_build_config $config_ini rpm vendor)\"/" \
+ -e "s/#RPM_LICENSE#/\"$(_read_build_config $config_ini rpm license)\"/" \
+ -e "s/#RPM_RELEASE_ID#/\"$(_read_build_config $config_ini rpm release_id)\"/" \
+ $mock_cfg
+
+docker_sock=/var/run/docker.sock
+if [ -S "$docker_sock" ]; then
+ sed -i "/#ADDITIONAL_CONFIG_OPTS#/a config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('$docker_sock', '$docker_sock'))" $mock_cfg
+fi
+if [ -n "$DOCKER_HOST" ]; then
+ sed -i "/#ADDITIONAL_CONFIG_OPTS#/a config_opts['environment']['DOCKER_HOST'] = '${DOCKER_HOST}:2375'" $mock_cfg
+fi
+
+# HACK ??
+# include kernels in order for yum development group to install
+sed -i -e 's/exclude=kernel/#exclude=kernel/' $mock_cfg
+
+echo "Wrote mock configuration to: $output_mock_dir"
--- /dev/null
+#!/bin/sh
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
+source $scriptdir/lib.sh
+
+docker_image=buildtools:2.0
+_load_docker_image $docker_image
+
+function _resolve_abs_path() {
+ if ! echo $1 | grep -q "^/"; then
+ echo $(pwd)/$1
+ else
+ echo $1
+ fi
+}
+
+input_dir=$(_resolve_abs_path ${1:?ERROR, please give input dir as argument})
+output_dir=$(_resolve_abs_path ${2:?ERROR, please give output dir as argument})
+shift; shift
+
+# Run!
+input_mp=/input
+output_mp=/output
+docker run \
+ --rm \
+ -e PYTHONPATH=/work \
+ -e BUILD_URL -e JENKINS_USERNAME -e JENKINS_TOKEN -e WORKSPACE \
+ -v $scriptdir:/work \
+ -v $input_dir:$input_mp \
+ -v $output_dir:$output_mp \
+ -w /work \
+ $docker_image \
+ python tools/script/create_rpm_data.py \
+ --build-config-path $input_mp/build_config.ini \
+ --yum-info-path $input_mp/yum_info_installed \
+ --rpm-info-path $input_mp/rpm_info_installed \
+ --crypto-info-path $input_mp/crypto_rpms.json \
+ --boms-path $input_mp/boms \
+ --output-json $output_mp/rpmdata.json \
+ --output-csv $output_mp/rpmdata.csv \
+ --output-ms-csv $output_mp/rpmdata-ms.csv \
+ --output-rpmlist $output_mp/rpmlist \
+ $*
+
+docker run \
+ --rm \
+ -e PYTHONPATH=/work \
+ -v $scriptdir:/work \
+ -v $output_dir:$output_mp \
+ -w /work \
+ $docker_image \
+ python tools/script/process_rpmdata.py \
+ --rpmdata-path $output_mp/rpmdata.json \
+ --output-components $output_mp/components.json \
+ --output-components-csv $output_mp/components-ms.csv \
+
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+ set -x
+fi
+set -eu
+set -o pipefail
+
+[ -n "$TARGET_ROOT" ]
+mv $TARGET_ROOT/root/yum_info_installed $DIB_RPMLISTS
+mv $TARGET_ROOT/root/rpm_info_installed $DIB_RPMLISTS
+mv $TARGET_ROOT/root/boms $DIB_RPMLISTS
+mv $TARGET_ROOT/root/binary_checksum.md5 $DIB_CHECKSUM
+mv $TARGET_ROOT/root/binary_checksum.sha256 $DIB_CHECKSUM
+mv $TARGET_ROOT/crypto_rpms.json $DIB_RPMLISTS
--- /dev/null
+#!/bin/sh
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+SCRIPT_DIR="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
+
+cp -rf ${SCRIPT_DIR}/* $TMP_HOOKS_PATH/
--- /dev/null
+#!/usr/bin/python
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+import json
+import sys
+
+# following libs are found from openssl, libgcrypto, openssh with rpm -qR
+# in the future, the libs need to be updated with exact version number
+
+input_capabilities = []
+output_rpms = []
+input_packages = [
+ 'openssl',
+ 'openssh',
+ 'p11-kit',
+ 'openssl-libs',
+ 'libgcrypt',
+ 'libgcrypt.so',
+ 'python2-cryptography',
+ 'sshpass',
+ 'm2crypto',
+ 'erlang-crypto',
+ 'openssh-clients',
+ 'python2-passlib',
+ 'python-paramiko',
+ 'python-keyring',
+ 'python2-asn1crypto',
+ 'python2-cryptography',
+ 'python2-pyasn1',
+ 'xstatic-jsencrypt-common',
+ 'krb5-libs'
+ ]
+crypto_capabilities = [
+ 'rtld(',
+ 'libcrypt.so'
+ ]
+
+
+def output_rpm_command(lib, command):
+ command.append(lib)
+ try:
+ output = subprocess.check_output(command)
+ except:
+ output = ""
+ return output
+
+
+def build_capability_list_dynamically():
+ map_dict = {}
+ command = ['rpm',
+ '-qa',
+ '--queryformat',
+ '[%{=NAME}-%{VERSION}-%{RELEASE}.%{ARCH} %{PROVIDES}\n]']
+ output = subprocess.check_output(command)
+ for item in output.splitlines():
+ items = item.split(' ')
+ if items:
+ if items[0] in map_dict:
+ capa_list = map_dict[items[0]]
+ capa_list.append(items[1])
+ map_dict[items[0]] = capa_list
+ else:
+ map_dict[items[0]] = [items[1]]
+ for rpms, caps in map_dict.items():
+ for cap in caps:
+ for item in crypto_capabilities:
+ if cap.startswith(item):
+ if cap not in input_capabilities:
+ input_capabilities.append(cap)
+ command = ['rpm', '-q', '--whatprovides']
+ output = output_rpm_command(cap, command)
+ name = output.strip('\n')
+ result = filter(lambda rpm: rpm['name'] == name, output_rpms)
+ if not result:
+ output_rpms.append({'name': name, 'provides': [cap]})
+ else:
+ sources_list = result[0]['provides']
+ if cap not in sources_list:
+ sources_list.append(cap)
+
+
+def capability_dependencies_with_whatrequires():
+ for capability in input_capabilities:
+ command = ['rpm', '-q', '--whatrequires']
+ output = output_rpm_command(capability, command)
+ if output:
+ for item in output.splitlines():
+ result = filter(lambda rpm: rpm['name'] == item, output_rpms)
+ if not result:
+ output_rpms.append({'name': item, 'requires': [capability]})
+ else:
+ sources_list = result[0]['requires']
+ if capability not in sources_list:
+ sources_list.append(capability)
+
+
+def package_dependencies_with_provides():
+ for package in input_packages:
+ command = ['rpm', '-q']
+ name = output_rpm_command(package, command)
+ name = name.strip('\n')
+ output_rpms.append({'name': name, 'requires': []})
+ command = ['rpm', '-ql', '--provides']
+ output = output_rpm_command(package, command)
+ for item in output.splitlines():
+ input_capabilities.append(item)
+
+
+if __name__ == '__main__':
+ build_capability_list_dynamically()
+ package_dependencies_with_provides()
+ capability_dependencies_with_whatrequires()
+ rpms_json = json.dumps(output_rpms, sort_keys=True, indent=4)
+ with open(sys.argv[1], "w") as f:
+ f.write(rpms_json)
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+ set -x
+fi
+set -eu
+set -o pipefail
+
+rpm -qa kernel | sort -V | head -n -1 | xargs rpm -e || true
+
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+ set -x
+fi
+set -eu
+set -o pipefail
+
+yum info installed > /root/yum_info_installed
+rpm -qai --queryformat "Obsoletes : [%{OBSOLETES},]\n" > /root/rpm_info_installed
+
+# Collect bills of material, RPM subcomponent lists
+boms_dir=/root/boms
+mkdir $boms_dir
+for bom_path in $(rpm -qal | egrep "/[^/]+-bom.json$" || :); do
+ cp $bom_path $boms_dir/$(rpm -qf $bom_path)
+done
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+ set -x
+fi
+set -eu
+set -o pipefail
+
+echo "softdep bonding pre: mlx5_core ixgbe" > /etc/modprobe.d/bonding.conf
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+ set -x
+fi
+set -eu
+set -o pipefail
+
+sed -i -e "s/console=ttyS0,115200/console=ttyS1,115200/g" /etc/default/grub
+grub2-mkconfig -o /boot/grub2/grub.cfg
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+ set -x
+fi
+set -eu
+set -o pipefail
+
+md5sum $(find / ! -path "/tmp/*" -executable -type f) > /root/binary_checksum.md5
+sha256sum $(find / ! -path "/tmp/*" -executable -type f) > /root/binary_checksum.sha256
+
+# Remove checksum that has been modified during deployment
+changed_checksums=( "resolv.conf" "mariadb.sg" "rabbitmq-server.sg" "audit.rules" "influxdb.conf" "influxdb-1" )
+for checksum in ${changed_checksums[@]}
+do
+ sed -i "/${checksum}/d" /root/binary_checksum.md5
+ sed -i "/${checksum}/d" /root/binary_checksum.sha256
+done
+
+# Add checksums that will be added during deployment
+checksums_to_add=(\
+"/etc/openstack-dashboard/enabled/_2200_ironic.py" \
+"/etc/openstack-dashboard/enabled/_2200_ironic.pyc" \
+"/etc/openstack-dashboard/enabled/_2200_ironic.pyo" \
+"/etc/openstack-dashboard/local_settings" \
+"/etc/openstack-dashboard/nova_policy.json" \
+"/etc/openstack-dashboard/neutron_policy.json" \
+"/etc/openstack-dashboard/keystone_policy.json" \
+"/etc/openstack-dashboard/glance_policy.json" \
+"/etc/openstack-dashboard/cinder_policy.json" \
+"/etc/openstack-dashboard/cinder_policy.d/consistencygroup.yaml" \
+"/etc/openstack-dashboard/nova_policy.d/api-extensions.yaml" \
+)
+for f in ${checksums_to_add[@]}; do
+ if test -f ${f}; then
+ md5sum ${f} >> /root/binary_checksum.md5
+ sha256sum ${f} >> /root/binary_checksum.sha256
+ fi
+done
+
+# Include docker images
+docker_images=$( find /var/lib/crf-images/docker-images/infra/ -type f ) || :
+if [ -n "${docker_images}" ]; then
+ md5sum ${docker_images} >> /root/binary_checksum.md5
+ sha256sum ${docker_images} >> /root/binary_checksum.sha256
+fi
+
+# Sort the checksum files
+sort -k2 -o /root/binary_checksum.md5 /root/binary_checksum.md5
+sort -k2 -o /root/binary_checksum.sha256 /root/binary_checksum.sha256
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+ set -x
+fi
+set -eu
+set -o pipefail
+
+rm -rf /etc/udev/rules.d/99-dhcp-all-interfaces.rules
+# below file is coming from the base image we use
+rm -rf /etc/sysconfig/network-scripts/ifcfg-eth0
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+ set -x
+fi
+set -eu
+set -o pipefail
+
+sed -i 's/^#*UseDNS .*/UseDNS no/' /etc/ssh/sshd_config
+sed -i 's/^#*GSSAPIAuthentication .*/GSSAPIAuthentication no/' /etc/ssh/sshd_config
+
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script should be removed as soon as possible.
+# It is hardcoding root password to root.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+ set -x
+fi
+set -eu
+set -o pipefail
+
+echo "root:root" | chpasswd
+
+sed -i 's/.*PermitRootLogin yes/PermitRootLogin yes/' /etc/ssh/sshd_config
+sed -i 's/.*PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+ set -x
+fi
+set -eu
+set -o pipefail
+
+rm -rf /etc/yum.repos.d/localrepo.repo
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+ set -x
+fi
+set -eu
+set -o pipefail
+
+cp -f /tmp/in_target.d/collect_ecc.py .
+python collect_ecc.py crypto_rpms.json
+rm -f collect_ecc.py
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+ set -x
+fi
+set -eu
+set -o pipefail
+
+scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
+
+rpm -q --queryformat '' $PACKAGES_TO_INSTALL
+
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+ set -x
+fi
+set -eu
+set -o pipefail
+
+pkgs="yum-plugin-priorities"
+yum install -y $pkgs
+rpm -q --queryformat '' $pkgs
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+ set -x
+fi
+set -eu
+set -o pipefail
+
+[ -n "$TARGET_ROOT" ]
+
+# Remove all the repo file content in the target root.
+for file in `ls $TMP_MOUNT_PATH/etc/yum.repos.d/*.repo`; do
+ sudo echo > $file
+done
+
+# mount the local repo dir on target loop device
+mkdir -p $TARGET_ROOT/$DIB_LOCAL_REPO
+sudo mount --bind $DIB_LOCAL_REPO $TARGET_ROOT/$DIB_LOCAL_REPO
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+ set -x
+fi
+set -eu
+set -o pipefail
+
+[ -n "$TARGET_ROOT" ]
+
+# Empty the grub defaults file coming from the base image.
+# Native DIB element running in the finalise stage is appending to the original file.
+# This was resulting in duplicate entries.
+echo > $TMP_MOUNT_PATH/etc/default/grub
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM fedora:27
+
+RUN dnf install -y python wget yum-utils && \
+ pip install requests
+WORKDIR /work
+
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM centos:7.5.1804
+RUN \
+ yum-config-manager --add-repo http://mirror.centos.org/centos/7/cloud/x86_64/openstack-pike/ && \
+ yum-config-manager --add-repo https://trunk.rdoproject.org/centos7-pike/57/c7/57c7be250c919c04b51361d4d42e95818cfec5a5_15fc9723 && \
+ yum install --nogpgcheck -y diskimage-builder \
+ git \
+ python \
+ wget \
+ which \
+ findutils \
+ systemd-udev \
+ PyYAML \
+ parted \
+ sudo \
+ e2fsprogs \
+ xfsprogs
+
+WORKDIR /work
--- /dev/null
+Rules
+-----
+
+- CI must pull images from the intranet and not from the internet
+- CI must use always versioned tags, not the "latest"
+
+
+How to use?
+-----------
+
+e.g.
+ $ curl -L http://<some-url>/dib:1.2.tar | docker load
+
+
+How to develop?
+---------------
+
+#. Create local image
+
+ .. code-block::
+
+ $ docker build --rm -f Dockerfile-dib -t dib:<my-version> .
+ e.g.
+ $ docker build --rm -f Dockerfile-dib -t dib:1.2 .```
+
+#. Upload to file server
+
+ .. code-block::
+
+ $ docker save dib:1.2 > dib:1.2.tar
+ $ scp ./dib\:1.2.tar <user>@<server>:/var/www/<some-path>/
--- /dev/null
+default vesamenu.c32
+timeout 100
+
+display boot.msg
+
+# Clear the screen when exiting the menu, instead of leaving the menu displayed.
+# For vesamenu, this means the graphical background is still displayed without
+# the menu itself for as long as the screen remains in graphics mode.
+menu clear
+menu background splash.png
+menu title Cloud
+menu vshift 8
+menu rows 18
+menu margin 8
+menu hidden
+
+menu color border 0 #ffffffff #ee000000 std
+menu color title 0 #ffffffff #ee000000 std
+menu color disabled 0 #ffffffff #ee000000 std
+menu color sel 0 #ffffffff #85000000 std
+menu color unsel 0 #ffffffff #ee000000 std
+menu color pwdheader 0 #ff000000 #99ffffff rev
+menu color pwdborder 0 #ff000000 #99ffffff rev
+menu color pwdentry 0 #ff000000 #99ffffff rev
+menu color hotkey 0 #ff00ff00 #ee000000 std
+menu color hotsel 0 #ffffffff #85000000 std
+menu color cmdmark 0 #ffffffff #ee000000 std
+menu color cmdline 0 #ffffffff #ee000000 std
+
+# Do not display the actual menu unless the user presses a key. All that is displayed is a timeout message.
+
+menu tabmsg Press Tab for full configuration options on menu items.
+
+menu separator # insert an empty line
+menu separator # insert an empty line
+
+label linux
+menu default
+ menu label ^Install Cloud
+ kernel vmlinuz
+ append initrd=initrd.img inst.stage2=hd:LABEL=CentOS\x207\x20x86_64 quiet console=tty0 console=ttyS1,115200
+
+label check
+ menu label Test this ^media & install Cloud
+ kernel vmlinuz
+ append initrd=initrd.img inst.stage2=hd:LABEL=CentOS\x207\x20x86_64 rd.live.check quiet console=tty0 console=ttyS1,115200
+
+menu end
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o pipefail
+set -e
+
+LIBDIR="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
+
+PUBLISH_RESULTS="${PUBLISH_RESULTS:-false}"
+VIRT_CUSTOMIZE_MEM="${VIRT_CUSTOMIZE_MEM:-}"
+VIRT_CUSTOMIZE_SMP="${VIRT_CUSTOMIZE_SMP:-}"
+PARALLEL_BUILD_TIMEOUT="${PARALLEL_BUILD_TIMEOUT:-0}"
+ENABLE_GOLDEN_IMAGE_ROOT_PASSWORD="${ENABLE_GOLDEN_IMAGE_ROOT_PASSWORD:-true}"
+GOLDEN_BASE_IMAGE_TAR_URL=${GOLDEN_BASE_IMAGE_TAR_URL:-}
+GOLDEN_BASE_IMAGE_FETCH_USER=${GOLDEN_BASE_IMAGE_FETCH_USER:-}
+GOLDEN_BASE_IMAGE_FETCH_PASSWORD=${GOLDEN_BASE_IMAGE_FETCH_PASSWORD:-}
+
+WORK=$(dirname $(dirname $LIBDIR))
+RPM_BUILDER=$(find $WORK -maxdepth 2 -type d -name rpmbuilder)
+
+WORKTMP=$WORK/tmp
+WORKLOGS=$WORKTMP/logs
+DURATION_LOG=$WORKLOGS/durations.log
+MANIFEST_PATH=$WORK/.repo/manifests
+BUILD_CONFIG_INI=$WORK/.repo/manifests/build_config.ini
+GOLDEN_IMAGE_NAME=guest-image.img
+TMP_GOLDEN_IMAGE=$WORKTMP/$GOLDEN_IMAGE_NAME
+
+WORKRESULTS=$WORK/results
+REPO_FILES=$WORKRESULTS/repo_files
+REPO_DIR=$WORKRESULTS/repo
+SRC_REPO_DIR=$WORKRESULTS/src_repo
+RPMLISTS=$WORKRESULTS/rpmlists
+CHECKSUM_DIR=$WORKRESULTS/bin_checksum
+RESULT_IMAGES_DIR=$WORKRESULTS/images
+RPM_BUILDER_SETTINGS=$WORKTMP/mocksettings/mock.cfg
+
+function _read_build_config()
+{
+ local config_ini=$BUILD_CONFIG_INI
+ if [[ -f "$1" ]] && [[ $1 == *.ini ]]; then
+ config_ini=$1
+ shift
+ fi
+ PYTHONPATH=$LIBDIR $LIBDIR/tools/script/read_build_config.py $config_ini $@
+}
+
+function _read_manifest_vars()
+{
+ PRODUCT_RELEASE_BUILD_ID="${BUILD_NUMBER:?0}"
+ PRODUCT_RELEASE_LABEL="$(_read_build_config DEFAULT product_release_label)"
+}
+
+function _initialize_work_dirs()
+{
+ rm -rf $WORKRESULTS
+ mkdir -p $WORKRESULTS $REPO_FILES $REPO_DIR $RPMLISTS $CHECKSUM_DIR
+ # dont clear tmp, can be used for caching
+ mkdir -p $WORKTMP
+ rm -rf $WORKLOGS
+ mkdir -p $WORKLOGS
+}
+
+function _log()
+{
+ echo "$(date) $@"
+}
+
+function _info()
+{
+ _log INFO: $@
+}
+
+function _header()
+{
+ _info "##################################################################"
+ _info "# $@"
+ _info "##################################################################"
+}
+
+
+function _divider()
+{
+ _info "------------------------------------------------------------------"
+}
+
+
+function _step()
+{
+ _header "STEP START: $@"
+}
+
+
+function _abort()
+{
+ _header "ERROR: $@"
+ exit 1
+}
+
+
+function _success()
+{
+ _header "STEP OK: $@"
+}
+
+
+function _run_cmd()
+{
+ _info "[cmd-start]: $@"
+ stamp_start=$(date +%s)
+ time $@ 2>&1 || _abort "Command failed: $@"
+ stamp_end=$(date +%s)
+ echo "$((stamp_end - stamp_start)) $@" >> $DURATION_LOG.unsorted
+ sort -nr $DURATION_LOG.unsorted > $DURATION_LOG
+ _log "[cmd-end]: $@"
+}
+
+
+function _run_cmd_as_step()
+{
+ if [ $# -eq 1 -a -f $1 ]; then
+ step="$(basename $1)"
+ else
+ step="$@"
+ fi
+ _step $step
+ _run_cmd $@
+ _success $step
+}
+
+
+function _add_rpms_to_repo()
+{
+ local repo_dir=$1
+ local rpm_dir=$2
+ mkdir -p $repo_dir
+ cp -f $(repomanage --keep=1 --new $rpm_dir) $repo_dir/
+}
+
+function _create_localrepo()
+{
+ pushd $REPO_DIR
+ _run_cmd createrepo --workers=8 --update .
+ popd
+ pushd $SRC_REPO_DIR
+ _run_cmd createrepo --workers=8 --update .
+ popd
+}
+
+function _add_rpms_to_repos_from_workdir()
+{
+ _add_rpms_to_repo $REPO_DIR $1/buildrepository/mock/rpm
+ _add_rpms_to_repo $SRC_REPO_DIR $1/buildrepository/mock/srpm
+ #find $1/ -name '*.tar.gz' | xargs rm -f
+ true
+}
+
+function _publish_results()
+{
+ local from=$1
+ local to=$2
+ mkdir -p $(dirname $to)
+ mv -f $from $to
+}
+
+function _publish_image()
+{
+ _publish_results $1 $2
+ _create_checksum $2
+}
+
+function _create_checksum()
+{
+ _create_md5_checksum $1
+ _create_sha256_checksum $1
+}
+
+function _create_sha256_checksum()
+{
+ pushd $(dirname $1)
+ time sha256sum $(basename $1) > $(basename $1).sha256
+ popd
+}
+
+function _create_md5_checksum()
+{
+ pushd $(dirname $1)
+ time md5sum $(basename $1) > $(basename $1).md5
+ popd
+}
+
+function _is_true()
+{
+ # e.g. for Jenkins boolean parameters
+ [ "$1" == "true" ]
+}
+
+function _join_array()
+{
+ local IFS="$1"
+ shift
+ echo "$*"
+}
+
+function _get_package_list()
+{
+ PYTHONPATH=$LIBDIR $LIBDIR/tools/script/read_package_config.py $@
+}
+
+function _load_docker_image()
+{
+ local docker_image=$1
+ local docker_image_url="$(_read_build_config DEFAULT docker_images)/${docker_image}.tar"
+ if docker inspect ${docker_image} &> /dev/null; then
+ echo "Using already built ${docker_image} image"
+ else
+ echo "Loading ${docker_image} image"
+ curl -L $docker_image_url | docker load
+ fi
+}
+
--- /dev/null
+[formatters]
+keys: detailed,simple,unadorned,state
+
+[handlers]
+keys: simple_console,detailed_console,unadorned_console,simple_console_warnings_only
+
+[loggers]
+keys: root,build,state,mockbuild
+
+[formatter_state]
+format: %(asctime)s - %(message)s
+
+[formatter_unadorned]
+format: %(message)s
+
+[formatter_simple]
+format: %(levelname)s: %(message)s
+
+;useful for debugging:
+[formatter_detailed]
+format: %(levelname)s %(filename)s:%(lineno)d: %(message)s
+
+[handler_unadorned_console]
+class: StreamHandler
+args: []
+formatter: unadorned
+level: INFO
+
+[handler_simple_console]
+class: StreamHandler
+args: []
+formatter: simple
+level: INFO
+
+[handler_simple_console_warnings_only]
+class: StreamHandler
+args: []
+formatter: simple
+level: WARNING
+
+[handler_detailed_console]
+class: StreamHandler
+args: []
+formatter: detailed
+level: WARNING
+
+; usually dont want to set a level for loggers
+; this way all handlers get all messages, and messages can be filtered
+; at the handler level
+;
+; all these loggers default to a console output handler
+;
+[logger_root]
+level: NOTSET
+handlers: simple_console
+
+; mockbuild logger normally has no output
+; catches stuff like mockbuild.trace_decorator and mockbuild.util
+; dont normally want to propagate to root logger, either
+[logger_mockbuild]
+level: NOTSET
+handlers:
+qualname: mockbuild
+propagate: 1
+
+[logger_state]
+level: NOTSET
+; unadorned_console only outputs INFO or above
+handlers: unadorned_console
+qualname: mockbuild.Root.state
+propagate: 0
+
+[logger_build]
+level: NOTSET
+handlers: simple_console_warnings_only
+qualname: mockbuild.Root.build
+propagate: 0
+
+; the following is a list mock logger qualnames used within the code:
+;
+; qualname: mockbuild.util
+; qualname: mockbuild.uid
+; qualname: mockbuild.trace_decorator
+
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Root name to be used for chroot and caching, must differ between products
+config_opts['root'] = 'akrainolite'
+
+config_opts['target_arch'] = 'x86_64'
+config_opts['legal_host_arches'] = ('x86_64',)
+config_opts['dist'] = 'el7' # only useful for --resultdir variable subst
+config_opts['chroot_setup_cmd'] = 'install createrepo yum-utils bison byacc cscope ctags cvs diffstat doxygen flex gcc gcc-c++ gcc-gfortran gettext git indent intltool libtool patch patchutils rcs redhat-rpm-config rpm-build subversion swig systemtap sudo'
+config_opts['plugin_conf']['yum_cache_enable'] = False
+config_opts['plugin_conf']['ccache_enable'] = False
+config_opts['plugin_conf']['ccache_opts']['max_cache_size'] = '1G'
+config_opts['plugin_conf']['ccache_opts']['dir'] = "/dev/shm/ccache.lcc-epel7"
+config_opts['rpmbuild_networking'] = True
+config_opts['cleanup_on_success'] = True
+config_opts['cleanup_on_failure'] = False
+config_opts['exclude_from_homedir_cleanup'] = ('build/SOURCES', '.bash_history', '.bashrc', 'build/RPMS', )
+#ADDITIONAL_CONFIG_OPTS#
+
+# Common RPM directive values
+config_opts['macros']['%packager'] = #RPM_PACKAGER#
+config_opts['macros']['%dist'] = ".el7.centos%{_platform_product}"
+config_opts['macros']['%_platform_product'] = #RPM_RELEASE_ID#
+config_opts['macros']['%_platform_dist'] = ".el7.centos"
+config_opts['macros']['%_platform_vendor'] = #RPM_VENDOR#
+config_opts['macros']['%_platform_license'] = #RPM_LICENSE#
+config_opts['macros']['%_platform_licence'] = "%{_platform_license}"
+
+# Product specific macros
+config_opts['macros']['%_playbooks_path'] = "/opt/openstack-ansible/playbooks"
+config_opts['macros']['%_inventory_path'] = "/opt/openstack-ansible/inventory"
+config_opts['macros']['%_roles_path'] = "/etc/ansible/roles"
+config_opts['macros']['%_installation_root_path'] = "/etc/lcm/playbooks/installation"
+config_opts['macros']['%_bootstrapping_path'] = "%{_installation_root_path}/bootstrapping"
+config_opts['macros']['%_provisioning_path'] = "%{_installation_root_path}/provisioning"
+config_opts['macros']['%_postconfig_path'] = "%{_installation_root_path}/postconfig"
+config_opts['macros']['%_finalize_path'] = "%{_installation_root_path}/finalize"
+config_opts['macros']['%_ansible_filter_plugins_path'] = "%{_roles_path}/plugins/filter"
+config_opts['macros']['%_caas_path'] = "/var/lib/caas"
+config_opts['macros']['%_caas_container_tar_path'] = "%{_caas_path}/images"
+config_opts['macros']['%_caas_manifest_path'] = "%{_caas_path}/manifests"
+config_opts['macros']['%_caas_chart_path'] = "${_caas_path}/chart/"
+config_opts['macros']['%_platform_bin_path'] = "/usr/local/bin"
+config_opts['macros']['%_platform_lib_path'] = "/usr/local/lib"
+config_opts['macros']['%_platform_etc_path'] = "/etc"
+config_opts['macros']['%_platform_share_path'] = "/share"
+config_opts['macros']['%_platform_man_path'] = "%{_platform_share_path}/man"
+config_opts['macros']['%_platform_doc_path'] = "%{_platform_share_path}/doc"
+config_opts['macros']['%_platform_var_path'] = "/var"
+config_opts['macros']['%_platform_python'] = "/python2.7"
+config_opts['macros']['%_platform_python_site_packages_path'] = "%{_platform_lib_path}%{_platform_python}/site-packages"
+config_opts['macros']['%_platform_ocf_resource_path'] = "/usr/lib/ocf/resource.d"
+config_opts['macros']['%_python_site_packages_path'] = "/usr/lib/python2.7/site-packages"
+config_opts['macros']['%_secrets_path'] = "/etc/required-secrets"
+
+# Compilation
+#config_opts['macros']['%_smp_mflags'] = "-j6"
+#config_opts['macros']['%_smp_ncpus_max'] = 0
+
+# Yum configuration
+config_opts['yum.conf'] = """
+[main]
+cachedir=/var/cache/yum
+keepcache=1
+debuglevel=2
+reposdir=/dev/null
+logfile=/var/log/yum.log
+retries=20
+obsoletes=1
+gpgcheck=0
+assumeyes=1
+syslog_ident=mock
+syslog_device=
+
+#REPOSITORIES#
+
+"""
--- /dev/null
+# mock defaults
+# vim:tw=0:ts=4:sw=4:et:
+#
+# This config file is for site-specific default values that apply across all
+# configurations. Options specified in this config file can be overridden in
+# the individual mock config files.
+#
+# The site-defaults.cfg delivered by default has NO options set. Only set
+# options here if you want to override the defaults.
+#
+# Entries in this file follow the same format as other mock config files.
+# config_opts['foo'] = bar
+
+#############################################################################
+#
+# Things that we recommend you set in site-defaults.cfg:
+#
+# config_opts['basedir'] = '/var/lib/mock/'
+# config_opts['cache_topdir'] = '/var/cache/mock'
+# Note: the path pointed to by basedir and cache_topdir must be owned
+# by group 'mock' and must have mode: g+rws
+# config_opts['rpmbuild_timeout'] = 0
+# config_opts['use_host_resolv'] = True
+
+# You can configure log format to pull from logging.ini formats of these names:
+# config_opts['build_log_fmt_name'] = "unadorned"
+# config_opts['root_log_fmt_name'] = "detailed"
+# config_opts['state_log_fmt_name'] = "state"
+#
+# mock will normally set up a minimal chroot /dev.
+# If you want to use a pre-configured /dev, disable this and use the bind-mount
+# plugin to mount your special /dev
+# config_opts['internal_dev_setup'] = True
+#
+# internal_setarch defaults to 'True' if the python 'ctypes' package is
+# available. It is in the python std lib on >= python 2.5. On older versions,
+# it is available as an addon. On systems w/o ctypes, it will default to 'False'
+# config_opts['internal_setarch'] = False
+#
+# the cleanup_on_* options allow you to automatically clean and remove the
+# mock build directory, but only take effect if --resultdir is used.
+# config_opts provides fine-grained control. cmdline only has big hammer
+#
+# config_opts['cleanup_on_success'] = 1
+# config_opts['cleanup_on_failure'] = 1
+
+# if you want mock to automatically run createrepo on the rpms in your
+# resultdir.
+# config_opts['createrepo_on_rpms'] = False
+# config_opts['createrepo_command'] = '/usr/bin/createrepo -d -q -x *.src.rpm'
+
+# if you want mock to backup the contents of a result dir before clean
+# config_opts['backup_on_clean'] = False
+# config_opts('backup_base_dir'] = config_opts['basedir'] + "backup"
+
+
+#############################################################################
+#
+# plugin related. Below are the defaults. Change to suit your site
+# policy. site-defaults.cfg is a good place to do this.
+#
+# NOTE: Some of the caching options can theoretically affect build
+# reproducability. Change with care.
+#
+# config_opts['plugin_conf']['package_state_enable'] = True
+# config_opts['plugin_conf']['ccache_enable'] = True
+# config_opts['plugin_conf']['ccache_opts']['max_cache_size'] = '4G'
+# config_opts['plugin_conf']['ccache_opts']['compress'] = None
+# config_opts['plugin_conf']['ccache_opts']['dir'] = "%(cache_topdir)s/%(root)s/ccache/"
+# config_opts['plugin_conf']['yum_cache_enable'] = True
+# config_opts['plugin_conf']['yum_cache_opts']['max_age_days'] = 30
+# config_opts['plugin_conf']['yum_cache_opts']['dir'] = "%(cache_topdir)s/%(root)s/yum_cache/"
+# config_opts['plugin_conf']['root_cache_enable'] = True
+# config_opts['plugin_conf']['root_cache_opts']['max_age_days'] = 15
+# config_opts['plugin_conf']['root_cache_opts']['dir'] = "%(cache_topdir)s/%(root)s/root_cache/"
+# config_opts['plugin_conf']['root_cache_opts']['compress_program'] = "pigz"
+# config_opts['plugin_conf']['root_cache_opts']['extension'] = ".gz"
+# config_opts['plugin_conf']['root_cache_opts']['exclude_dirs'] = ["./proc", "./sys", "./dev",
+# "./tmp/ccache", "./var/cache/yum" ]
+#
+# bind mount plugin is enabled by default but has no configured directories to
+# mount
+# config_opts['plugin_conf']['bind_mount_enable'] = True
+# config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/host/path', '/bind/mount/path/in/chroot/' ))
+#
+# config_opts['plugin_conf']['tmpfs_enable'] = False
+# config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 1024
+# config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = '512m'
+# config_opts['plugin_conf']['tmpfs_opts']['mode'] = '0755'
+# config_opts['plugin_conf']['chroot_scan_enable'] = False
+# config_opts['plugin_conf']['chroot_scan_opts'] = [ "core(\.\d+)?", "\.log$",]
+
+#############################################################################
+#
+# environment for chroot
+#
+# config_opts['environment']['TERM'] = 'vt100'
+# config_opts['environment']['SHELL'] = '/bin/bash'
+# config_opts['environment']['HOME'] = '/builddir'
+# config_opts['environment']['HOSTNAME'] = 'mock'
+# config_opts['environment']['PATH'] = '/usr/bin:/bin:/usr/sbin:/sbin'
+# config_opts['environment']['PROMPT_COMMAND'] = 'echo -n "<mock-chroot>"'
+# config_opts['environment']['LANG'] = os.environ.setdefault('LANG', 'en_US.UTF-8')
+# config_opts['environment']['TZ'] = os.environ.setdefault('TZ', 'EST5EDT')
+
+#############################################################################
+#
+# Things that you can change, but we dont recommend it:
+# config_opts['chroothome'] = '/builddir'
+# config_opts['clean'] = True
+
+#############################################################################
+#
+# Things that must be adjusted if SCM integration is used:
+#
+# config_opts['scm'] = True
+# config_opts['scm_opts']['method'] = 'git'
+# config_opts['scm_opts']['cvs_get'] = 'cvs -d /srv/cvs co SCM_BRN SCM_PKG'
+# config_opts['scm_opts']['git_get'] = 'git clone SCM_BRN git://localhost/SCM_PKG.git SCM_PKG'
+# config_opts['scm_opts']['svn_get'] = 'svn co file:///srv/svn/SCM_PKG/SCM_BRN SCM_PKG'
+# config_opts['scm_opts']['spec'] = 'SCM_PKG.spec'
+# config_opts['scm_opts']['ext_src_dir'] = '/dev/null'
+# config_opts['scm_opts']['write_tar'] = True
+# config_opts['scm_opts']['git_timestamps'] = True
+
+# These options are also recognized but usually defined in cmd line
+# with --scm-option package=<pkg> --scm-option branch=<branch>
+# config_opts['scm_opts']['package'] = 'mypkg'
+# config_opts['scm_opts']['branch'] = 'master'
+
+#############################################################################
+#
+# Things that are best suited for individual chroot config files:
+#
+# MUST SET (in individual chroot cfg file):
+# config_opts['root'] = 'name-of-yum-build-dir'
+# config_opts['target_arch'] = 'i386'
+# config_opts['yum.conf'] = ''
+# config_opts['yum_common_opts'] = []
+#
+# CAN SET, defaults usually work ok:
+# config_opts['chroot_setup_cmd'] = 'install buildsys-build'
+# config_opts['log_config_file'] = 'logging.ini'
+# config_opts['more_buildreqs']['srpm_name-version-release'] = 'dependencies'
+# config_opts['macros']['%Add_your_macro_name_here'] = "add macro value here"
+# config_opts['files']['path/name/no/leading/slash'] = "put file contents here."
+# config_opts['chrootuid'] = os.getuid()
+
+# If you change chrootgid, you must also change "mock" to the correct group
+# name in this line of the mock PAM config:
+# auth sufficient pam_succeed_if.so user ingroup mock use_uid quiet
+# config_opts['chrootgid'] = grp.getgrnam("mock")[2]
+
+# config_opts['useradd'] = '/usr/sbin/useradd -m -u %(uid)s -g %(gid)s -d %(home)s -n %(user)s' # Fedora/RedHat
+#
+# Security related
+# config_opts['no_root_shells'] = False
+#
+# Proxy settings (https_proxy, ftp_proxy, and no_proxy can also be set)
+# config_opts['http_proxy'] = 'http://localhost:3128'
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+NEXUS_URL=$1
+NEXUS_REPOSITORY=$2
+NEXUS_REPOSITORY_BASE_PATH=$3
+shift;shift;shift
+NEXUS_REPOSITORY_SEARCH_PATTERNS=$@
+
+_abort() {
+ echo "ERROR: $@"
+ exit 1
+}
+
+_search_group() {
+ local params=""
+ [ -n "$1" ] && params+="&group=$1"
+ [ -n "${2:-}" ] && params+="&name=$2"
+ curl "${NEXUS_URL}/service/rest/v1/search?repository=${NEXUS_REPOSITORY}${params}"
+}
+
+for pat in $NEXUS_REPOSITORY_SEARCH_PATTERNS; do
+ search_group="/$NEXUS_REPOSITORY_BASE_PATH/$(echo $pat | cut -d':' -f1)"
+ search_name=""
+ if echo $pat | grep ':'; then
+ search_name="$(echo $pat | cut -d':' -f2)"
+ fi
+ resp=$(_search_group $search_group $search_name)
+ if [ "$(echo $resp | jq -r '.continuationToken')" != "null" ]; then
+ _abort "Pagination not implemented"
+ fi
+ for url in $(echo $resp | jq -r '.items[].assets[].downloadUrl'); do
+ to=${url#$NEXUS_URL/repository/$NEXUS_REPOSITORY/$NEXUS_REPOSITORY_BASE_PATH/}
+ mkdir -p $(dirname $to)
+ echo "Fetch $url"
+ curl $url > $to
+ done
+done
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+
+scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
+source $scriptdir/lib.sh
+_read_manifest_vars
+
+pushd $MANIFEST_PATH
+sed -i -e "s/^Version: .*/Version: $PRODUCT_RELEASE_BUILD_ID/" rpmbuild.spec
+cat > product-release <<EOF
+release=$PRODUCT_RELEASE_LABEL
+build=$PRODUCT_RELEASE_BUILD_ID
+EOF
+popd
--- /dev/null
+[MASTER]
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code
+extension-pkg-whitelist=
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=CVS
+
+# Add files or directories matching the regex patterns to the blacklist. The
+# regex matches against base names, not paths.
+ignore-patterns=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Use multiple processes to speed up Pylint.
+jobs=1
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# Specify a configuration file.
+#rcfile=
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
+confidence=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once).You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use"--disable=all --enable=classes
+# --disable=W"
+disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,
+ backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,
+ raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,
+ file-ignored,suppressed-message,useless-suppression,deprecated-pragma,
+ apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,
+ execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,
+ standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,
+ delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,
+ dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,
+ indexing-exception,raising-string,reload-builtin,oct-method,hex-method,
+ nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,
+ unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,
+ range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,
+ eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,
+ invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,
+ deprecated-str-translate-call,
+ missing-docstring,
+ too-few-public-methods,
+ superfluous-parens,
+ logging-format-interpolation
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=
+
+
+[REPORTS]
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details
+#msg-template=
+
+# Set the output format. Available formats are text, parseable, colorized, json
+# and msvs (visual studio).You can also give a reporter class, eg
+# mypackage.mymodule.MyReporterClass.
+output-format=text
+
+# Tells whether to display a full report or only the messages
+reports=no
+
+# Activate the evaluation score.
+score=yes
+
+
+[REFACTORING]
+
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
+
+
+[LOGGING]
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format
+logging-modules=logging
+
+
+[TYPECHECK]
+
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# This flag controls whether pylint should warn about no-member and similar
+# checks whenever an opaque object is returned when inferring. The inference
+# can return multiple potential results while evaluating a Python object, but
+# some branches might not be evaluated, which results in partial inference. In
+# that case, it might be useful to still emit no-member and other checks for
+# the rest of the inferred objects.
+ignore-on-opaque-inference=yes
+
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis. It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=
+
+# Show a hint with possible names when a member name was not found. The aspect
+# of finding the hint is based on edit distance.
+missing-member-hint=yes
+
+# The minimum edit distance a name should have in order to be considered a
+# similar match for a missing member name.
+missing-member-hint-distance=1
+
+# The total number of similar names that should be taken in consideration when
+# showing a hint for a missing member.
+missing-member-max-choices=1
+
+
+[VARIABLES]
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid to define new builtins when possible.
+additional-builtins=
+
+# Tells whether unused global variables should be treated as a violation.
+allow-global-unused-variables=yes
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,_cb
+
+# A regular expression matching the name of dummy variables (i.e. expectedly
+# not used).
+dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore
+ignored-argument-names=_.*|^ignored_|^unused_
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,future.builtins
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,XXX,TODO
+
+
+[SIMILARITIES]
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=no
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+
+[BASIC]
+
+# Naming hint for argument names
+argument-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Regular expression matching correct argument names
+argument-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Naming hint for attribute names
+attr-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Regular expression matching correct attribute names
+attr-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Bad variable names which should always be refused, separated by a comma
+bad-names=foo,bar,baz,toto,tutu,tata
+
+# Naming hint for class attribute names
+class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
+
+# Regular expression matching correct class attribute names
+class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
+
+# Naming hint for class names
+class-name-hint=[A-Z_][a-zA-Z0-9]+$
+
+# Regular expression matching correct class names
+class-rgx=[A-Z_][a-zA-Z0-9]+$
+
+# Naming hint for constant names
+const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
+
+# Regular expression matching correct constant names
+const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# Naming hint for function names
+function-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Regular expression matching correct function names
+function-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Good variable names which should always be accepted, separated by a comma
+good-names=i,j,k,ex,Run,_
+
+# Include a hint for the correct naming format with invalid-name
+include-naming-hint=no
+
+# Naming hint for inline iteration names
+inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
+
+# Regular expression matching correct inline iteration names
+inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
+
+# Naming hint for method names
+method-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Regular expression matching correct method names
+method-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Naming hint for module names
+module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+
+# Regular expression matching correct module names
+module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=^_
+
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+property-classes=abc.abstractproperty
+
+# Naming hint for variable names
+variable-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Regular expression matching correct variable names
+variable-rgx=(([a-z][a-z0-9_]{0,30})|(_[a-z0-9_]*))$
+
+
+[SPELLING]
+
+# Spelling dictionary name. Available dictionaries: none. To make it working
+# install python-enchant package.
+spelling-dict=
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# A path to a file that contains private dictionary; one word per line.
+spelling-private-dict-file=
+
+# Tells whether to store unknown words to indicated private dictionary in
+# --spelling-private-dict-file option instead of raising a message.
+spelling-store-unknown-words=no
+
+
+[FORMAT]
+
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
+# tab).
+indent-string=' '
+
+# Maximum number of characters on a single line.
+max-line-length=100
+
+# Maximum number of lines in a module
+max-module-lines=1000
+
+# List of optional constructs for which whitespace checking is disabled. `dict-
+# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
+# `trailing-comma` allows a space between comma and closing bracket: (a, ).
+# `empty-line` allows space-only lines.
+no-space-check=trailing-comma,dict-separator
+
+# Allow the body of a class to be on the same line as the declaration if body
+# contains single statement.
+single-line-class-stmt=no
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
+
+
+[CLASSES]
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,__new__,setUp
+
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,_fields,_replace,_source,_make
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=mcs
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method
+max-args=5
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Maximum number of boolean expressions in a if statement
+max-bool-expr=5
+
+# Maximum number of branch for function / method body
+max-branches=12
+
+# Maximum number of locals for function / method body
+max-locals=15
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+# Maximum number of return / yield for function / method body
+max-returns=6
+
+# Maximum number of statements in function / method body
+max-statements=50
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+
+[IMPORTS]
+
+# Allow wildcard imports from modules that define __all__.
+allow-wildcard-with-all=no
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+# Deprecated modules which should not be used, separated by a comma
+deprecated-modules=optparse,tkinter.tix
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled)
+ext-import-graph=
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled)
+import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled)
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "Exception"
+overgeneral-exceptions=Exception
--- /dev/null
+#!/bin/bash
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
+source $scriptdir/lib.sh
+
+_run_repo_cmd()
+{
+ _divider
+ _run_cmd $@
+}
+
+pushd $WORK/.repo/repo
+_run_repo_cmd git rev-parse --short HEAD
+popd
+echo q | _run_repo_cmd repo info
+_run_repo_cmd repo status -o
+echo q | _run_repo_cmd repo forall -p -c git rev-parse HEAD
+
+# Store build manifest with project HEADS to a file
+repo manifest -o $WORKRESULTS/manifest.xml
+repo manifest -r -o $WORKRESULTS/manifest_revisions.xml
--- /dev/null
+requests
+pyyaml
+pytest
+pytest-cov
+pytest-flakes
+pytest-pep8
+mock
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import ConfigParser
+
+from tools.statics import BUILD_CONFIG_PATH
+
+
+class BuildConfigParser(ConfigParser.ConfigParser):
+ def __init__(self, ini_file=BUILD_CONFIG_PATH):
+ ConfigParser.ConfigParser.__init__(self)
+ self.ini_file = ini_file
+ self.optionxform = str
+ self.read(self.ini_file)
+
+ def items(self, section): # pylint: disable=arguments-differ
+ defaults = self.defaults()
+ resultlist = []
+ for item in ConfigParser.ConfigParser.items(self, section):
+ if item[0] not in defaults:
+ resultlist.append(item)
+ return resultlist
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import re
+
+
+def to_json(data):
+ return json.dumps(data, sort_keys=True, indent=4)
+
+
+class CsvConverter(object):
+ def __init__(self, data, preferred_field_order=None, escape_newlines=True):
+ self.data = data
+ self.preferred_field_order = preferred_field_order
+ self.escape_newlines = escape_newlines
+ self.csv_data = None
+ self._convert()
+
+ def __str__(self):
+ return self.convert()
+
+ def convert(self):
+ return self._render(CsvFormatter(self.csv_data))
+
+ def convert_to_ms_excel(self, text_fields=None):
+ """
+ CSV that Microsoft Excel can read well.
+
+ :param text_fields: list of columns to mark as text
+ NOTE: must not be used for fields that can contain comma(,) or
+ semicolon(;) as field will be split from these
+ :return:
+ """
+ return self._render(CsvMSFormatter(self.csv_data, text_fields=text_fields))
+
+ def _convert(self):
+ if not isinstance(self.data, list):
+ raise Exception('Input data given is NOT a list')
+ if not self.data:
+ self.csv_data = []
+ return
+ if not isinstance(self.data[0], dict):
+ raise Exception('First data element is NOT a dict')
+ headers = []
+ possible_fields = list(set([key for i in self.data for key in i.keys()]))
+ if self.preferred_field_order is not None:
+ for preferred_field in self.preferred_field_order:
+ if preferred_field in possible_fields:
+ headers.append(preferred_field)
+ possible_fields.remove(preferred_field)
+ headers += sorted(possible_fields)
+ self.csv_data = [headers]
+ for obj in self.data:
+ row_data = []
+ for header in headers:
+ field = obj.get(header)
+ if isinstance(field, (list, dict)):
+ x = json.dumps(field, sort_keys=True)
+ elif isinstance(field, unicode):
+ x = field.encode('utf-8')
+ else:
+ x = str(field)
+ row_data.append(x)
+ self.csv_data.append(row_data)
+
+ def _render(self, formatter):
+ return formatter.format(self.escape_newlines)
+
+
+class CsvFormatter(object):
+ def __init__(self, csv_data):
+ self.csv_data = csv_data
+
+ def format(self, escape_newlines=True):
+ f_file = []
+ for record in self.csv_data:
+ f_record = []
+ for field in record:
+ f_field = self._field_formatter(field, escape_newlines)
+ f_record.append(f_field)
+ f_file.append(','.join(self._record_formatter(f_record)))
+ return '\r\n'.join(self._file_formatter(f_file))
+
+ @staticmethod
+ def _file_formatter(_file):
+ return _file
+
+ @staticmethod
+ def _record_formatter(record):
+ return ['"{}"'.format(i) for i in record]
+
+ @staticmethod
+ def _field_formatter(field, escape_newlines):
+ out = field.replace('"', '""')
+ if escape_newlines:
+ out = out.replace('\n', '\\n')
+ return out
+
+
+class CsvMSFormatter(CsvFormatter):
+ max_cell_size = 32000
+
+ def __init__(self, csv_data, text_fields=None):
+ super(CsvMSFormatter, self).__init__(csv_data)
+ self.text_fields = text_fields
+
+ def _file_formatter(self, _file):
+ return ['sep=,'] + super(CsvMSFormatter, self)._file_formatter(_file)
+
+ def _record_formatter(self, record):
+ record = super(CsvMSFormatter, self)._record_formatter(record)
+ if self.text_fields:
+ formatted_record = []
+ for index, field in enumerate(record):
+ heading = self.csv_data[0][index]
+ if heading in self.text_fields:
+ formatted_field = '=' + field
+ else:
+ formatted_field = field
+ formatted_record.append(formatted_field)
+ record = formatted_record
+ return record
+
+ def _field_formatter(self, field, escape_newlines):
+ field = super(CsvMSFormatter, self)._field_formatter(field, escape_newlines)
+ if len(field) > self.max_cell_size:
+ field = field[:self.max_cell_size / 2] + "..." + field[-self.max_cell_size / 2:]
+ if not re.match(r'^-\d+$', field) and re.match(r'^-.*$', field):
+ return r'\{}'.format(field)
+ return field
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=invalid-name
+from collections import OrderedDict
+
+import pytest
+
+from tools.convert import to_json, CsvConverter
+
+
+class TestJson(object):
+ @staticmethod
+ def test_json():
+ _input = dict(b=1, a=2)
+ expected = '\n'.join(['{',
+ ' "a": 2, ',
+ ' "b": 1',
+ '}'])
+ assert to_json(_input) == expected
+
+
+class TestCsv(object):
+ @staticmethod
+ @pytest.mark.parametrize('_input, expected', [
+ ([], ''),
+ ([{}], '\r\n'),
+ ([dict(a=1)], '\r\n'.join(['"a"',
+ '"1"'])),
+ ([dict(a=1), dict(a=2)], '\r\n'.join(['"a"',
+ '"1"',
+ '"2"'])),
+ ([dict(a=1, b=2), dict(a=3, b=4)], '\r\n'.join(['"a","b"',
+ '"1","2"',
+ '"3","4"'])),
+ ([dict(x=OrderedDict({'b': 1, 'a': 2})),
+ dict(x={'a': 2, 'b': 1})], '\r\n'.join(['"x"',
+ '"{""a"": 2, ""b"": 1}"',
+ '"{""a"": 2, ""b"": 1}"']))
+
+ ])
+ def test_csv(_input, expected):
+ assert str(CsvConverter(_input)) == expected
+
+ @staticmethod
+ def test_str():
+ _input = [dict(a=1)]
+ assert str(CsvConverter(_input)) == CsvConverter(_input).convert()
+
+ @staticmethod
+ @pytest.mark.parametrize('_input, exception_re', [
+ ('boom', r'NOT a list'),
+ ([['boom']], r'NOT a dict')
+ ])
+ def test_to_csv_fail(_input, exception_re):
+ with pytest.raises(Exception, match=exception_re):
+ str(CsvConverter(_input))
+
+ @staticmethod
+ def test_newlines_are_escaped():
+ """
+ ...in order to get properly formatted file when written
+ """
+ _input = [dict(a='1"2')]
+ expected = '\r\n'.join(['"a"',
+ '"1""2"'])
+ assert CsvConverter(_input).convert() == expected
+
+ @staticmethod
+ def test_missing_fields_are_filled_with_none():
+ _input = [dict(a='1"2')]
+ expected = '\r\n'.join(['"a"',
+ '"1""2"'])
+ assert CsvConverter(_input).convert() == expected
+
+ @staticmethod
+ def test_double_quote_is_escaped_with_double_quote():
+ """
+ RFC 4180
+ """
+ _input = [dict(a='1"2')]
+ expected = '\r\n'.join(['"a"',
+ '"1""2"'])
+ assert CsvConverter(_input).convert() == expected
+
+ @staticmethod
+ @pytest.mark.parametrize('_input, expected', [
+ ([dict(a=1, b=2)], '\r\n'.join(['"b","a"', '"2","1"'])),
+ ([dict(a=1, c=2)], '\r\n'.join(['"a","c"', '"1","2"']))
+ ])
+ def test_csv_preferred_order(_input, expected):
+ assert CsvConverter(_input, preferred_field_order=['b', 'a']).convert() == expected
+
+ @staticmethod
+ @pytest.mark.parametrize('_input, expected', [
+ ([dict(a='Copyright \xc2\xa9 2014')], '\r\n'.join(['"a"',
+ '"Copyright \xc2\xa9 2014"'])),
+ ([dict(a=u'Copyright \xa9 2014')], '\r\n'.join(['"a"',
+ '"Copyright \xc2\xa9 2014"'])),
+ ])
+ def test_unicode_input(_input, expected):
+ assert str(CsvConverter(_input)) == expected
+
+
+class TestCsvMSExcel(object):
+ @staticmethod
+ def test_ms_excel_format():
+ """
+ MS Excel treats CSV files with 'sep=,' as the first line to get automatically columnized
+ """
+ _input = [dict(a=1, b=2)]
+ expected = '\r\n'.join(['sep=,',
+ '"a","b"',
+ '"1","2"'])
+ assert CsvConverter(_input).convert_to_ms_excel() == expected
+
+ @staticmethod
+ def test_text_fields():
+ """
+ MS Excel CSV fields prefixed with '=' will be treated as equations to string.
+ This makes it possible to e.g. to have all RPM version strings treated equally instead
+ of making some of them treated as generic and some of them as integers.
+ """
+ _input = [dict(a=1, b=2)]
+ expected = '\r\n'.join(['sep=,',
+ '"a",="b"',
+ '"1",="2"'])
+ assert CsvConverter(_input).convert_to_ms_excel(text_fields=['b']) == expected
+
+ @staticmethod
+ def test_field_with_beginning_minus_is_prefixed():
+ """
+ MS Excel CSV fields beginning with '-' are treated as an equation even they would be
+ essentially just strings. Make sure to escape the beginning signs with something in order
+ not to get field braking equation.
+ """
+ _input = [dict(a=-1), dict(a="-2a")]
+ expected = '\r\n'.join(['sep=,',
+ '"a"',
+ '"-1"',
+ r'"\-2a"'])
+ assert CsvConverter(_input).convert_to_ms_excel() == expected
+
+ @staticmethod
+ def test_too_big_cell_is_truncated():
+ """
+ MS Excel has ~32k character limit per cell. BOM information can easily exceed this.
+ """
+ _input = [dict(a='1' * 32000), dict(a='2' * 32001)]
+ expected = '\r\n'.join(['sep=,',
+ '"a"',
+ '"{}"'.format('1' * 32000),
+ '"{}...{}"'.format('2' * 16000, '2' * 16000)])
+ assert CsvConverter(_input).convert_to_ms_excel() == expected
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import logging
+import shlex
+import subprocess
+
+
+class ExecutionError(Exception):
+ def __init__(self, msg, result):
+ super(ExecutionError, self).__init__(msg)
+ self.result = result
+
+
+class Result(object):
+ def __init__(self, status, stdout, stderr):
+ self.status = status
+ self.stdout = stdout
+ self.stderr = stderr
+
+ @property
+ def str_status(self):
+ return 'Status:"{}"'.format(self.status)
+
+ @property
+ def str_stdout(self):
+ return 'Stdout:"{}"'.format(self.stdout)
+
+ @property
+ def str_stderr(self):
+ return 'Stderr:"{}"'.format(self.stderr)
+
+ def __str__(self):
+ return '\n'.join([self.str_status, self.str_stdout, self.str_stderr])
+
+
+def run(*args, **kwargs):
+ return Executor().run(*args, **kwargs)
+
+
+class Executor(object):
+ def __init__(self, shell=False, chdir=None):
+ self.shell = shell
+ self.chdir = chdir
+
+ def run(self, cmd, raise_on_error=True, raise_on_stderr=True, retries=0):
+ result = self._run(cmd, retries)
+ if raise_on_error and result.status != 0:
+ raise ExecutionError('Command "{}" execution status NOT zero: {}'.format(
+ cmd, str(result)), result.status)
+ if raise_on_stderr and result.stderr:
+ raise ExecutionError('Command "{}" execution stderr not empty: {}'.format(
+ cmd, str(result)), result.status)
+ return result
+
+ @staticmethod
+ def _log_result(result):
+ logging.debug('Result: %s', str(result))
+
+ def _run(self, cmd, retries):
+ logstr = 'Executing command: "{}"'.format(cmd)
+ cwd = os.getcwd()
+ if self.chdir is not None:
+ os.chdir(self.chdir)
+ logstr += ' in dir {}'.format(self.chdir)
+ logging.debug(logstr)
+ result = self._run_with_retries(cmd, retries)
+ if self.chdir is not None:
+ os.chdir(cwd)
+ return result
+
+ def _run_with_retries(self, cmd, retries):
+ result = self._execute(cmd)
+ while result.status != 0 and retries > 0:
+ logging.debug('Retrying, retries left: %s', retries)
+ retries -= 1
+ result = self._execute(cmd)
+ self._log_result(result)
+ if result.status == 0:
+ break
+ return result
+
+ def _execute(self, cmd):
+ if not self.shell:
+ if isinstance(cmd, list):
+ cmd_args = cmd[:]
+ else:
+ cmd_args = shlex.split(cmd)
+ else:
+ cmd_args = cmd
+ process = subprocess.Popen(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ shell=self.shell)
+ stdout, stderr = process.communicate()
+ result = Result(process.returncode, stdout, stderr)
+ return result
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+import mock
+
+from tools.executor import Executor, ExecutionError
+from tools.executor import Result
+
+
+@mock.patch.object(Executor, '_execute')
+def test_success(mocky):
+ mocky.side_effect = [Result(0, 'fake-stdout', '')]
+ Executor().run('test-cmd')
+ mocky.assert_called_once_with('test-cmd')
+
+
+@mock.patch.object(Executor, '_execute')
+def test_fail_returncode(mocky):
+ mocky.side_effect = [Result(1, 'fake-stdout', '')]
+ with pytest.raises(ExecutionError, match=r'execution status NOT zero'):
+ Executor().run('test-cmd')
+ mocky.assert_called_once_with('test-cmd')
+
+
+@mock.patch.object(Executor, '_execute')
+def test_fail_stderr(mocky):
+ mocky.side_effect = [Result(0, 'fake-stdout', 'fake-stderr')]
+ with pytest.raises(ExecutionError, match=r'stderr not empty'):
+ Executor().run('test-cmd')
+ mocky.assert_called_once_with('test-cmd')
+
+
+@mock.patch.object(Executor, '_execute')
+def test_success_ignore_returncode(mocky):
+ mocky.side_effect = [Result(1, 'fake-stdout', '')]
+ Executor().run('test-cmd', raise_on_error=False)
+ mocky.assert_called_once_with('test-cmd')
+
+
+@mock.patch.object(Executor, '_execute')
+def test_success_ignore_stderr(mocky):
+ mocky.side_effect = [Result(0, 'fake-stdout', 'fake-stderr')]
+ Executor().run('test-cmd', raise_on_stderr=False)
+ mocky.assert_called_once_with('test-cmd')
+
+
+@mock.patch.object(Executor, '_execute')
+def test_no_retry_on_success(mocky):
+ mocky.side_effect = [Result(0, 'fake-stdout', '')]
+ Executor().run('test-cmd', retries=1)
+ mocky.assert_called_once_with('test-cmd')
+
+
+@mock.patch.object(Executor, '_execute')
+def test_retry_on_fail(mocky):
+ mocky.side_effect = [Result(1, 'fake-stdout', ''),
+ Result(0, 'fake-stdout', '')]
+ Executor().run('test-cmd', retries=1)
+ expected = [mock.call('test-cmd'),
+ mock.call('test-cmd')]
+ assert mocky.mock_calls == expected
+
+
+@mock.patch.object(Executor, '_execute')
+def test_error_on_retry_exceeded(mocky):
+ mocky.side_effect = [Result(1, 'fake-stdout', ''),
+ Result(1, 'fake-stdout', '')]
+ with pytest.raises(ExecutionError):
+ Executor().run('test-cmd', retries=1)
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+
+from tools.convert import to_json
+
+
+def write_to(file_path, data):
+ with open(file_path, 'w') as f:
+ f.write(data)
+ logging.debug('Wrote: {}'.format(file_path))
+
+
+def read_from(file_path):
+ with open(file_path, 'r') as f:
+ return f.read()
+
+
+def write_json(file_path, data):
+ write_to(file_path, to_json(data))
+
+
+def read_json(file_path):
+ return json.loads(read_from(file_path))
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import sys
+
+
+def set_logging(debug=True, timestamps=False):
+ _format = '{}%(levelname)s:%(message)s'
+ if timestamps:
+ _format = _format.format('%(asctime)-15s:')
+ else:
+ _format = _format.format('')
+ if debug:
+ level = logging.DEBUG
+ else:
+ level = logging.INFO
+ logging.basicConfig(stream=sys.stdout, level=level, format=_format)
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import yaml
+
+from tools.statics import PACKAGES_CONFIG_PATH
+
+
+class PackageConfigReader(object):
+ def __init__(self, config_file=PACKAGES_CONFIG_PATH):
+ self.config_file = config_file
+ self.config = None
+ self._read_file()
+
+ def _read_file(self):
+ with open(self.config_file, 'r') as fp:
+ self.config = yaml.load(fp)
+
+ def get(self, package_operation_type):
+ return self._get_packages_by_type(package_operation_type)
+
+ def _get_packages_by_type(self, package_operation_type):
+ if self.config is None:
+ return []
+ if package_operation_type == 'install':
+ return self._get_installed_packages()
+ elif package_operation_type == 'uninstall':
+ return self._get_uninstalled_packages()
+ raise Exception('Never here')
+
+ def _get_uninstalled_packages(self):
+ return sorted([p for p in self.config if not self._is_install_package(p)])
+
+ def _get_installed_packages(self):
+ return sorted([p for p in self.config if self._is_install_package(p)])
+
+ def _is_install_package(self, package):
+ if self.config[package] is None:
+ return True
+ if self.config[package].get('uninstall') is True:
+ return False
+ return True
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+
+from tools.package import PackageConfigReader
+
+
+def test_packages():
+ with mock.patch('tools.package.open', mock.mock_open(read_data=FAKE_PACKAGES_YAML)) as _:
+ config = PackageConfigReader('test-config')
+ assert config.get('install') == ['install-pkg-1',
+ 'install-pkg-2']
+ assert config.get('uninstall') == ['remove-pkg-1',
+ 'remove-pkg-2']
+
+
+def test_empty():
+ with mock.patch('tools.package.open', mock.mock_open(read_data="")) as _:
+ config = PackageConfigReader('test-config')
+ assert config.get('install') == []
+
+
+FAKE_PACKAGES_YAML = """
+remove-pkg-1:
+ uninstall: True
+remove-pkg-2:
+ uninstall: True
+
+install-pkg-1:
+ uninstall: False
+install-pkg-2:
+"""
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+from tools.executor import Executor
+
+
+class ReleaseReader(object):
+ def __init__(self, manifest_dir):
+ self.manifest_dir = manifest_dir
+
+ def read_current_release(self):
+ return Executor(chdir=self.manifest_dir).run(
+ ["git", "describe", "--tags", "--abbrev=0"]).stdout
+
+ def get_next_release_label(self):
+ current = self.read_current_release()
+ match = re.search(r'^(.+[\.-])(\d+)$', current)
+ return '{}{}'.format(match.group(1), (int(match.group(2)) + 1))
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+import mock
+
+from tools.releasereader import ReleaseReader
+from tools.executor import Result
+
+
+@pytest.mark.parametrize('curr_release, next_release', [
+ ('ABC_D19-0', 'ABC_D19-1'),
+ ('ABC_D19-19', 'ABC_D19-20'),
+ ('ABC_D19A-1', 'ABC_D19A-2'),
+ ('ABC_D19A-1.0', 'ABC_D19A-1.1'),
+ ('ABC_D20-0', 'ABC_D20-1')
+])
+@mock.patch('tools.releasereader.Executor')
+def test_next_release(mock_exec, curr_release, next_release):
+ mock_exec.return_value.run.return_value = Result(0, curr_release + '\n', '')
+ assert ReleaseReader('test-path').get_next_release_label() == next_release
+ mock_exec.assert_called_once_with(chdir='test-path')
+ mock_exec.return_value.run.assert_called_once_with(['git', 'describe', '--tags', '--abbrev=0'])
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from tools.buildconfig import BuildConfigParser
+from tools.statics import BUILD_CONFIG_PATH
+from tools.utils import validate_environ
+
+
+class RepositoryConfig(object):
+ def __init__(self, ini_file=BUILD_CONFIG_PATH):
+ self.config = BuildConfigParser(ini_file=ini_file)
+
+ def read_section(self, section):
+ repositories = []
+ for repo_name, repo_value in self.config.items(section):
+ parts = repo_value.split('#')
+ repodata = dict(name=repo_name, baseurl=parts[0])
+ for p in parts[1:]:
+ key, value = p.split('=', 1)
+ repodata[key] = value
+ repositories.append(repodata)
+ return repositories
+
+ def read_sections(self, sections):
+ repositories = []
+ for s in sections:
+ repositories += self.read_section(s)
+ return repositories
+
+ @classmethod
+ def get_localrepo(cls, remote=False):
+ dirname = 'repo'
+ if remote:
+ validate_environ(['BUILD_URL'])
+ baseurl = os.path.join(os.environ['BUILD_URL'], 'artifact/results', dirname)
+ else:
+ validate_environ(['WORKSPACE'])
+ baseurl = 'file://' + \
+ os.path.join(os.environ['WORKSPACE'], 'results', dirname)
+ return dict(name='localrepo', baseurl=baseurl)
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+
+class RpmData(dict):
+
+ @property
+ def name(self):
+ return self['Name']
+
+ @property
+ def epoch(self):
+ return self.get('Epoch', '0')
+
+ @property
+ def version(self):
+ return self['Version']
+
+ @property
+ def release(self):
+ return self['Release']
+
+ @property
+ def arch(self):
+ return self['Architecture']
+
+ @property
+ def vendor(self):
+ return self.get('Vendor', '')
+
+ @property
+ def license(self):
+ return self.get('License', '')
+
+ def __str__(self):
+ return '{}-{}-{}.{}'.format(self.name,
+ self.version,
+ self.release,
+ self.arch)
+
+ def is_same_package_as(self, other):
+ for attr in ['name', 'epoch', 'version', 'release', 'arch']:
+ if getattr(self, attr) != getattr(other, attr):
+ return False
+ return True
+
+
+class RpmInfoParser(object):
+ """
+ Parse 'rpm -qi' output
+ """
+
+ def parse_file(self, rpm_info_installed_file_path):
+ with open(rpm_info_installed_file_path, 'r') as f:
+ return self.parse_multiple(f.read())
+
+ def parse_multiple(self, rpm_info_output_multiple):
+ packages = []
+ package_index = -1
+ for line in rpm_info_output_multiple.splitlines():
+ if re.match(r'^Name\s+:.*', line):
+ packages.append(line)
+ package_index += 1
+ else:
+ packages[package_index] += '\n' + line
+ return [self.parse_package(pkg) for pkg in packages]
+
+ @staticmethod
+ def parse_package(rpm_info_output):
+ result = RpmData()
+ current_key = None
+ colon_location = rpm_info_output.splitlines()[0].find(':')
+ matcher = re.compile(r'^([A-Z][A-Za-z0-9 ]{{{}}}):( ?| .+)$'.format(colon_location - 1))
+ for line in rpm_info_output.splitlines():
+ match = matcher.match(line)
+ if match:
+ parsed_key = match.group(1).rstrip()
+ parsed_value = match.group(2).strip()
+ result[parsed_key] = parsed_value
+ current_key = parsed_key
+ else:
+ if not result[current_key]:
+ result[current_key] = line
+ else:
+ result[current_key] = result[current_key] + '\n' + line
+ return result
--- /dev/null
+Name : python-futures
+Version : 3.0.3
+Release : 1.el7
+Architecture: noarch
+Install Date: Wed Feb 7 13:49:03 2018
+Group : Development/Libraries
+Size : 88366
+License : BSD
+Signature : RSA/SHA1, Fri Sep 1 15:07:30 2017, Key ID f9b9fee7764429e6
+Source RPM : python-futures-3.0.3-1.el7.src.rpm
+Build Date : Tue Jul 28 12:27:11 2015
+Build Host : c1bj.rdu2.centos.org
+Relocations : (not relocatable)
+Packager : CBS <cbs@centos.org>
+Vendor : CentOS
+URL : https://github.com/agronholm/pythonfutures
+Summary : Backport of the concurrent.futures package from Python 3.2
+Description :
+The concurrent.futures module provides a high-level interface for
+asynchronously executing callables.
+Name : libgnome-keyring
+Version : 3.8.0
+Release : 3.el7
+Architecture: x86_64
+Install Date: Wed Feb 7 13:50:17 2018
+Group : System Environment/Libraries
+Size : 303457
+License : GPLv2+ and LGPLv2+
+Signature : RSA/SHA256, Fri Jul 4 02:48:34 2014, Key ID 24c6a8a7f4a80eb5
+Source RPM : libgnome-keyring-3.8.0-3.el7.src.rpm
+Build Date : Tue Jun 10 08:58:52 2014
+Build Host : worker1.bsys.centos.org
+Relocations : (not relocatable)
+Packager : CentOS BuildSystem <http://bugs.centos.org>
+Vendor : CentOS
+URL : http://live.gnome.org/GnomeKeyring
+Summary : Framework for managing passwords and other secrets
+Description :
+gnome-keyring is a program that keep password and other secrets for
+users. The library libgnome-keyring is used by applications to integrate
+with the gnome-keyring system.
+Name : python-ldap3
+Version : 0.9.8.6
+Release : 2.el7
+Architecture: noarch
+Install Date: Wed Feb 7 13:52:25 2018
+Group : Unspecified
+Size : 3120182
+License : LGPLv2+
+Signature : RSA/SHA1, Fri Sep 1 15:07:38 2017, Key ID f9b9fee7764429e6
+Source RPM : python-ldap3-0.9.8.6-2.el7.src.rpm
+Build Date : Fri Nov 20 16:28:01 2015
+Build Host : c1bk.rdu2.centos.org
+Relocations : (not relocatable)
+Packager : CBS <cbs@centos.org>
+Vendor : CentOS
+URL : https://pypi.python.org/pypi/ldap3/0.9.8.6
+Summary : Strictly RFC 4511 conforming LDAP V3 pure Python client
+Description :
+python-ldap3 is a strictly RFC 4511 conforming LDAP V3 pure Python client.
+The same codebase works with Python, Python 3, PyPy and PyPy3.
+Name : python-dracclient
+Version : 1.3.1
+Release : 0.20170926083913.8f368ed.el7.centos
+Architecture: noarch
+Install Date: Wed Feb 7 13:49:30 2018
+Group : Unspecified
+Size : 1325637
+License : ASL 2.0
+Signature : (none)
+Source RPM : python-dracclient-1.3.1-0.20170926083913.8f368ed.el7.centos.src.rpm
+Build Date : Tue Sep 26 08:39:32 2017
+Build Host : trunk-primary.rdoproject.org.rdocloud
+Relocations : (not relocatable)
+URL : http://github.com/openstack/python-dracclient
+Summary : Library for managing machines with Dell iDRAC cards.
+Description :
+Library for managing machines with Dell iDRAC cards.
+Name : dialog
+Version : 1.2
+Release : 4.20130523.el7
+Architecture: x86_64
+Install Date: Wed Feb 7 13:49:48 2018
+Group : Applications/System
+Size : 517263
+License : LGPLv2
+Signature : RSA/SHA256, Fri Jul 4 01:08:16 2014, Key ID 24c6a8a7f4a80eb5
+Source RPM : dialog-1.2-4.20130523.el7.src.rpm
+Build Date : Tue Jun 10 01:38:26 2014
+Build Host : worker1.bsys.centos.org
+Relocations : (not relocatable)
+Packager : CentOS BuildSystem <http://bugs.centos.org>
+Vendor : CentOS
+URL : http://invisible-island.net/dialog/dialog.html
+Summary : A utility for creating TTY dialog boxes
+Description :
+Dialog is a utility that allows you to show dialog boxes (containing
+questions or messages) in TTY (text mode) interfaces. Dialog is called
+from within a shell script. The following dialog boxes are implemented:
+yes/no, menu, input, message, text, info, checklist, radiolist, and
+gauge.
+
+Install dialog if you would like to create TTY dialog boxes.
+Name : filesystem
+Version : 3.2
+Release : 21.el7
+Architecture: x86_64
+Install Date: Wed Feb 7 13:44:24 2018
+Group : System Environment/Base
+Size : 0
+License : Public Domain
+Signature : RSA/SHA256, Sun Nov 20 17:43:00 2016, Key ID 24c6a8a7f4a80eb5
+Source RPM : filesystem-3.2-21.el7.src.rpm
+Build Date : Sat Nov 5 15:39:14 2016
+Build Host : worker1.bsys.centos.org
+Relocations : (not relocatable)
+Packager : CentOS BuildSystem <http://bugs.centos.org>
+Vendor : CentOS
+URL : https://fedorahosted.org/filesystem
+Summary : The basic directory layout for a Linux system
+Description :
+The filesystem package is one of the basic packages that is installed
+on a Linux system. Filesystem contains the basic directory layout
+for a Linux operating system, including the correct permissions for
+the directories.
+Name : python2-tenacity
+Version : 4.4.0
+Release : 1.el7
+Architecture: noarch
+Install Date: Wed Feb 7 13:49:04 2018
+Group : Unspecified
+Size : 159111
+License : ASL 2.0
+Signature : RSA/SHA1, Fri Sep 1 15:11:29 2017, Key ID f9b9fee7764429e6
+Source RPM : python-tenacity-4.4.0-1.el7.src.rpm
+Build Date : Sat Aug 5 13:47:41 2017
+Build Host : c1bd.rdu2.centos.org
+Relocations : (not relocatable)
+Packager : CBS <cbs@centos.org>
+Vendor : CentOS
+URL : https://github.com/jd/tenacity
+Summary : Tenacity is a general purpose retrying library
+Description :
+ Tenacity is a general purpose retrying library
\ No newline at end of file
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+
+import pytest
+
+from tools.rpm import RpmInfoParser
+from tools.rpm_test_data import \
+ bash_expected, conntrack_tools_expected, cpp_expected, usbredir_expected, perl_compress_expected
+from tools.test_data_rpm import \
+ bash_rpm_info, cpp_rpm_info, conntrack_tools_rpm_info, usbredir_rpm_info, perl_compress_rpm_info
+
+
+@pytest.mark.parametrize('rpm_info, expected_output', [
+ (bash_rpm_info, bash_expected),
+ (conntrack_tools_rpm_info, conntrack_tools_expected),
+ (cpp_rpm_info, cpp_expected),
+ (usbredir_rpm_info, usbredir_expected),
+ (perl_compress_rpm_info, perl_compress_expected)
+])
+def test_parse_package(rpm_info, expected_output):
+ parsed = RpmInfoParser().parse_package(rpm_info)
+ assert parsed == expected_output
+
+
+def test_parse_multiple():
+ parsed = RpmInfoParser().parse_multiple('\n'.join([bash_rpm_info, conntrack_tools_rpm_info]))
+ assert parsed == [bash_expected, conntrack_tools_expected]
+
+
+def test_parse_file():
+ test_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+ 'rpm_info_installed.sample')
+ parsed = RpmInfoParser().parse_file(test_file)
+ with open(test_file, 'r') as f:
+ expected_rpms = re.findall(r'^Name\s+:.*$', f.read(), re.MULTILINE)
+ assert len(parsed) == len(expected_rpms)
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=invalid-name,line-too-long
+
+
+bash_expected = {
+ 'Name': 'bash',
+ 'Version': '4.2.46',
+ 'Release': '21.el7_3',
+ 'Architecture': 'x86_64',
+ 'Install Date': 'Thu 11 Jan 2018 12:32:51 PM EET',
+ 'Group': 'System Environment/Shells',
+ 'Size': '3663714',
+ 'License': 'GPLv3+',
+ 'Signature': 'RSA/SHA256, Wed 07 Dec 2016 02:11:28 AM EET, Key ID 24c6a8a7f4a80eb5',
+ 'Source RPM': 'bash-4.2.46-21.el7_3.src.rpm',
+ 'Build Date': 'Wed 07 Dec 2016 01:21:54 AM EET',
+ 'Build Host': 'c1bm.rdu2.centos.org',
+ 'Relocations': '(not relocatable)',
+ 'Packager': 'CentOS BuildSystem <http://bugs.centos.org>',
+ 'Vendor': 'CentOS',
+ 'URL': 'http://www.gnu.org/software/bash',
+ 'Summary': 'The GNU Bourne Again shell',
+ 'Description': '\n'.join(
+ ['The GNU Bourne Again shell (Bash) is a shell or command language',
+ 'interpreter that is compatible with the Bourne shell (sh). Bash',
+ 'incorporates useful features from the Korn shell (ksh) and the C shell',
+ '(csh). Most sh scripts can be run by bash without modification.'])
+}
+
+conntrack_tools_expected = {
+ 'Name': 'conntrack-tools',
+ 'Version': '1.4.4',
+ 'Release': '3.el7_3',
+ 'Architecture': 'x86_64',
+ 'Install Date': 'Thu 11 Jan 2018 12:39:20 PM EET',
+ 'Group': 'System Environment/Base',
+ 'Size': '562826',
+ 'License': 'GPLv2',
+ 'Signature': 'RSA/SHA256, Thu 29 Jun 2017 03:36:05 PM EEST, Key ID 24c6a8a7f4a80eb5',
+ 'Source RPM': 'conntrack-tools-1.4.4-3.el7_3.src.rpm',
+ 'Build Date': 'Thu 29 Jun 2017 03:18:42 AM EEST',
+ 'Build Host': 'c1bm.rdu2.centos.org',
+ 'Relocations': '(not relocatable)',
+ 'Packager': 'CentOS BuildSystem <http://bugs.centos.org>',
+ 'Vendor': 'CentOS',
+ 'URL': 'http://netfilter.org',
+ 'Summary': 'Manipulate netfilter connection tracking table and run High Availability',
+ 'Description': '\n'.join([
+ 'With conntrack-tools you can setup a High Availability cluster and',
+ 'synchronize conntrack state between multiple firewalls.',
+ '',
+ 'The conntrack-tools package contains two programs:',
+ '- conntrack: the command line interface to interact with the connection',
+ ' tracking system.',
+ '- conntrackd: the connection tracking userspace daemon that can be used to',
+ ' deploy highly available GNU/Linux firewalls and collect',
+ ' statistics of the firewall use.',
+ '',
+ 'conntrack is used to search, list, inspect and maintain the netfilter',
+ 'connection tracking subsystem of the Linux kernel.',
+ 'Using conntrack, you can dump a list of all (or a filtered selection of)',
+ 'currently tracked connections, delete connections from the state table,',
+ 'and even add new ones.',
+ 'In addition, you can also monitor connection tracking events, e.g.',
+ 'show an event message (one line) per newly established connection.'
+ ])
+}
+
+cpp_expected = {
+ 'Name': 'cpp',
+ 'Version': '4.8.5',
+ 'Release': '11.el7',
+ 'Architecture': 'x86_64',
+ 'Install Date': 'Thu 11 Jan 2018 12:37:55 PM EET',
+ 'Group': 'Development/Languages',
+ 'Size': '15632501',
+ 'License': 'GPLv3+ and GPLv3+ with exceptions and GPLv2+ with exceptions and LGPLv2+ and BSD',
+ 'Signature': 'RSA/SHA256, Sun 20 Nov 2016 07:27:00 PM EET, Key ID 24c6a8a7f4a80eb5',
+ 'Source RPM': 'gcc-4.8.5-11.el7.src.rpm',
+ 'Build Date': 'Fri 04 Nov 2016 06:01:22 PM EET',
+ 'Build Host': 'worker1.bsys.centos.org',
+ 'Relocations': '(not relocatable)',
+ 'Packager': 'CentOS BuildSystem <http://bugs.centos.org>',
+ 'Vendor': 'CentOS',
+ 'URL': 'http://gcc.gnu.org',
+ 'Summary': 'The C Preprocessor',
+ 'Description':
+ '\n'.join(['Cpp is the GNU C-Compatible Compiler Preprocessor.',
+ 'Cpp is a macro processor which is used automatically',
+ 'by the C compiler to transform your program before actual',
+ 'compilation. It is called a macro processor because it allows',
+ 'you to define macros, abbreviations for longer',
+ 'constructs.',
+ '',
+ 'The C preprocessor provides four separate functionalities: the',
+ 'inclusion of header files (files of declarations that can be',
+ 'substituted into your program); macro expansion (you can define macros,',
+ 'and the C preprocessor will replace the macros with their definitions',
+ 'throughout the program); conditional compilation (using special',
+ 'preprocessing directives, you can include or exclude parts of the',
+ 'program according to various conditions); and line control (if you use',
+ 'a program to combine or rearrange source files into an intermediate',
+ 'file which is then compiled, you can use line control to inform the',
+ 'compiler about where each source line originated).',
+ '',
+ 'You should install this package if you are a C programmer and you use',
+ 'macros.']),
+}
+
+usbredir_expected = {
+ 'Name': 'usbredir',
+ 'Version': '0.7.1',
+ 'Release': '1.el7',
+ 'Architecture': 'x86_64',
+ 'Install Date': 'Wed Feb 7 13:49:24 2018',
+ 'Group': 'System Environment/Libraries',
+ 'Size': '108319',
+ 'License': 'LGPLv2+',
+ 'Signature': 'RSA/SHA256, Sun Nov 20 20:56:49 2016, Key ID 24c6a8a7f4a80eb5',
+ 'Source RPM': 'usbredir-0.7.1-1.el7.src.rpm',
+ 'Build Date': 'Sat Nov 5 18:33:15 2016',
+ 'Build Host': 'worker1.bsys.centos.org',
+ 'Relocations': '(not relocatable)',
+ 'Packager': 'CentOS BuildSystem <http://bugs.centos.org>',
+ 'Vendor': 'CentOS',
+ 'URL': 'http://spice-space.org/page/UsbRedir',
+ 'Summary': 'USB network redirection protocol libraries',
+ 'Description': '\n'.join([
+ 'The usbredir libraries allow USB devices to be used on remote and/or virtual',
+ 'hosts over TCP. The following libraries are provided:',
+ '',
+ 'usbredirparser:',
+ 'A library containing the parser for the usbredir protocol',
+ '',
+ 'usbredirhost:',
+ 'A library implementing the USB host side of a usbredir connection.',
+ 'All that an application wishing to implement a USB host needs to do is:',
+ '* Provide a libusb device handle for the device',
+ '* Provide write and read callbacks for the actual transport of usbredir data',
+ '* Monitor for usbredir and libusb read/write events and call their handlers'])
+}
+
+perl_compress_expected = {
+ 'Name': 'perl-Compress-Raw-Zlib',
+ 'Epoch': '1',
+ 'Version': '2.061',
+ 'Release': '4.el7',
+ 'Architecture': 'x86_64',
+ 'Install Date': 'Sat Jan 26 20:05:50 2019',
+ 'Group': 'Development/Libraries',
+ 'Size': '139803',
+ 'License': 'GPL+ or Artistic',
+ 'Signature': 'RSA/SHA256, Fri Jul 4 04:15:33 2014, Key ID 24c6a8a7f4a80eb5',
+ 'Source RPM': 'perl-Compress-Raw-Zlib-2.061-4.el7.src.rpm',
+ 'Build Date': 'Tue Jun 10 01:12:08 2014',
+ 'Build Host': 'worker1.bsys.centos.org',
+ 'Relocations': '(not relocatable)',
+ 'Packager': 'CentOS BuildSystem <http://bugs.centos.org>',
+ 'Vendor': 'CentOS',
+ 'URL': 'http://search.cpan.org/dist/Compress-Raw-Zlib/',
+ 'Summary': 'Low-level interface to the zlib compression library',
+ 'Description': '\n'.join([
+ 'The Compress::Raw::Zlib module provides a Perl interface to the zlib',
+ 'compression library, which is used by IO::Compress::Zlib.']),
+ 'Obsoletes': ''
+}
--- /dev/null
+#!/usr/bin/env python
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import sys
+import argparse
+from operator import itemgetter
+from pprint import pformat
+
+from tools.convert import CsvConverter
+from tools.io import read_json, write_to, write_json
+from tools.log import set_logging
+
+
+class SwComponent(dict):
+
+ @property
+ def name(self):
+ return self['Name']
+
+ @property
+ def version(self):
+ return self['Version']
+
+ @property
+ def foss_id(self):
+ return self.name, self.version, self.get('Release')
+
+ def __str__(self):
+ return '{}:{}({})'.format(self.name,
+ self.version,
+ self['Source RPM'])
+
+
+class BuildDiffReader(object):
+
+ def __init__(self):
+ self.changes = {}
+ self.summary = {}
+
+ @staticmethod
+ def get_component_names(data):
+ return [i['Name'] for i in data]
+
+ @staticmethod
+ def get_components(name, components):
+ out = []
+ for component in components:
+ if component['Name'] == name:
+ out.append(SwComponent(component))
+ return sorted(out, key=itemgetter('Name', 'Version', 'Source RPM'))
+
+ def read(self, json_old, json_new):
+ old_build = read_json(json_old)
+ new_build = read_json(json_new)
+ self.summary['input'] = {}
+ self.summary['input']['from'] = len(old_build)
+ self.summary['input']['to'] = len(new_build)
+ self.changes = self.read_build_diff(old_build, new_build)
+ self.summary['output'] = self._generate_summary(self.changes)
+
+ @staticmethod
+ def _generate_summary(changes):
+ summary = {}
+ summary['counts'] = changes['counts']
+ summary['added'] = {name: [str(c) for c in compos] for name, compos in
+ changes['added'].items()}
+ summary['removed'] = {name: [str(c) for c in compos] for name, compos in
+ changes['removed'].items()}
+ summary['changed'] = {name: {'old': [str(c) for c in change['old']],
+ 'new': [str(c) for c in change['new']]} for name, change in
+ changes['changed'].items()}
+ return summary
+
+ def read_build_diff(self, old_build, new_build):
+ old_names = self.get_component_names(old_build)
+ logging.debug('Old names: {}'.format(old_names))
+ new_names = self.get_component_names(new_build)
+ logging.debug('New names: {}'.format(new_names))
+ added = {n: self.get_components(n, new_build) for n in set(new_names) - set(old_names)}
+ self._mark('[MARK] added', [j for i in added.values() for j in i])
+ removed = {n: self.get_components(n, old_build) for n in set(old_names) - set(new_names)}
+ self._mark('[MARK] removed', [j for i in removed.values() for j in i])
+ changed = {}
+ for n in set(old_names) & set(new_names):
+ old_components = self.get_components(n, old_build)
+ new_components = self.get_components(n, new_build)
+ if sorted([i.foss_id for i in old_components]) != \
+ sorted([i.foss_id for i in new_components]):
+ changed[n] = {'old': old_components, 'new': new_components}
+ self._mark('[MARK] changed old', changed[n]['old'])
+ self._mark('[MARK] changed new', changed[n]['new'])
+ return dict(counts=dict(added=len(added),
+ changed=len(changed),
+ removed=len(removed)),
+ added=added,
+ removed=removed,
+ changed=changed)
+
+ @staticmethod
+ def _mark(title, components):
+ logging.debug(
+ '[MARK] {}: {}'.format(title, pformat([i.foss_id for i in components])))
+
+ @staticmethod
+ def _get_csv_cells(name, old_components, new_components):
+ cells = dict(name=name)
+ if old_components:
+ cells.update(dict(old_components='\n'.join([str(i) for i in old_components]),
+ old_srpms='\n'.join([i['Source RPM'] for i in old_components]),
+ old_licenses='\n'.join(
+ [i.get('License', 'Unknown') for i in old_components])))
+ if new_components:
+ cells.update(dict(new_components='\n'.join([str(i) for i in new_components]),
+ new_srpms='\n'.join([i['Source RPM'] for i in new_components]),
+ new_licenses='\n'.join(
+ [i.get('License', 'Unknown') for i in new_components])))
+ return cells
+
+ def write_csv(self, path):
+ data = []
+ for name, components in self.changes['added'].items():
+ data += [self._get_csv_cells(name, [], components)]
+
+ for name, components in self.changes['removed'].items():
+ data += [self._get_csv_cells(name, components, [])]
+
+ for name, components in self.changes['changed'].items():
+ data += [self._get_csv_cells(name, components['old'], components['new'])]
+
+ csv = CsvConverter(sorted(data, key=itemgetter('name')),
+ preferred_field_order=['name',
+ 'old_components', 'old_srpms', 'old_licenses',
+ 'new_components', 'new_srpms', 'new_licenses'],
+ escape_newlines=False)
+ write_to(path, csv.convert_to_ms_excel())
+
+
+def parse(args):
+ parser = argparse.ArgumentParser(description='Outputs RPM changes between two CI builds')
+ parser.add_argument('--verbose', '-v', action='store_true',
+ help='More verbose logging')
+ parser.add_argument('components_json_1',
+ help='Components json file path (CI build artifact)')
+ parser.add_argument('components_json_2',
+ help='Components json file path (CI build artifact)')
+ parser.add_argument('--output-json',
+ help='output to json file')
+ parser.add_argument('--output-csv',
+ help='output to $MS csv file')
+ return parser.parse_args(args)
+
+
+def main(input_args):
+ args = parse(input_args)
+ set_logging(debug=args.verbose)
+ x = BuildDiffReader()
+ x.read(args.components_json_1, args.components_json_2)
+ logging.info('----- SUMMARY ------\n{}'.format(pformat(x.summary)))
+ if args.output_json:
+ write_json(args.output_json, x.changes)
+ if args.output_csv:
+ x.write_csv(args.output_csv)
+
+
+if __name__ == '__main__':
+ main(sys.argv[1:])
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=line-too-long,invalid-name
+import json
+
+from tools.script.ci_build_diff import main
+from tools.script.ci_build_diff_test_data import component_added, component_changed_old, \
+ component_changed_new, component_removed, caas_grafana1, caas_grafana2, caas_grafana2_sub, \
+ caas_grafana1_sub, caas_grafana1_v2, caas_grafana1_v2sub, caas_grafana1_r2, \
+ caas_grafana1_r2sub, grafana_v1, grafana_v2, caas_grafana3, caas_grafana3_sub, caas_abc1_sub, \
+ caas_abc1, caas_abc1_r2, caas_abc1_sub_r2, abc1_v2, abc1, abc2, abc3, \
+ caas_grafana1_sub_new_field
+
+
+def test_no_output_args(tmpdir):
+ input_old, input_new = _gen_input_json(tmpdir)
+ main([str(input_old), str(input_new)])
+
+
+def test_no_changes(tmpdir):
+ input_old, _ = _gen_input_json(tmpdir)
+ output_csv = tmpdir.join('diff.csv')
+ output_json = tmpdir.join('diff.json')
+ main([str(input_old), str(input_old),
+ '--output-csv', str(output_csv),
+ '--output-json', str(output_json)])
+ assert output_csv.read() == 'sep=,'
+ assert json.loads(output_json.read()) == {"added": {},
+ "changed": {},
+ "removed": {},
+ "counts": {
+ "added": 0,
+ "changed": 0,
+ "removed": 0
+ }}
+
+
+def test_no_changes_if_json_field_is_added(tmpdir):
+ """
+ Make sure diff is ok e.g. when new build has new field in json
+ """
+ _assert_json_out(tmpdir,
+ [caas_grafana1_sub],
+ [caas_grafana1_sub_new_field],
+ {"added": {},
+ "changed": {},
+ "removed": {},
+ "counts": {
+ "added": 0,
+ "changed": 0,
+ "removed": 0
+ }})
+
+
+def test_multiple_rpms(tmpdir):
+ _assert_json_out(tmpdir,
+ [component_changed_old, component_removed],
+ [component_changed_new, component_added],
+ {"added": {component_added['Name']: [component_added]},
+ "changed": {component_changed_old['Name']: {'old': [component_changed_old],
+ 'new': [component_changed_new]}},
+ "removed": {component_removed['Name']: [component_removed]},
+ "counts": {
+ "added": 1,
+ "changed": 1,
+ "removed": 1
+ }})
+
+
+def test_component_name_change(tmpdir):
+ """
+ Sub-component does not change although RPM containing it changes
+ """
+ _assert_json_out(tmpdir,
+ [caas_grafana1, caas_grafana1_sub],
+ [caas_grafana2, caas_grafana2_sub],
+ {"added": {caas_grafana2['Name']: [caas_grafana2]},
+ "changed": {},
+ "removed": {caas_grafana1['Name']: [caas_grafana1]},
+ "counts": {
+ "added": 1,
+ "changed": 0,
+ "removed": 1
+ }})
+
+
+def test_component_version_change(tmpdir):
+ _assert_json_out(tmpdir,
+ [caas_grafana1, caas_grafana1_sub],
+ [caas_grafana1_v2, caas_grafana1_v2sub],
+ {"added": {},
+ "changed": {caas_grafana1['Name']: {'old': [caas_grafana1],
+ 'new': [caas_grafana1_v2]},
+ caas_grafana1_sub['Name']: {'old': [caas_grafana1_sub],
+ 'new': [caas_grafana1_v2sub]}},
+ "removed": {},
+ "counts": {
+ "added": 0,
+ "changed": 2,
+ "removed": 0
+ }})
+
+
+def test_component_release_change(tmpdir):
+ _assert_json_out(tmpdir,
+ [caas_grafana1, caas_grafana1_sub],
+ [caas_grafana1_r2, caas_grafana1_r2sub],
+ {"added": {},
+ "changed": {caas_grafana1['Name']: {'old': [caas_grafana1],
+ 'new': [caas_grafana1_r2]}},
+ "removed": {},
+ "counts": {
+ "added": 0,
+ "changed": 1,
+ "removed": 0
+ }})
+
+
+def test_same_name_component_added(tmpdir):
+ _assert_json_out(tmpdir,
+ [caas_grafana1, caas_grafana1_sub],
+ [caas_grafana1, caas_grafana1_sub, grafana_v1],
+ {"added": {},
+ "changed": {grafana_v1['Name']: {'old': [caas_grafana1_sub],
+ 'new': [caas_grafana1_sub, grafana_v1]}},
+ "removed": {},
+ "counts": {
+ "added": 0,
+ "changed": 1,
+ "removed": 0
+ }})
+
+
+def test_same_name_component_removed(tmpdir):
+ _assert_json_out(tmpdir,
+ [caas_grafana1, caas_grafana1_sub, grafana_v1],
+ [caas_grafana1, caas_grafana1_sub],
+ {"added": {},
+ "changed": {grafana_v1['Name']: {'old': [caas_grafana1_sub, grafana_v1],
+ 'new': [caas_grafana1_sub]}},
+ "removed": {},
+ "counts": {
+ "added": 0,
+ "changed": 1,
+ "removed": 0
+ }})
+
+
+def test_same_name_component_changed(tmpdir):
+ _assert_json_out(tmpdir,
+ [caas_grafana1, caas_grafana1_sub, grafana_v1],
+ [caas_grafana1, caas_grafana1_sub, grafana_v2],
+ {"added": {},
+ "changed": {grafana_v1['Name']: {'old': [caas_grafana1_sub, grafana_v1],
+ 'new': [caas_grafana1_sub, grafana_v2]}},
+ "removed": {},
+ "counts": {
+ "added": 0,
+ "changed": 1,
+ "removed": 0
+ }})
+
+
+def test_epic(tmpdir):
+ _assert_json_out(tmpdir,
+ [caas_grafana1, caas_grafana1_sub,
+ caas_grafana2, caas_grafana2_sub,
+ grafana_v1],
+ [caas_grafana1_r2, caas_grafana1_r2sub,
+ grafana_v2,
+ caas_grafana3, caas_grafana3_sub],
+ {"added": {caas_grafana3['Name']: [caas_grafana3]},
+ "changed": {caas_grafana1['Name']: {'old': [caas_grafana1],
+ 'new': [caas_grafana1_r2]},
+ grafana_v1['Name']: {
+ 'old': [caas_grafana1_sub, caas_grafana2_sub, grafana_v1],
+ 'new': [caas_grafana1_r2sub, caas_grafana3_sub, grafana_v2]}},
+ "removed": {caas_grafana2['Name']: [caas_grafana2]},
+ "counts": {
+ "added": 1,
+ "changed": 2,
+ "removed": 1
+ }})
+
+
+def _assert_json_out(tmpdir, from_build, to_build, expected_output):
+ input_old, input_new = _gen_input_json(tmpdir, from_build, to_build)
+ output_json = tmpdir.join('diff.json')
+ main([str(input_old), str(input_new), '--output-json', str(output_json)])
+ assert json.loads(output_json.read()) == expected_output
+
+
+def test_csv(tmpdir):
+ input_old, input_new = _gen_input_json(tmpdir,
+ [caas_abc1, caas_abc1_sub, abc1, abc2],
+ [caas_abc1_r2, caas_abc1_sub_r2, abc1_v2, abc3])
+ output_csv = tmpdir.join('diff.csv')
+ main([str(input_old), str(input_new), '--output-csv', str(output_csv)])
+ rows = [['name',
+ 'old_components', 'old_srpms', 'old_licenses',
+ 'new_components', 'new_srpms', 'new_licenses'],
+ ['abc',
+ 'abc:v1(abc-v1-r1.src.rpm)\nabc:v1(caas-abc-v1-r1.src.rpm)',
+ 'abc-v1-r1.src.rpm\ncaas-abc-v1-r1.src.rpm',
+ 'GPL\nUnknown',
+ 'abc:v1(caas-abc-v1-r2.src.rpm)\nabc:v2(abc-v2-r1.src.rpm)',
+ 'caas-abc-v1-r2.src.rpm\nabc-v2-r1.src.rpm',
+ 'Unknown\nGPL'],
+ ['abc2',
+ 'abc2:v1(abc2-v1-r1.src.rpm)', 'abc2-v1-r1.src.rpm', 'GPL',
+ 'None', 'None', 'None'],
+ ['abc3',
+ 'None', 'None', 'None',
+ 'abc3:v1(abc3-v1-r1.src.rpm)', 'abc3-v1-r1.src.rpm', 'GPL'],
+ ['caas-abc',
+ 'caas-abc:v1(caas-abc-v1-r1.src.rpm)', 'caas-abc-v1-r1.src.rpm', 'Commercial',
+ 'caas-abc:v1(caas-abc-v1-r2.src.rpm)', 'caas-abc-v1-r2.src.rpm', 'Commercial']]
+ expected_csv = '\r\n'.join(['sep=,'] + [_get_csv_row(row) for row in rows])
+ assert output_csv.read() == expected_csv
+
+
+def _get_csv_row(items):
+ return ','.join(['"{}"'.format(i) for i in items])
+
+
+def _gen_input_json(tmpdir, _from=None, _to=None):
+ if _from is None:
+ _from = [component_changed_old, component_removed]
+ if _to is None:
+ _to = [component_changed_new, component_added]
+ input_old = tmpdir.join('input_old.json')
+ input_new = tmpdir.join('input_new.json')
+ json_old = json.dumps(_from)
+ json_new = json.dumps(_to)
+ input_old.write(json_old)
+ input_new.write(json_new)
+ return input_old, input_new
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=line-too-long,invalid-name
+import re
+
+from copy import deepcopy
+
+
+def _copy(d, pattern, repl):
+ x = deepcopy(d)
+ for k, v in x.items():
+ if isinstance(v, str):
+ x[k] = re.sub(pattern, repl, v)
+ return x
+
+
+caas_grafana1 = {
+ "Architecture": "noarch",
+ "Build Date": "Wed May 30 11:39:05 2018",
+ "Build Host": "crf-dev",
+ "Crypto capable": False,
+ "Description": "This is grafana, this is SPARTAAAAA!",
+ "FOSS": "No",
+ "From repo": "caas-artifactory",
+ "Group": "Unspecified",
+ "Install Date": "Sun Nov 18 22:15:41 2018",
+ "Is sane": True,
+ "License": "Commercial",
+ "Name": "caas.grafana",
+ "Obsoletes": "",
+ "Release": "1.el7.centos",
+ "Relocations": "(not relocatable)",
+ "Repo": "installed",
+ "Repo data": {
+ "baseurl": "http://files/20181023",
+ "name": "caas-artifactory"
+ },
+ "Signature": "(none)",
+ "Size": "109683450",
+ "Source RPM": "caas.grafana-4.4.1.9-1.el7.centos.src.rpm",
+ "Source repo data": {
+ "baseurl": "http://files/20181023",
+ "name": "caas-artifactory"
+ },
+ "Source to be delivered": "No",
+ "Summary": "caasgrafana",
+ "Vendor": "Something",
+ "Version": "4.4.1.9"
+}
+
+caas_grafana1_sub = {
+ "Name": "grafana",
+ "Version": "4.4.1.9",
+ "Source RPM": "caas.grafana-4.4.1.9-1.el7.centos.src.rpm",
+ "Source URL": "https://some/grafana/url",
+ "FOSS": "yes"
+}
+
+caas_grafana1_sub_new_field = deepcopy(caas_grafana1_sub)
+caas_grafana1_sub_new_field['ABC'] = True
+
+caas_grafana2 = _copy(caas_grafana1, 'caas.grafana', 'caas.grafana2')
+caas_grafana2_sub = _copy(caas_grafana1_sub, 'caas.grafana', 'caas.grafana2')
+
+caas_grafana3 = _copy(caas_grafana1, 'caas.grafana', 'caas.grafana3')
+caas_grafana3_sub = _copy(caas_grafana1_sub, 'caas.grafana', 'caas.grafana3')
+
+caas_grafana1_v2 = _copy(caas_grafana1, '4.4.1.9', '4.4.1.10')
+caas_grafana1_v2sub = _copy(caas_grafana1_sub, '4.4.1.9', '4.4.1.10')
+
+caas_grafana1_r2 = _copy(caas_grafana1, '1.el7.centos', '2.el7.centos')
+caas_grafana1_r2sub = _copy(caas_grafana1_sub, '1.el7.centos', '2.el7.centos')
+
+caas_abc1 = {
+ "License": "Commercial",
+ "Name": "caas-abc",
+ "Version": "v1",
+ "Release": "r1",
+ "Source RPM": "caas-abc-v1-r1.src.rpm",
+}
+
+caas_abc1_sub = {
+ "Name": "abc",
+ "Version": "v1",
+ "Source RPM": "caas-abc-v1-r1.src.rpm",
+}
+caas_abc1_r2 = _copy(caas_abc1, 'r1', 'r2')
+caas_abc1_sub_r2 = _copy(caas_abc1_sub, 'r1', 'r2')
+
+abc1 = {
+ "License": "GPL",
+ "Name": "abc",
+ "Version": "v1",
+ "Release": "r1",
+ "Source RPM": "abc-v1-r1.src.rpm",
+}
+abc1_v2 = _copy(abc1, 'v1', 'v2')
+abc2 = _copy(abc1, 'abc', 'abc2')
+abc3 = _copy(abc1, 'abc', 'abc3')
+
+component_added = {
+ 'Architecture': 'noarch',
+ 'Build Date': 'Sun Nov 11 12:54:39 2018',
+ 'Build Host': 'build-7.novalocal',
+ 'Description': 'This RPM contains configuration management openstack configuration override '
+ 'plugin',
+ 'FOSS': 'No',
+ 'From repo': 'localrepo',
+ 'Group': 'Unspecified',
+ 'Install Date': 'Tue Nov 13 19:14:29 2018',
+ 'Is sane': True,
+ 'License': 'Commercial',
+ 'Name': 'openstack-config-overrides-validator',
+ 'Obsoletes': '',
+ 'Packager': 'Something',
+ 'Release': '1.el7.centos',
+ 'Relocations': '(not relocatable)',
+ 'Repo': 'installed',
+ 'Repo data': {
+ 'baseurl': 'https://jenkins/ci-build/2490/artifact/results/repo',
+ 'name': 'localrepo'},
+ 'Signature': '(none)',
+ 'Size': '3097',
+ 'Source RPM': 'openstack-config-overrides-validator-c2.gd1b7aec-1.el7.centos.src.rpm',
+ 'Source repo data': {
+ 'baseurl': 'https://jenkins/ci-build/2490/artifact/results/src_repo',
+ 'name': 'localrepo'},
+ 'Source to be delivered': 'No',
+ 'Summary': 'Openstack configuration override CM validator plugin.',
+ 'Vendor': 'Something',
+ 'Version': 'c2.gd1b7aec'
+}
+
+component_removed = {
+ 'Architecture': 'x86_64',
+ 'Build Date': 'Thu Aug 16 14:46:11 2018',
+ 'Build Host': 'x86-01.bsys.centos.org',
+ 'Description': 'The fence-agents-ibmblade package contains a fence agent for IBM BladeCenter '
+ 'devices that are accessed via the SNMP protocol.',
+ 'FOSS': 'Undefined',
+ 'From repo': 'purkki-centos-updates',
+ 'Group': 'System Environment/Base',
+ 'Install Date': 'Wed Nov 7 21:20:01 2018',
+ 'Is sane': False,
+ 'License': 'GPLv2+ and LGPLv2+',
+ 'Name': 'fence-agents-ibmblade',
+ 'Obsoletes': 'fence-agents,',
+ 'Packager': 'CentOS BuildSystem <http://bugs.centos.org>',
+ 'Release': '86.el7_5.3',
+ 'Relocations': '(not relocatable)',
+ 'Repo': 'installed',
+ 'Repo data': {
+ 'baseurl': 'http://purkki/mirror/centos/snapshot/20181024/7/updates/x86_64/',
+ 'exclude': 'libgudev1 httpd httpd-devel systemd-libs.i686 resource-agents '
+ 'dhcp-libs dhclient dhcp-common php-fpm php-common php-cli php',
+ 'name': 'purkki-centos-updates'},
+ 'Signature': 'RSA/SHA256, Mon Aug 20 14:15:17 2018, Key ID 24c6a8a7f4a80eb5',
+ 'Size': '3898',
+ 'Source RPM': 'fence-agents-4.0.11-86.el7_5.3.src.rpm',
+ 'Source repo data': {
+ 'baseurl': 'http://purkki/mirror/centos/snapshot/20181024/7/updates/Source/',
+ 'name': 'purkki-centos-updates'},
+ 'Source to be delivered': 'Undefined',
+ 'Summary': 'Fence agent for IBM BladeCenter',
+ 'URL': 'https://github.com/ClusterLabs/fence-agents',
+ 'Vendor': 'CentOS',
+ 'Version': '4.0.11'
+}
+component_changed_old = {
+ 'Architecture': 'noarch',
+ 'Build Date': 'Fri Oct 19 00:22:00 2018',
+ 'Build Host': 'f32725a719ce4c82a53b44644dfd2718',
+ 'Description': 'This RPM contains source code for the Authentication, Authorization and '
+ 'Accounting cli',
+ 'FOSS': 'No',
+ 'From repo': 'localrepo',
+ 'Group': 'Unspecified',
+ 'Install Date': 'Wed Nov 7 21:22:14 2018',
+ 'Is sane': True,
+ 'License': 'Commercial',
+ 'Name': 'aaacli',
+ 'Obsoletes': '',
+ 'Packager': 'Something',
+ 'Release': '2.el7.centos',
+ 'Relocations': '(not relocatable)',
+ 'Repo': 'installed',
+ 'Repo data': {
+ 'baseurl': 'https://jenkins/ci-build/2432/artifact/results/repo',
+ 'name': 'localrepo'},
+ 'Signature': '(none)',
+ 'Size': '43968',
+ 'Source RPM': 'aaacli-c12.gc62d348-2.el7.centos.src.rpm',
+ 'Source repo data': {
+ 'baseurl': 'https://jenkins/ci-build/2432/artifact/results/src_repo',
+ 'name': 'localrepo'},
+ 'Source to be delivered': 'No',
+ 'Summary': 'Authentication, Authorization and Accounting Command Line Interface',
+ 'Vendor': 'Something',
+ 'Version': 'c12.gc62d348'
+}
+
+component_changed_new = {
+ 'Architecture': 'noarch',
+ 'Build Date': 'Thu Nov 8 05:05:07 2018',
+ 'Build Host': 'build-3.novalocal',
+ 'Description': 'This RPM contains source code for the Authentication, '
+ 'Authorization and Accounting cli',
+ 'FOSS': 'No',
+ 'From repo': 'localrepo',
+ 'Group': 'Unspecified',
+ 'Install Date': 'Tue Nov 13 19:09:30 2018',
+ 'Is sane': True,
+ 'License': 'Commercial',
+ 'Name': 'aaacli',
+ 'Obsoletes': '',
+ 'Packager': 'Something',
+ 'Release': '1.el7.centos',
+ 'Relocations': '(not relocatable)',
+ 'Repo': 'installed',
+ 'Repo data': {
+ 'baseurl': 'https://jenkins/ci-build/2490/artifact/results/repo',
+ 'name': 'localrepo'},
+ 'Signature': '(none)',
+ 'Size': '44034',
+ 'Source RPM': 'aaacli-c13.gcb6b490-1.el7.centos.src.rpm',
+ 'Source repo data': {
+ 'baseurl': 'https://jenkins/ci-build/2490/artifact/results/src_repo',
+ 'name': 'localrepo'},
+ 'Source to be delivered': 'No',
+ 'Summary': 'Authentication, Authorization and Accounting Command Line Interface',
+ 'Vendor': 'Something',
+ 'Version': 'c13.gcb6b490'
+}
+
+grafana_v1 = {
+ 'Architecture': 'x86_64',
+ 'Build Date': 'Mon Sep 17 14:30:25 2018',
+ 'Build Host': 'd8fb00edf57f4254bb45073a941929ff',
+ 'Description': 'Grafana is an open source, feature rich metrics dashboard and graph '
+ 'editor for\nGraphite, InfluxDB & OpenTSDB.',
+ 'FOSS': 'Modified',
+ 'From repo': 'localrepo',
+ 'Group': 'Unspecified',
+ 'Install Date': 'Fri Nov 16 02:23:02 2018',
+ 'Is sane': True,
+ 'License': 'Commercial and ASL 2.0 and others',
+ 'Name': 'grafana',
+ 'Obsoletes': '',
+ 'Packager': 'Something',
+ 'Release': '1.el7.centos.1',
+ 'Relocations': '(not relocatable)',
+ 'Repo': 'installed',
+ 'Repo data': {
+ 'baseurl': 'https://jenkins/ci-build/2506/artifact/results/repo',
+ 'name': 'localrepo'},
+ 'Signature': '(none)',
+ 'Size': '93957067',
+ 'Source RPM': 'grafana-5.2.3-1.el7.centos.1.src.rpm',
+ 'Source repo data': {
+ 'baseurl': 'https://jenkins/ci-build/2506/artifact/results/src_repo',
+ 'name': 'localrepo'},
+ 'Source to be delivered': 'Upstream',
+ 'Summary': 'Grafana is an open source, feature rich metrics dashboard and graph editor',
+ 'URL': 'https://github.com/grafana/grafana',
+ 'Vendor': 'Something and Grafana modified',
+ 'Version': '5.2.3'
+}
+
+grafana_v2 = _copy(grafana_v1, '1.el7.centos.1', '1.el7.centos.2')
--- /dev/null
+#!/usr/bin/env python
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=too-many-instance-attributes,too-many-arguments
+
+import argparse
+import copy
+import sys
+import logging
+import re
+import json
+from pprint import pformat
+
+import os
+
+from tools.rpm import RpmInfoParser
+from tools.utils import apply_jenkins_auth
+from tools.yum import Yum, YumInfoParser
+from tools.repository import RepositoryConfig
+from tools.log import set_logging
+from tools.io import read_from, write_to, read_json
+from tools.convert import to_json, CsvConverter
+
+
+class RpmDataBuilder(object):
+ def __init__(self, build_config, yum_info_installed, rpm_info_installed,
+ crypto_info_installed, boms, remote=False):
+ self.remote = remote
+ self.yum_info_installed = yum_info_installed
+ self.rpm_info_installed = rpm_info_installed
+ self.crypto_info_installed = json.loads(crypto_info_installed)
+ self.boms = boms
+ logging.debug('BOMS: {}'.format(pformat(self.boms)))
+ self.repoconfig = RepositoryConfig(build_config)
+ self.installed_rpms = None
+ self.repos = None
+
+ def run(self):
+ self.installed_rpms = self.read_installed_rpms()
+ srpms = set([rpm['Source RPM'] for rpm in self.installed_rpms])
+ logging.info('Installed RPMs:{} SRPMs:{}'.format(len(self.installed_rpms), len(srpms)))
+ self.repos = self._read_configured_repos()
+ logging.info('Configured repos: {}'.format(len(self.repos)))
+ available_rpms = self._read_available_rpms(self.repos)
+ logging.info('Found {} available RPMs in binary repos'.format(
+ len([rpm for repo_rpms in available_rpms.values() for rpm in repo_rpms])))
+ for i_rpm in self.installed_rpms:
+ i_rpm_repo_name = self._get_rpm_available_in(i_rpm, available_rpms)
+ i_rpm['Repo data'] = self._get_repo(i_rpm_repo_name)
+ i_rpm['Obsoletes'] = self._resolve_obsoletes(i_rpm)
+ i_rpm['Crypto capable'] = self._resolve_ecc(str(i_rpm))
+ i_rpm['BOM'] = self._resolve_bom(i_rpm)
+ self._log_repo_rpm_statistics()
+ self._log_rpm_statistics()
+ return self.installed_rpms
+
+ @staticmethod
+ def _resolve_obsoletes(rpm):
+ if 'Obsoletes' not in rpm:
+ return 'N/A'
+ elif rpm['Obsoletes'] == '(none)':
+ return 'N/A'
+ return rpm['Obsoletes']
+
+ def _resolve_ecc(self, rpm):
+ for item in self.crypto_info_installed:
+ if item['name'] == rpm:
+ return True
+ return False
+
+ def _resolve_bom(self, rpm):
+ bom_content = self.boms.get(str(rpm))
+ if bom_content is None:
+ return ''
+ self._validate_bom(str(rpm), bom_content)
+ return bom_content['bom']
+
+ @staticmethod
+ def _validate_bom(rpm_name, bom_content):
+ try:
+ if 'bom' not in bom_content:
+ raise Exception('BOM base object "bom" missing')
+ bom = bom_content['bom']
+ for material in bom:
+ for key in ['name', 'version', 'source-url', 'foss']:
+ if key not in material:
+ raise Exception('Key "{}" not found in BOM'.format(key))
+ if material['foss'].lower() not in ['yes', 'no', 'modified']:
+ raise Exception('BOM foss value not valid')
+ missing_crypto_count = len([material for material in bom if
+ 'crypto-capable' not in material])
+ if missing_crypto_count != 0:
+ logging.warning(
+ 'crypto-capable missing from %s materials in RPM %s',
+ missing_crypto_count, rpm_name)
+ except Exception as e:
+ correct_format = {'bom': [
+ {'name': '<component-name>',
+ 'version': '<component-version>',
+ 'source-url': '<source-url>',
+ 'foss': '<yes/no/modified>',
+ 'crypto-capable': '<true/false (OPTIONAL)>'}]}
+ msg_fmt = 'BOM for {rpm} is not correct format. {error}:\n{correct_format}'
+ raise Exception(msg_fmt.format(rpm=rpm_name,
+ error=str(e),
+ correct_format=pformat(correct_format)))
+
+ def _get_repo(self, name):
+ for r in self.repos:
+ if r['name'] == name:
+ return r
+ raise Exception('No repository found with name: {}'.format(name))
+
+ def read_installed_rpms(self):
+ installed_rpms = []
+ yum_rpms = YumInfoParser().parse_installed(self.yum_info_installed)
+ rpm_rpms = RpmInfoParser().parse_multiple(self.rpm_info_installed)
+ self._validate_rpm_lists_identical(yum_rpms, rpm_rpms)
+ yum_rpms_dict = {rpm['Name']: rpm for rpm in yum_rpms}
+ for rpm_data in rpm_rpms:
+ yum_data = yum_rpms_dict[rpm_data['Name']]
+ combined_data = self._combine_rpm_data(rpm_data, yum_data)
+ installed_rpms.append(combined_data)
+ logging.debug('One parsed RPM data as example:\n{}'.format(pformat(installed_rpms[0])))
+ return installed_rpms
+
+ def _combine_rpm_data(self, rpm_data, yum_data):
+ combined_data = copy.deepcopy(rpm_data)
+ fields_known_to_differ = ['Description', # May contain deffering newline and indentation
+ 'Size'] # Bytes in RPM, humanreadable in yum
+ yum2rpm_field_name_map = {'Arch': 'Architecture'}
+ for yum_key in yum_data:
+ if yum_key in yum2rpm_field_name_map:
+ rpm_key = yum2rpm_field_name_map[yum_key]
+ else:
+ rpm_key = yum_key
+ if rpm_key in combined_data:
+ yum_comparable_rpm_string = self._rpm_info_str_to_yum_info_str(
+ combined_data[rpm_key])
+ if yum_comparable_rpm_string != yum_data[yum_key]:
+ if rpm_key in fields_known_to_differ:
+ continue
+ raise Exception(
+ 'RPM data in "{}" not match in rpm "{}" vs yum "{}" for package {}'.format(
+ rpm_key,
+ repr(combined_data[rpm_key]),
+ repr(yum_data[yum_key]),
+ combined_data))
+ else:
+ combined_data[rpm_key] = yum_data[yum_key]
+ return combined_data
+
+ @staticmethod
+ def _rpm_info_str_to_yum_info_str(string):
+ try:
+ string.decode()
+ except (UnicodeEncodeError, UnicodeDecodeError):
+ return re.sub(r'[^\x00-\x7F]+', '?', string)
+ except Exception as e:
+ logging.error('{}: for string {}'.format(str(e), repr(string)))
+ raise
+ return string
+
+ @staticmethod
+ def _validate_rpm_lists_identical(yum_rpms, rpm_rpms):
+ yum_rpms_dict = {rpm['Name']: rpm for rpm in yum_rpms}
+ rpm_rpms_dict = {rpm['Name']: rpm for rpm in rpm_rpms}
+ if len(yum_rpms) != len(rpm_rpms):
+ raise Exception(
+ 'Given RPM lists are unequal: yum RPM count {} != rpm RPM count {}'.format(
+ len(yum_rpms), len(rpm_rpms)))
+ assert sorted(yum_rpms_dict.keys()) == sorted(rpm_rpms_dict.keys())
+ for name in yum_rpms_dict.keys():
+ if not yum_rpms_dict[name].is_same_package_as(rpm_rpms_dict[name]):
+ raise Exception(
+ 'Packages are not same: yum {} != rpm {}'.format(yum_rpms_dict[name],
+ rpm_rpms_dict[name]))
+
+ def _read_configured_repos(self):
+ repos = self.repoconfig.read_sections(
+ ['baseimage-repositories', 'repositories'])
+ repos.append(self.repoconfig.get_localrepo(remote=True))
+ logging.debug('Configured repos: {}'.format(pformat(repos)))
+ return repos
+
+ def _read_available_rpms(self, repos):
+ Yum.clean_and_remove_cache()
+ yum = Yum()
+ for repo in repos:
+ name = repo['name']
+ if name == 'localrepo':
+ if self.remote:
+ url = self.repoconfig.get_localrepo(remote=True)['baseurl']
+ yum.add_repo(name, apply_jenkins_auth(url))
+ else:
+ url = self.repoconfig.get_localrepo(remote=False)['baseurl']
+ yum.add_repo(name, url)
+ else:
+ yum.add_repo(name, repo['baseurl'])
+ yum_available_output = yum.read_all_packages()
+ available_rpms = YumInfoParser().parse_available(yum_available_output)
+ rpms_per_repo = {}
+ for rpm in available_rpms:
+ repo = rpm.get('Repo')
+ if repo not in rpms_per_repo:
+ rpms_per_repo[repo] = []
+ rpms_per_repo[repo].append(rpm)
+ return rpms_per_repo
+
+ def _log_repo_rpm_statistics(self):
+ logging.info('--- RPM repo statistics ---')
+ for repo in self.repos:
+ name = repo['name']
+ repo_url = repo['baseurl']
+ if name in [r['name'] for r in self._get_nonerepos()]:
+ expected_from_repo = None
+ else:
+ expected_from_repo = name
+ repo_installed_rpm_count = len([rpm for rpm in self.installed_rpms if
+ rpm['Repo data']['baseurl'] == repo_url and rpm.get(
+ 'From repo') == expected_from_repo])
+ logging.info(
+ 'RPMs installed from repo "{}": {}'.format(name, repo_installed_rpm_count))
+ if repo_installed_rpm_count is 0:
+ logging.warning(
+ 'Repository configured but no RPMs installed: {}={}'.format(name, repo_url))
+
+ return self.installed_rpms
+
+ def _log_rpm_statistics(self):
+ def _get_count(func):
+ return len([rpm for rpm in self.installed_rpms if func(rpm)])
+
+ logging.info('----- RPMs per type -----')
+ logging.info(' => Total: %s', len(self.installed_rpms))
+ logging.info('----- RPMs per attribute -----')
+ logging.info(' * Crypto capable: %s', _get_count(lambda rpm: rpm['Crypto capable']))
+ logging.info(' * Complex (BOM): %s', _get_count(lambda rpm: rpm['BOM']))
+
+ def _get_rpm_available_in(self, rpm, available_rpms):
+ if 'From repo' in rpm.keys():
+ if rpm['From repo'] == 'localrepo':
+ return 'localrepo'
+ available_repo_rpms = available_rpms[rpm['From repo']]
+ for a_rpm in available_repo_rpms:
+ if self._is_same_rpm(a_rpm, rpm):
+ return rpm['From repo']
+ rpms_in_matching_repo = [str(a_rpm) for a_rpm in available_repo_rpms]
+ rpms_with_matching_name = [str(a_rpm) for a_rpm in available_repo_rpms if
+ rpm['Name'] == a_rpm['Name']]
+ if len(rpms_in_matching_repo) <= 1000:
+ logging.debug(
+ 'Available RPMs in {}: {}'.format(rpm['From repo'], rpms_in_matching_repo))
+ error_str = 'RPM "{}" is not available in configured repo: {}, ' \
+ 'RPMs with correct name: {}'.format(str(rpm), rpm['From repo'],
+ rpms_with_matching_name)
+ raise Exception(error_str)
+ else:
+ none_repos = self._get_nonerepos()
+ for repo in [r['name'] for r in none_repos]:
+ for a_rpm in available_rpms[repo]:
+ if self._is_same_rpm(a_rpm, rpm):
+ return repo
+ msg = 'RPM "{}" is not available in any configured "none*" repos: {}'.format(
+ rpm['Name'], none_repos)
+ raise Exception(msg)
+
+ def _get_nonerepos(self):
+ return [repo for repo in self.repos if re.match(r'^none\d+$', repo['name'])]
+
+ @staticmethod
+ def _is_same_rpm(rpm1, rpm2):
+ return rpm1['Name'] == rpm2['Name'] and \
+ rpm1['Version'] == rpm2['Version'] and \
+ rpm1['Release'] == rpm2['Release'] and \
+ rpm1['Arch'] == rpm2['Architecture']
+
+
+def parse(args):
+ p = argparse.ArgumentParser(
+ description='Generate package info',
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+ p.add_argument('--verbose', '-v', action='store_true',
+ help='More verbose logging')
+ p.add_argument('--yum-info-path', required=True,
+ help='"yum info all" output as file')
+ p.add_argument('--rpm-info-path', required=True,
+ help='"rpm -qai" output as file')
+ p.add_argument('--crypto-info-path',
+ help='Dir from where to find ECC file')
+ p.add_argument('--boms-path',
+ help='Dir from where to find RPM bill of material files')
+ p.add_argument('--output-rpmlist',
+ help='output as rpm list like "rpm-qa"')
+ p.add_argument('--output-json',
+ help='output json file path')
+ p.add_argument('--output-csv',
+ help='output csv file path')
+ p.add_argument('--output-ms-csv',
+ help='output Microsoft Excel compatible csv file path')
+ p.add_argument('--build-config-path', required=True,
+ help='Build configuration ini path')
+ p.add_argument('--remote', action='store_true',
+ help='Read localrepo from remote defined by BUILD_URL, '
+ 'otherwise use localrepo from WORKSPACE')
+ args = p.parse_args(args)
+ return args
+
+
+def read_files(boms_dir):
+ boms = {}
+ for f in os.listdir(boms_dir):
+ boms[f] = read_json(boms_dir + '/' + f)
+ return boms
+
+
+def main(input_args):
+ args = parse(input_args)
+ if args.verbose:
+ set_logging(debug=True, timestamps=True)
+ else:
+ set_logging(debug=False)
+ rpmdata = RpmDataBuilder(args.build_config_path,
+ read_from(args.yum_info_path),
+ read_from(args.rpm_info_path),
+ read_from(args.crypto_info_path),
+ read_files(args.boms_path),
+ remote=args.remote).run()
+ if args.output_rpmlist:
+ write_to(args.output_rpmlist, '\n'.join(sorted([str(rpm) for rpm in rpmdata])))
+ if args.output_json:
+ write_to(args.output_json, to_json(rpmdata))
+ csv = CsvConverter(rpmdata, preferred_field_order=['Name', 'Version', 'Release',
+ 'License', 'Vendor', 'From repo',
+ 'Source RPM'])
+ if args.output_csv:
+ write_to(args.output_csv, str(csv))
+ if args.output_ms_csv:
+ write_to(args.output_ms_csv,
+ csv.convert_to_ms_excel(text_fields=['Version', 'Size', 'Release']))
+ if not args.output_json and not args.output_csv:
+ print(rpmdata)
+
+
+if __name__ == "__main__":
+ main(sys.argv[1:])
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=invalid-name
+import os
+import sys
+import logging
+import pytest
+import mock
+
+from tools.script.create_rpm_data import RpmDataBuilder
+from tools.repository import BuildConfigParser
+from tools.executor import Result
+import tools.repository
+from tools.script.create_rpm_data_test_data import \
+ yum_installed_output, \
+ rpm_info_output, \
+ expected_output, \
+ basesystem_combined, \
+ cpp_combined, \
+ centos_logos_combined, \
+ dejavu_fonts_common_combined, yum_available_output, \
+ crypto_rpms_json, boms_output
+
+from tools.test_data_rpm import basesystem_rpm_info, cpp_rpm_info, centos_logos_rpm_info, \
+ dejavu_fonts_common_rpm_info
+from tools.test_data_yum import basesystem_yum_info, cpp_yum_info, centos_logos_yum_info, \
+ dejavu_fonts_common_yum_info
+from tools.yum import YumConfig
+
+
+@mock.patch.object(YumConfig, 'write')
+@mock.patch.object(tools.yum, 'run')
+@mock.patch.object(BuildConfigParser, 'items')
+def test_complete_parse(mock_config, mock_reporeader, _):
+ logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+ mock_config.side_effect = [
+ [('base', 'test-url-for-base'),
+ ('none1', 'test-url-for-none1'),
+ ('none2', 'test-url-for-none2')],
+ [('purkki-3rdparty', 'test-url-for-purkki-3rdparty#test-extra-option=test-value')],
+ [('base', 'src-test-url-for-base'),
+ ('none1', 'src-test-url-for-none1'),
+ ('none2', 'src-test-url-for-none2')],
+ [('purkki-3rdparty', 'src-test-url-for-purkki-3rdparty#test-extra-option2=test-value2')]
+ ]
+ mock_reporeader.side_effect = [Result(0, '', ''), # yum clean all
+ Result(0, '', ''), # rm -rf /var/yum/cache
+ Result(0, yum_available_output, '')]
+ os.environ['BUILD_URL'] = 'test-url/'
+ os.environ['WORKSPACE'] = '/foo/path'
+ result = RpmDataBuilder('fake_build_config_path',
+ yum_installed_output,
+ rpm_info_output,
+ crypto_rpms_json,
+ boms_output).run()
+ assert mock_reporeader.call_count == 3
+ assert result == expected_output
+
+
+@pytest.mark.parametrize('yum_info, rpm_info, expected_combined', [
+ (basesystem_yum_info, basesystem_rpm_info, basesystem_combined),
+ (cpp_yum_info, cpp_rpm_info, cpp_combined),
+ (centos_logos_yum_info, centos_logos_rpm_info, centos_logos_combined),
+ (dejavu_fonts_common_yum_info, dejavu_fonts_common_rpm_info, dejavu_fonts_common_combined),
+])
+def test_combine_rpm_data(yum_info, rpm_info, expected_combined):
+ result = RpmDataBuilder('fake_build_config_path',
+ yum_info,
+ rpm_info,
+ '[]',
+ {}).read_installed_rpms()
+ assert result[0] == expected_combined
--- /dev/null
+# -*- coding: utf-8 -*-
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=invalid-name
+from tools.rpm_test_data import cpp_expected
+
+yum_installed_output = """Loaded plugins: fastestmirror, priorities
+Loading mirror speeds from cached hostfile
+Installed Packages
+Name : non-repo-pkg-1
+Arch : x86_64
+Version : 1
+Release : 1
+Repo : installed
+
+Name : non-repo-pkg-2
+Arch : noarch
+Version : 2
+Release : 2
+Repo : installed
+Obsoletes : (none)
+
+Name : base-image-pkg
+Arch : x86_64
+Version : 3
+Release : 3
+Repo : installed
+From repo : base
+
+Name : internal-pkg
+Arch : noarch
+Version : 4
+Release : 4
+Repo : installed
+From repo : localrepo
+
+Name : 3rdparty-pkg
+Arch : x86_64
+Version : 5
+Release : 5
+Repo : installed
+From repo : purkki-3rdparty
+Obsoletes : 2ndparty-pkg
+
+"""
+
+rpm_info_output = """Name : non-repo-pkg-1
+Version : 1
+Release : 1
+Architecture: x86_64
+Source RPM : non-repo-pkg-1-1-1.src.rpm
+Name : non-repo-pkg-2
+Version : 2
+Release : 2
+Architecture: noarch
+Source RPM : non-repo-pkg-2-2-2.src.rpm
+Name : base-image-pkg
+Version : 3
+Release : 3
+Architecture: x86_64
+Source RPM : base-image-pkg-3-3.src.rpm
+Name : internal-pkg
+Version : 4
+Release : 4
+Architecture: noarch
+Source RPM : internal-pkg-4-4.src.rpm
+Name : 3rdparty-pkg
+Version : 5
+Release : 5
+Architecture: x86_64
+Source RPM : 3rdparty-pkg-5-5.src.rpm
+""" # noqa, PEP-8 disabled because of example output has trailing spaces
+
+yum_available_output_header = """Added tmprepo repo from http://url1/
+Available Packages
+"""
+
+yum_available_output_base = """
+Name : base-image-pkg
+Arch : x86_64
+Epoch : 0
+Version : 3
+Release : 3
+Size : 195 k
+Repo : base
+"""
+
+yum_available_output_none2 = """
+Name : non-repo-pkg-1
+Arch : x86_64
+Epoch : 0
+Version : 1
+Release : 1
+Size : 195 k
+Repo : none2
+"""
+
+yum_available_output_none1 = """
+Name : non-repo-pkg-2
+Arch : noarch
+Version : 2
+Release : 2
+Size : 195 k
+Repo : none1
+"""
+
+yum_available_output_localrepo = """
+Name : internal-pkg
+Arch : x86_64
+Epoch : 0
+Version : 4
+Release : 4
+Size : 195 k
+Repo : localrepo
+"""
+
+yum_available_output_purkki_3rdparty = """
+Name : 3rdparty-pkg
+Arch : x86_64
+Epoch : 0
+Version : 5
+Release : 5
+Size : 195 k
+Repo : purkki-3rdparty
+"""
+
+yum_available_output = yum_available_output_header + yum_available_output_base + \
+ yum_available_output_none1 + yum_available_output_none2 + \
+ yum_available_output_localrepo + yum_available_output_purkki_3rdparty
+
+internal_pkg_bom = [{'name': '@types/d3-axis',
+ 'version': '1.0.10',
+ 'foss': 'yes',
+ 'source-url': 'http://some.url/1',
+ 'crypto-capable': True},
+ {'name': '@types/d3-array@*',
+ 'version': '1.2.1',
+ 'foss': 'Yes',
+ 'source-url': 'http://some.url/2'}]
+boms_output = {'internal-pkg-4-4.noarch': {"bom": internal_pkg_bom}}
+
+expected_output = [
+ {
+ 'Name': 'non-repo-pkg-1',
+ 'Architecture': 'x86_64',
+ 'Version': '1',
+ 'Release': '1',
+ 'Repo': 'installed',
+ 'Repo data': {'baseurl': 'test-url-for-none2', 'name': 'none2'},
+ 'Obsoletes': 'N/A',
+ 'Source RPM': 'non-repo-pkg-1-1-1.src.rpm',
+ 'Crypto capable': False,
+ 'BOM': '',
+ }, {
+ 'Name': 'non-repo-pkg-2',
+ 'Architecture': 'noarch',
+ 'Version': '2',
+ 'Release': '2',
+ 'Repo': 'installed',
+ 'Repo data': {'baseurl': 'test-url-for-none1', 'name': 'none1'},
+ 'Obsoletes': 'N/A',
+ 'Source RPM': 'non-repo-pkg-2-2-2.src.rpm',
+ 'Crypto capable': False,
+ 'BOM': '',
+ }, {
+ 'Name': 'base-image-pkg',
+ 'Architecture': 'x86_64',
+ 'Version': '3',
+ 'Release': '3',
+ 'Repo': 'installed',
+ 'From repo': 'base',
+ 'Repo data': {'baseurl': 'test-url-for-base', 'name': 'base'},
+ 'Obsoletes': 'N/A',
+ 'Source RPM': 'base-image-pkg-3-3.src.rpm',
+ 'Crypto capable': False,
+ 'BOM': '',
+ }, {
+ 'Name': 'internal-pkg',
+ 'Architecture': 'noarch',
+ 'Version': '4',
+ 'Release': '4',
+ 'Repo': 'installed',
+ 'From repo': 'localrepo',
+ 'Repo data': {'baseurl': 'test-url/artifact/results/repo',
+ 'name': 'localrepo'},
+ 'Obsoletes': 'N/A',
+ 'Source RPM': 'internal-pkg-4-4.src.rpm',
+ 'Crypto capable': True,
+ 'BOM': internal_pkg_bom,
+ }, {
+ 'Name': '3rdparty-pkg',
+ 'Architecture': 'x86_64',
+ 'Version': '5',
+ 'Release': '5',
+ 'Repo': 'installed',
+ 'From repo': 'purkki-3rdparty',
+ 'Repo data': {'baseurl': 'test-url-for-purkki-3rdparty', 'name': 'purkki-3rdparty',
+ 'test-extra-option': 'test-value'},
+ 'Obsoletes': '2ndparty-pkg',
+ 'Source RPM': '3rdparty-pkg-5-5.src.rpm',
+ 'Crypto capable': False,
+ 'BOM': '',
+ }]
+
+basesystem_combined = {
+ # From RPM info
+ 'Name': 'basesystem',
+ 'Version': '10.0',
+ 'Release': '7.el7.centos',
+ 'Architecture': 'noarch',
+ 'Install Date': 'Fri 01 Apr 2016 11:47:25 AM EEST',
+ 'Group': 'System Environment/Base',
+ 'Size': '0',
+ 'License': 'Public Domain',
+ 'Signature': 'RSA/SHA256, Fri 04 Jul 2014 03:46:57 AM EEST, Key ID 24c6a8a7f4a80eb5',
+ 'Source RPM': 'basesystem-10.0-7.el7.centos.src.rpm',
+ 'Build Date': 'Fri 27 Jun 2014 01:37:10 PM EEST',
+ 'Build Host': 'worker1.bsys.centos.org',
+ 'Relocations': '(not relocatable)',
+ 'Packager': 'CentOS BuildSystem <http://bugs.centos.org>',
+ 'Vendor': 'CentOS',
+ 'Summary': 'The skeleton package which defines a simple CentOS Linux system',
+ 'Description': '\n'.join(
+ ['Basesystem defines the components of a basic CentOS Linux',
+ 'system (for example, the package installation order to use during',
+ 'bootstrapping). Basesystem should be in every installation of a system,',
+ 'and it should never be removed.']),
+ # From yum info
+ 'Repo': 'installed',
+}
+
+centos_logos_combined = {
+ 'Name': 'centos-logos',
+ 'Version': '70.0.6',
+ 'Release': '3.el7.centos',
+ 'Architecture': 'noarch',
+ 'License': u'Copyright © 2014 The CentOS Project. All rights reserved.',
+}
+
+cpp_combined = cpp_expected.copy()
+cpp_combined.update({'Repo': 'installed',
+ 'From repo': 'purkki-centos-base'})
+
+dejavu_fonts_common_combined = {
+ 'Name': 'dejavu-fonts-common',
+ 'Version': '2.33',
+ 'Release': '6.el7',
+ 'Architecture': 'noarch',
+ 'Install Date': 'Wed Feb 7 13:49:27 2018',
+ 'Group': 'User Interface/X',
+ 'Size': '130455',
+ 'License': 'Bitstream Vera and Public Domain',
+ 'Signature': 'RSA/SHA256, Fri Jul 4 01:06:50 2014, Key ID 24c6a8a7f4a80eb5',
+ 'Source RPM': 'dejavu-fonts-2.33-6.el7.src.rpm',
+ 'Build Date': 'Mon Jun 9 21:34:30 2014',
+ 'Build Host': 'worker1.bsys.centos.org',
+ 'Relocations': '(not relocatable)',
+ 'Packager': 'CentOS BuildSystem <http://bugs.centos.org>',
+ 'Vendor': 'CentOS',
+ 'URL': 'http://dejavu-fonts.org/',
+ 'Summary': 'Common files for the Dejavu font set',
+ 'Description': '\n'.join(
+ ['The DejaVu font set is based on the “Bitstream Vera” fonts, release 1.10. Its',
+ 'purpose is to provide a wider range of characters, while maintaining the',
+ 'original style, using an open collaborative development process.',
+ '',
+ 'This package consists of files used by other DejaVu packages.']),
+ 'Repo': 'installed',
+ 'From repo': 'purkki-centos-base'
+}
+
+crypto_rpms_json = """
+[
+ {
+ "name": "internal-pkg-4-4.noarch",
+ "requires": [
+ "libgssapi_krb5.so.2()(64bit)",
+ "libk5crypto.so.3()(64bit)",
+ "libkrb5.so.3()(64bit)",
+ "libcrypto.so.10()(64bit)",
+ "libcrypto.so.10(OPENSSL_1.0.1_EC)(64bit)",
+ "libcrypto.so.10(OPENSSL_1.0.2)(64bit)",
+ "libcrypto.so.10(libcrypto.so.10)(64bit)",
+ "libssl.so.10()(64bit)",
+ "libssl.so.10(libssl.so.10)(64bit)",
+ "openssl-libs(x86-64)",
+ "rtld(GNU_HASH)"
+ ]
+ }
+]
+"""
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import argparse
+import logging
+
+from tools.buildconfig import BuildConfigParser
+
+
+def _parse(args):
+ parser = argparse.ArgumentParser(description='Generate repo files from config ini sections')
+ parser.add_argument('config_sections', metavar='config_section', nargs='+',
+ help='Config ini section')
+ parser.add_argument('--output-dir', '-d', required=True,
+ help='Directory to output the repo files')
+ parser.add_argument('--config-ini', '-c', required=True,
+ help='Path to the config ini to read')
+ args = parser.parse_args(args)
+ return args
+
+
+class RepoGen(object):
+
+ def __init__(self, output_dir, config_ini, config_sections):
+ self.output_dir = output_dir
+ self.config_sections = config_sections
+ self.config = BuildConfigParser(ini_file=config_ini)
+
+ def run(self):
+ for section in self.config_sections:
+ repo_file_path = os.path.join(self.output_dir, section + '.repo')
+ self._write_repo_file(section, repo_file_path)
+
+ def _write_repo_file(self, section, repo_file_path):
+ with open(repo_file_path, 'w') as f:
+ for repo in self.config.items(section):
+ name = repo[0]
+ parts = repo[1].split('#')
+ url = parts[0]
+ f.write('[%s]\n' % name)
+ f.write('name=%s\n' % name)
+ f.write('baseurl=%s\n' % url)
+ f.write('enabled=1\n')
+ f.write('gpgcheck=0\n')
+ for part in parts[1:]:
+ f.write('%s\n' % part)
+ f.write('\n')
+ if os.path.getsize(repo_file_path) == 0:
+ logging.error('Zero size output: {}'.format(repo_file_path))
+ sys.exit(1)
+ logging.info('Wrote repo: {}'.format(repo_file_path))
+
+
+def main(input_args):
+ logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+ args = _parse(input_args)
+ RepoGen(args.output_dir, args.config_ini, args.config_sections).run()
+
+
+if __name__ == "__main__":
+ main(sys.argv[1:])
--- /dev/null
+#!/usr/bin/env python
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import sys
+
+from tools.convert import to_json, CsvConverter
+from tools.log import set_logging
+from tools.io import write_to, read_json
+
+
+class RpmDataProcesser(object):
+ def __init__(self, rpmdata):
+ self.components = self._get_components(rpmdata)
+
+ @staticmethod
+ def _get_components(rpmdata):
+ components = []
+ bom_field_map = {'name': 'Name',
+ 'version': 'Version',
+ 'foss': 'FOSS',
+ 'source-url': 'Source URL',
+ 'crypto-capable': 'Crypto capable'}
+ for rpm in rpmdata:
+ components.append(unicode_recursively(rpm))
+ bom = rpm['BOM']
+ if bom:
+ for material in bom:
+ component = {'Source RPM': rpm['Source RPM']}
+ for field in material:
+ component[bom_field_map[field]] = material[field]
+ components.append(unicode_recursively(component))
+ return components
+
+ def gen_components(self, path):
+ write_to(path, to_json(self.components))
+
+ def gen_components_csv(self, path):
+ csv = CsvConverter(self.components,
+ preferred_field_order=['Name', 'Version', 'Release', 'Source RPM'])
+ write_to(path, csv.convert_to_ms_excel(text_fields=['Version']))
+
+
+def unicode_recursively(something):
+ if isinstance(something, dict):
+ return {unicode_recursively(key): unicode_recursively(value) for key, value in
+ something.iteritems()}
+ elif isinstance(something, list):
+ return [unicode_recursively(element) for element in something]
+ elif isinstance(something, unicode):
+ return something.encode('utf-8')
+ return something
+
+
+def parse(args):
+ p = argparse.ArgumentParser(
+ description='Process rpmdata for multitude of tools',
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+ p.add_argument('--verbose', '-v', action='store_true',
+ help='More verbose logging')
+ p.add_argument('--rpmdata-path', required=True,
+ help='RPM data json file path')
+ p.add_argument('--output-components',
+ help='Component list that includes also RPM sub-components')
+ p.add_argument('--output-components-csv',
+ help='Component list that includes also RPM sub-components as CSV')
+ args = p.parse_args(args)
+ return args
+
+
+def main(input_args):
+ args = parse(input_args)
+ if args.verbose:
+ set_logging(debug=True)
+ else:
+ set_logging(debug=False)
+ p = RpmDataProcesser(read_json(args.rpmdata_path))
+ if args.output_components:
+ p.gen_components(args.output_components)
+ if args.output_components_csv:
+ p.gen_components_csv(args.output_components_csv)
+
+
+if __name__ == "__main__":
+ main(sys.argv[1:])
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+from tools.convert import CsvConverter
+from tools.script.create_rpm_data_test_data import expected_output
+from tools.script.process_rpmdata import main
+
+
+def gen_input(tmpdir, rpmdata):
+ p = tmpdir.join('rpmdata.json')
+ p.write(rpmdata)
+ return str(p)
+
+
+def test_no_output(tmpdir):
+ main(['--rpmdata-path', gen_input(tmpdir, {})])
+
+
+def test_components(tmpdir):
+ input_content = [expected_output[0],
+ expected_output[3]]
+ input_ = json.dumps(input_content)
+ output_json = tmpdir.join('components.json')
+ output_csv = tmpdir.join('components.csv')
+ main(['--rpmdata-path', gen_input(tmpdir, input_),
+ '--output-components', str(output_json),
+ '--output-components-csv', str(output_csv)])
+ additions = [{'FOSS': u'yes',
+ 'Source RPM': 'internal-pkg-4-4.src.rpm',
+ 'Name': '@types/d3-axis',
+ 'Source URL': 'http://some.url/1',
+ 'Version': '1.0.10',
+ 'Crypto capable': True},
+ {'FOSS': u'Yes',
+ 'Source RPM': 'internal-pkg-4-4.src.rpm',
+ 'Name': '@types/d3-array@*',
+ 'Source URL': 'http://some.url/2',
+ 'Version': '1.2.1'}]
+ assert json.loads(output_json.read()) == input_content + additions
+ assert output_csv.read() == _gen_components_csv(input_content + additions)
+
+
+def _gen_components_csv(_list):
+ csv = CsvConverter(_list, preferred_field_order=['Name', 'Version', 'Release', 'Source RPM'])
+ return csv.convert_to_ms_excel(text_fields=['Version'])
--- /dev/null
+#!/usr/bin/env python
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+from tools.buildconfig import BuildConfigParser
+
+
+def main():
+ config = BuildConfigParser(sys.argv[1])
+ if len(sys.argv) == 3:
+ print(config.items(sys.argv[2]))
+ elif len(sys.argv) == 4:
+ print(config.get(sys.argv[2], sys.argv[3]))
+ else:
+ raise Exception('Invalid parameter count: {}'.format(len(sys.argv)))
+
+
+if __name__ == "__main__":
+ main()
--- /dev/null
+#!/usr/bin/env python
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import sys
+
+from tools.package import PackageConfigReader
+
+
+def parse(args):
+ p = argparse.ArgumentParser(
+ description='Read package configuration',
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+ p.add_argument('--config', required=False, help='Package YAML config file path')
+ p.add_argument('--separator', required=False, default=' ',
+ help='Separator for the resulting stdout list')
+ p.add_argument('package_operation_type', choices=['install', 'uninstall'],
+ help='list packages by operation type')
+ args = p.parse_args(args)
+ return args
+
+
+def main(input_args):
+ args = parse(input_args)
+ kwargs = {}
+ if args.config is not None:
+ kwargs['config_file'] = args.config
+ print(args.separator.join(PackageConfigReader(**kwargs).get(args.package_operation_type)))
+
+
+if __name__ == "__main__":
+ main(sys.argv[1:])
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+WORK_ROOT = os.path.dirname(
+ os.path.dirname(
+ os.path.dirname(
+ os.path.dirname(
+ os.path.realpath(__file__)))))
+
+MAINFEST_PATH = os.path.join(WORK_ROOT, '.repo/manifests')
+
+BUILD_CONFIG_PATH = os.path.join(MAINFEST_PATH, 'build_config.ini')
+PACKAGES_CONFIG_PATH = os.path.join(MAINFEST_PATH, 'packages.yaml')
--- /dev/null
+# -*- coding: utf-8 -*-
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=invalid-name,line-too-long
+
+bash_rpm_info = """Name : bash
+Version : 4.2.46
+Release : 21.el7_3
+Architecture: x86_64
+Install Date: Thu 11 Jan 2018 12:32:51 PM EET
+Group : System Environment/Shells
+Size : 3663714
+License : GPLv3+
+Signature : RSA/SHA256, Wed 07 Dec 2016 02:11:28 AM EET, Key ID 24c6a8a7f4a80eb5
+Source RPM : bash-4.2.46-21.el7_3.src.rpm
+Build Date : Wed 07 Dec 2016 01:21:54 AM EET
+Build Host : c1bm.rdu2.centos.org
+Relocations : (not relocatable)
+Packager : CentOS BuildSystem <http://bugs.centos.org>
+Vendor : CentOS
+URL : http://www.gnu.org/software/bash
+Summary : The GNU Bourne Again shell
+Description :
+The GNU Bourne Again shell (Bash) is a shell or command language
+interpreter that is compatible with the Bourne shell (sh). Bash
+incorporates useful features from the Korn shell (ksh) and the C shell
+(csh). Most sh scripts can be run by bash without modification.
+"""
+
+basesystem_rpm_info = """Name : basesystem
+Version : 10.0
+Release : 7.el7.centos
+Architecture: noarch
+Install Date: Fri 01 Apr 2016 11:47:25 AM EEST
+Group : System Environment/Base
+Size : 0
+License : Public Domain
+Signature : RSA/SHA256, Fri 04 Jul 2014 03:46:57 AM EEST, Key ID 24c6a8a7f4a80eb5
+Source RPM : basesystem-10.0-7.el7.centos.src.rpm
+Build Date : Fri 27 Jun 2014 01:37:10 PM EEST
+Build Host : worker1.bsys.centos.org
+Relocations : (not relocatable)
+Packager : CentOS BuildSystem <http://bugs.centos.org>
+Vendor : CentOS
+Summary : The skeleton package which defines a simple CentOS Linux system
+Description :
+Basesystem defines the components of a basic CentOS Linux
+system (for example, the package installation order to use during
+bootstrapping). Basesystem should be in every installation of a system,
+and it should never be removed.
+"""
+
+centos_logos_rpm_info = u"""Name : centos-logos
+Version : 70.0.6
+Release : 3.el7.centos
+Architecture: noarch
+License : Copyright © 2014 The CentOS Project. All rights reserved.
+
+"""
+
+conntrack_tools_rpm_info = """Name : conntrack-tools
+Version : 1.4.4
+Release : 3.el7_3
+Architecture: x86_64
+Install Date: Thu 11 Jan 2018 12:39:20 PM EET
+Group : System Environment/Base
+Size : 562826
+License : GPLv2
+Signature : RSA/SHA256, Thu 29 Jun 2017 03:36:05 PM EEST, Key ID 24c6a8a7f4a80eb5
+Source RPM : conntrack-tools-1.4.4-3.el7_3.src.rpm
+Build Date : Thu 29 Jun 2017 03:18:42 AM EEST
+Build Host : c1bm.rdu2.centos.org
+Relocations : (not relocatable)
+Packager : CentOS BuildSystem <http://bugs.centos.org>
+Vendor : CentOS
+URL : http://netfilter.org
+Summary : Manipulate netfilter connection tracking table and run High Availability
+Description :
+With conntrack-tools you can setup a High Availability cluster and
+synchronize conntrack state between multiple firewalls.
+
+The conntrack-tools package contains two programs:
+- conntrack: the command line interface to interact with the connection
+ tracking system.
+- conntrackd: the connection tracking userspace daemon that can be used to
+ deploy highly available GNU/Linux firewalls and collect
+ statistics of the firewall use.
+
+conntrack is used to search, list, inspect and maintain the netfilter
+connection tracking subsystem of the Linux kernel.
+Using conntrack, you can dump a list of all (or a filtered selection of)
+currently tracked connections, delete connections from the state table,
+and even add new ones.
+In addition, you can also monitor connection tracking events, e.g.
+show an event message (one line) per newly established connection.
+"""
+
+cpp_rpm_info = """Name : cpp
+Version : 4.8.5
+Release : 11.el7
+Architecture: x86_64
+Install Date: Thu 11 Jan 2018 12:37:55 PM EET
+Group : Development/Languages
+Size : 15632501
+License : GPLv3+ and GPLv3+ with exceptions and GPLv2+ with exceptions and LGPLv2+ and BSD
+Signature : RSA/SHA256, Sun 20 Nov 2016 07:27:00 PM EET, Key ID 24c6a8a7f4a80eb5
+Source RPM : gcc-4.8.5-11.el7.src.rpm
+Build Date : Fri 04 Nov 2016 06:01:22 PM EET
+Build Host : worker1.bsys.centos.org
+Relocations : (not relocatable)
+Packager : CentOS BuildSystem <http://bugs.centos.org>
+Vendor : CentOS
+URL : http://gcc.gnu.org
+Summary : The C Preprocessor
+Description :
+Cpp is the GNU C-Compatible Compiler Preprocessor.
+Cpp is a macro processor which is used automatically
+by the C compiler to transform your program before actual
+compilation. It is called a macro processor because it allows
+you to define macros, abbreviations for longer
+constructs.
+
+The C preprocessor provides four separate functionalities: the
+inclusion of header files (files of declarations that can be
+substituted into your program); macro expansion (you can define macros,
+and the C preprocessor will replace the macros with their definitions
+throughout the program); conditional compilation (using special
+preprocessing directives, you can include or exclude parts of the
+program according to various conditions); and line control (if you use
+a program to combine or rearrange source files into an intermediate
+file which is then compiled, you can use line control to inform the
+compiler about where each source line originated).
+
+You should install this package if you are a C programmer and you use
+macros.
+""" # noqa, PEP-8 disabled because of example output has trailing spaces
+
+dejavu_fonts_common_rpm_info = """Name : dejavu-fonts-common
+Version : 2.33
+Release : 6.el7
+Architecture: noarch
+Install Date: Wed Feb 7 13:49:27 2018
+Group : User Interface/X
+Size : 130455
+License : Bitstream Vera and Public Domain
+Signature : RSA/SHA256, Fri Jul 4 01:06:50 2014, Key ID 24c6a8a7f4a80eb5
+Source RPM : dejavu-fonts-2.33-6.el7.src.rpm
+Build Date : Mon Jun 9 21:34:30 2014
+Build Host : worker1.bsys.centos.org
+Relocations : (not relocatable)
+Packager : CentOS BuildSystem <http://bugs.centos.org>
+Vendor : CentOS
+URL : http://dejavu-fonts.org/
+Summary : Common files for the Dejavu font set
+Description :
+
+The DejaVu font set is based on the “Bitstream Vera” fonts, release 1.10. Its
+purpose is to provide a wider range of characters, while maintaining the
+original style, using an open collaborative development process.
+
+This package consists of files used by other DejaVu packages.
+"""
+
+usbredir_rpm_info = """Name : usbredir
+Version : 0.7.1
+Release : 1.el7
+Architecture: x86_64
+Install Date: Wed Feb 7 13:49:24 2018
+Group : System Environment/Libraries
+Size : 108319
+License : LGPLv2+
+Signature : RSA/SHA256, Sun Nov 20 20:56:49 2016, Key ID 24c6a8a7f4a80eb5
+Source RPM : usbredir-0.7.1-1.el7.src.rpm
+Build Date : Sat Nov 5 18:33:15 2016
+Build Host : worker1.bsys.centos.org
+Relocations : (not relocatable)
+Packager : CentOS BuildSystem <http://bugs.centos.org>
+Vendor : CentOS
+URL : http://spice-space.org/page/UsbRedir
+Summary : USB network redirection protocol libraries
+Description :
+The usbredir libraries allow USB devices to be used on remote and/or virtual
+hosts over TCP. The following libraries are provided:
+
+usbredirparser:
+A library containing the parser for the usbredir protocol
+
+usbredirhost:
+A library implementing the USB host side of a usbredir connection.
+All that an application wishing to implement a USB host needs to do is:
+* Provide a libusb device handle for the device
+* Provide write and read callbacks for the actual transport of usbredir data
+* Monitor for usbredir and libusb read/write events and call their handlers
+"""
+
+perl_compress_rpm_info = """Name : perl-Compress-Raw-Zlib
+Epoch : 1
+Version : 2.061
+Release : 4.el7
+Architecture: x86_64
+Install Date: Sat Jan 26 20:05:50 2019
+Group : Development/Libraries
+Size : 139803
+License : GPL+ or Artistic
+Signature : RSA/SHA256, Fri Jul 4 04:15:33 2014, Key ID 24c6a8a7f4a80eb5
+Source RPM : perl-Compress-Raw-Zlib-2.061-4.el7.src.rpm
+Build Date : Tue Jun 10 01:12:08 2014
+Build Host : worker1.bsys.centos.org
+Relocations : (not relocatable)
+Packager : CentOS BuildSystem <http://bugs.centos.org>
+Vendor : CentOS
+URL : http://search.cpan.org/dist/Compress-Raw-Zlib/
+Summary : Low-level interface to the zlib compression library
+Description :
+The Compress::Raw::Zlib module provides a Perl interface to the zlib
+compression library, which is used by IO::Compress::Zlib.
+Obsoletes :
+""" # noqa, PEP-8 disabled because of example output has trailing spaces
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=invalid-name
+
+
+basesystem_yum_info = """Loaded plugins: fastestmirror, priorities
+Loading mirror speeds from cached hostfile
+Installed Packages
+Name : basesystem
+Arch : noarch
+Version : 10.0
+Release : 7.el7.centos
+Size : 0.0
+Repo : installed
+Summary : The skeleton package which defines a simple CentOS Linux system
+License : Public Domain
+Description : Basesystem defines the components of a basic CentOS Linux
+ : system (for example, the package installation order to use during
+ : bootstrapping). Basesystem should be in every installation of a
+ : system, and it should never be removed.
+
+""" # noqa, PEP-8 disabled because of example output has trailing spaces
+
+bash_yum_info = """Name : bash
+Arch : x86_64
+Version : 4.2.46
+Release : 21.el7_3
+Size : 3.5 M
+Repo : installed
+From repo : updates
+Summary : The GNU Bourne Again shell
+URL : http://www.gnu.org/software/bash
+License : GPLv3+
+Description : The GNU Bourne Again shell (Bash) is a shell or command language
+ : interpreter that is compatible with the Bourne shell (sh). Bash
+ : incorporates useful features from the Korn shell (ksh) and the C
+ : shell (csh). Most sh scripts can be run by bash without
+ : modification.
+"""
+
+centos_logos_yum_info = """
+Installed Packages
+Name : centos-logos
+Arch : noarch
+Version : 70.0.6
+Release : 3.el7.centos
+License : Copyright ? 2014 The CentOS Project. All rights reserved.
+
+"""
+
+conntrack_tools_yum_info = """Name : conntrack-tools
+Arch : x86_64
+Version : 1.4.4
+Release : 3.el7_3
+Size : 550 k
+Repo : installed
+From repo : centos-updates
+Summary : Manipulate netfilter connection tracking table and run High
+ : Availability
+URL : http://netfilter.org
+License : GPLv2
+Description : With conntrack-tools you can setup a High Availability cluster and
+ : synchronize conntrack state between multiple firewalls.
+ :
+ : The conntrack-tools package contains two programs:
+ : - conntrack: the command line interface to interact with the
+ : connection tracking system.
+ : - conntrackd: the connection tracking userspace daemon that can be
+ : used to deploy highly available GNU/Linux firewalls and collect
+ : statistics of the firewall use.
+ :
+ : conntrack is used to search, list, inspect and maintain the
+ : netfilter connection tracking subsystem of the Linux kernel.
+ : Using conntrack, you can dump a list of all (or a filtered
+ : selection of) currently tracked connections, delete connections
+ : from the state table, and even add new ones.
+ : In addition, you can also monitor connection tracking events, e.g.
+ : show an event message (one line) per newly established connection.
+""" # noqa, PEP-8 disabled because of example output has trailing spaces
+
+cpp_yum_info = """
+Installed Packages
+Name : cpp
+Arch : x86_64
+Version : 4.8.5
+Release : 11.el7
+Size : 15 M
+Repo : installed
+From repo : purkki-centos-base
+Summary : The C Preprocessor
+URL : http://gcc.gnu.org
+License : GPLv3+ and GPLv3+ with exceptions and GPLv2+ with exceptions and
+ : LGPLv2+ and BSD
+Description : Cpp is the GNU C-Compatible Compiler Preprocessor.
+ : Cpp is a macro processor which is used automatically
+ : by the C compiler to transform your program before actual
+ : compilation. It is called a macro processor because it allows
+ : you to define macros, abbreviations for longer
+ : constructs.
+ :
+ : The C preprocessor provides four separate functionalities: the
+ : inclusion of header files (files of declarations that can be
+ : substituted into your program); macro expansion (you can define
+ : macros, and the C preprocessor will replace the macros with their
+ : definitions throughout the program); conditional compilation
+ : (using special preprocessing directives, you can include or
+ : exclude parts of the program according to various conditions); and
+ : line control (if you use a program to combine or rearrange source
+ : files into an intermediate file which is then compiled, you can
+ : use line control to inform the compiler about where each source
+ : line originated).
+ :
+ : You should install this package if you are a C programmer and you
+ : use macros.
+
+""" # noqa, PEP-8 disabled because of example output has trailing spaces
+
+dejavu_fonts_common_yum_info = """
+Installed Packages
+Name : dejavu-fonts-common
+Arch : noarch
+Version : 2.33
+Release : 6.el7
+Size : 127 k
+Repo : installed
+From repo : purkki-centos-base
+Summary : Common files for the Dejavu font set
+URL : http://dejavu-fonts.org/
+License : Bitstream Vera and Public Domain
+Description :
+ : The DejaVu font set is based on the ?Bitstream Vera? fonts,
+ : release 1.10. Its purpose is to provide a wider range of
+ : characters, while maintaining the original style, using an open
+ : collaborative development process.
+ :
+ : This package consists of files used by other DejaVu packages.
+
+""" # noqa, PEP-8 disabled because of example output has trailing spaces
+
+pacemaker_yum_info = """
+Name : pacemaker
+Arch : x86_64
+Version : 1.1.15
+Release : 11.el7_3.5
+Size : 1.1 M
+Repo : installed
+From repo : purkki-centos-updates
+Summary : Scalable High-Availability cluster resource manager
+URL : http://www.clusterlabs.org
+License : GPLv2+ and LGPLv2+
+Description : Pacemaker is an advanced, scalable High-Availability cluster
+ : resource manager for Corosync, CMAN and/or Linux-HA.
+ :
+ : It supports more than 16 node clusters with significant
+ : capabilities for managing resources and dependencies.
+ :
+ : It will run scripts at initialization, when machines go up or
+ : down, when related resources fail and can be configured to
+ : periodically check resource health.
+ :
+ : Available rpmbuild rebuild options:
+ : --with(out) : cman stonithd doc coverage profiling pre_release
+ : hardening
+""" # noqa, PEP-8 disabled because of example output has trailing spaces
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+
+def apply_jenkins_auth(url):
+ validate_environ(['JENKINS_USERNAME', 'JENKINS_TOKEN'])
+ protocol, address = url.split('://')
+ url = '{protocol}://{user}:{token}@{address}'.format(**dict(protocol=protocol,
+ user=os.environ['JENKINS_USERNAME'],
+ token=os.environ['JENKINS_TOKEN'],
+ address=address))
+ return url
+
+
+def validate_environ(expected_vars):
+ for env in expected_vars:
+ if env not in os.environ:
+ raise Exception('{} must be defined in environment'.format(env))
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+import logging
+
+from tools.executor import run
+from tools.rpm import RpmData
+
+
+class YumRpm(RpmData):
+ @property
+ def arch(self):
+ return self['Arch']
+
+
+class YumInfoParserException(BaseException):
+ pass
+
+
+class YumDownloaderException(BaseException):
+ def __init__(self, msg, failed_packages):
+ super(YumDownloaderException, self).__init__()
+ self.msg = msg
+ self.failed_packages = failed_packages
+
+ def __str__(self):
+ return self.msg
+
+
+class YumInfoParser(object):
+ """
+ Parse 'yum info' output
+ """
+
+ def parse_file(self, yum_info_installed_file_path):
+ with open(yum_info_installed_file_path, 'r') as f:
+ return self.parse_installed(f.read())
+
+ def parse_installed(self, yum_info_installed):
+ return self._parse_rpms_with_regexp(yum_info_installed, r'\nInstalled Packages\n')
+
+ def parse_available(self, yum_info_installed):
+ return self._parse_rpms_with_regexp(yum_info_installed, r'Available Packages\n')
+
+ def _parse_rpms_with_regexp(self, yum_output, regexp):
+ parsed_output = self._split_yum_output_with(yum_output, regexp)
+ return [self.parse_package(pkg) for pkg in parsed_output[1].split('\n\n') if pkg]
+
+ @staticmethod
+ def _split_yum_output_with(output, regexp):
+ parsed_output = re.split(regexp, output)
+ if len(parsed_output) != 2:
+ raise YumInfoParserException(
+ '{} not found from output: {}'.format(repr(regexp), output[:1000]))
+ return parsed_output
+
+ @staticmethod
+ def parse_package(yum_info_output):
+ result = YumRpm()
+ current_key = None
+ for line in re.findall(r'^(.+?) : (.*)$', yum_info_output, re.MULTILINE):
+ parsed_key = line[0].strip()
+ parsed_value = line[1].rstrip(' ')
+ if parsed_key:
+ result[parsed_key] = parsed_value
+ current_key = parsed_key
+ elif current_key in ['License', 'Summary']:
+ result[current_key] = result[current_key] + ' ' + parsed_value
+ else:
+ result[current_key] = result[current_key] + '\n' + parsed_value
+ return result
+
+
+class Yum(object):
+ def __init__(self):
+ self.config = YumConfig(filename='tmp_yum.conf')
+
+ @classmethod
+ def clean_and_remove_cache(cls):
+ cls.clean_all()
+ cls.remove_cache_dir()
+
+ @classmethod
+ def clean_all(cls):
+ run(['yum', 'clean', 'all'], raise_on_stderr=False)
+
+ @classmethod
+ def remove_cache_dir(cls):
+ run(['rm', '-rf', '/var/cache/yum'], raise_on_stderr=True)
+
+ @classmethod
+ def read_available_pkgs(cls, name, url):
+ filename = 'tmp_yum.conf'
+ yum_tmp_conf = YumConfig()
+ yum_tmp_conf.add_repository(name, url)
+ with open(filename, 'w') as f:
+ f.write(yum_tmp_conf.render())
+ cmd = ['yum',
+ '--config={}'.format(filename),
+ '--showduplicates',
+ '--setopt=keepcache=0',
+ '--disablerepo=*',
+ '--enablerepo={}'.format(name),
+ 'info',
+ 'available']
+ return run(cmd, raise_on_stderr=False).stdout
+
+ def add_repo(self, name, url):
+ self.config.add_repository(name, url)
+
+ def read_all_packages(self):
+ self.config.write()
+ logging.debug('Yum config:\n{}'.format(self.config))
+ cmd = ['yum',
+ '--config={}'.format(self.config.filename),
+ '--showduplicates',
+ '--setopt=keepcache=0',
+ '--enablerepo=*',
+ 'info',
+ 'available']
+ return run(cmd, raise_on_stderr=False).stdout
+
+
+class YumDownloader(object):
+ def download(self, rpms, repositories, to_dir=None, source=False):
+ logging.debug('Downloading {} RPMs from repositories: {}'.format(len(rpms),
+ [r['name'] for r in
+ repositories]))
+ result = self._download(rpms, repositories, to_dir=to_dir, source=source)
+ downloaded_rpms = [rpm.rstrip('.rpm') for rpm in os.listdir(to_dir)]
+ not_downloaded_rpms = [rpm for rpm in rpms if rpm not in downloaded_rpms]
+ if len(rpms) != len(downloaded_rpms):
+ logging.debug('Downloaded {}/{} RPMs: {}'.format(len(downloaded_rpms), len(rpms),
+ downloaded_rpms))
+ # Not precise way to list not downloaded RPMs - RPM name may not match the metadata
+ # We should read "rpm -qip" for each rpm and parse it
+ raise YumDownloaderException(
+ 'Failed to download {}/{} RPMs: {}.\nYumdownloader result: {}'.format(
+ len(not_downloaded_rpms), len(rpms), not_downloaded_rpms, str(result)),
+ not_downloaded_rpms
+ )
+
+ @staticmethod
+ def _download(rpms, repositories, to_dir=None, source=False):
+ filename = 'tmp_yum.conf'
+ yum_tmp_conf = YumConfig()
+ for r in repositories:
+ yum_tmp_conf.add_repository(r['name'], r['baseurl'])
+ with open(filename, 'w') as f:
+ f.write(yum_tmp_conf.render())
+ logging.debug('Downloading RPMs: {}'.format(rpms))
+ cmd = ['yumdownloader']
+ if to_dir is not None:
+ cmd += ['--destdir={}'.format(to_dir)]
+ cmd += ['--config={}'.format(filename),
+ '--disablerepo=*',
+ '--enablerepo={}'.format(','.join([r['name'] for r in repositories])),
+ '--showduplicates']
+ if source:
+ cmd.append('--source')
+ cmd += rpms
+ return run(cmd, raise_on_stderr=False)
+
+
+class YumConfig(object):
+ def __init__(self, filename=None):
+ self.filename = filename
+ self.config = ['[main]',
+ '#cachedir=/var/cache/yum/$basearch/$releasever',
+ 'reposdir=/foo/bar/xyz',
+ 'keepcache=0',
+ 'debuglevel=2',
+ '#logfile=/var/log/yum.log',
+ 'exactarch=1',
+ 'obsoletes=1',
+ 'gpgcheck=0',
+ 'plugins=1',
+ 'installonly_limit=5',
+ 'override_install_langs=en_US.UTF-8',
+ 'tsflags=nodocs']
+ self.repositories = []
+
+ def add_repository(self, name, url, exclude=None):
+ repo = ['[{}]'.format(name),
+ 'name = ' + name,
+ 'baseurl = ' + url,
+ 'enabled = 1',
+ 'gpgcheck = 0']
+ if exclude is not None:
+ repo.append('exclude = ' + exclude)
+ self.repositories.append(repo)
+
+ def __str__(self):
+ return self.render()
+
+ def render(self):
+ blocks = ['\n'.join(self.config)] + ['\n'.join(repo) for repo in self.repositories]
+ return '\n\n'.join(blocks)
+
+ def write(self):
+ with open(self.filename, 'w') as f:
+ f.write(self.render())
--- /dev/null
+Loaded plugins: fastestmirror, priorities
+Installed Packages
+Name : GeoIP
+Arch : x86_64
+Version : 1.5.0
+Release : 11.el7
+Size : 2.8 M
+Repo : installed
+From repo : purkki-centos-base
+Summary : Library for country/city/organization to IP address or hostname
+ : mapping
+URL : http://www.maxmind.com/app/c
+License : LGPLv2+ and GPLv2+ and CC-BY-SA
+Description : GeoIP is a C library that enables the user to find the country
+ : that any IP address or hostname originates from. It uses a file
+ : based database that is accurate as of June 2007 and can optionally
+ : be updated on a weekly basis by installing the GeoIP-update
+ : package. This database simply contains IP blocks as keys, and
+ : countries as values. This database should be more complete and
+ : accurate than using reverse DNS lookups.
+ :
+ : This package includes GeoLite data created by MaxMind, available
+ : from http://www.maxmind.com/
+
+Name : MySQL-python
+Arch : x86_64
+Version : 1.2.5
+Release : 1.el7
+Size : 284 k
+Repo : installed
+From repo : purkki-centos-base
+Summary : An interface to MySQL
+URL : https://github.com/farcepest/MySQLdb1
+License : GPLv2+
+Description : Python interface to MySQL
+ :
+ : MySQLdb is an interface to the popular MySQL database server for
+ : Python. The design goals are:
+ :
+ : - Compliance with Python database API version 2.0
+ : - Thread-safety
+ : - Thread-friendliness (threads will not block each other)
+ : - Compatibility with MySQL 3.23 and up
+ :
+ : This module should be mostly compatible with an older interface
+ : written by Joe Skinner and others. However, the older version is
+ : a) not thread-friendly, b) written for MySQL 3.21, c) apparently
+ : not actively maintained. No code from that version is used in
+ : MySQLdb.
+
+Name : OpenIPMI-libs
+Arch : x86_64
+Version : 2.0.19
+Release : 15.el7
+Size : 1.7 M
+Repo : installed
+From repo : purkki-centos-base
+Summary : The OpenIPMI runtime libraries
+URL : http://sourceforge.net/projects/openipmi/
+License : LGPLv2+ and GPLv2+ or BSD
+Description : The OpenIPMI-libs package contains the runtime libraries for
+ : shared binaries and applications.
+
+Name : OpenIPMI-modalias
+Arch : x86_64
+Version : 2.0.19
+Release : 15.el7
+Size : 210
+Repo : installed
+From repo : purkki-centos-base
+Summary : Module aliases for IPMI subsystem
+URL : http://sourceforge.net/projects/openipmi/
+License : LGPLv2+ and GPLv2+ or BSD
+Description : The OpenIPMI-modalias provides configuration file with module
+ : aliases of ACPI and PNP wildcards.
+
+Name : PyPAM
+Arch : x86_64
+Version : 0.5.0
+Release : 19.el7
+Size : 51 k
+Repo : installed
+From repo : purkki-centos-base
+Summary : PAM bindings for Python
+URL : http://www.pangalactic.org/PyPAM
+License : LGPLv2
+Description : PAM (Pluggable Authentication Module) bindings for Python.
+
+Name : PyYAML
+Arch : x86_64
+Version : 3.10
+Release : 11.el7
+Size : 630 k
+Repo : installed
+Summary : YAML parser and emitter for Python
+URL : http://pyyaml.org/
+License : MIT
+Description : YAML is a data serialization format designed for human readability
+ : and interaction with scripting languages. PyYAML is a YAML parser
+ : and emitter for Python.
+ :
+ : PyYAML features a complete YAML 1.1 parser, Unicode support,
+ : pickle support, capable extension API, and sensible error
+ : messages. PyYAML supports standard YAML tags and provides
+ : Python-specific tags that allow to represent an arbitrary Python
+ : object.
+ :
+ : PyYAML is applicable for a broad range of tasks from complex
+ : configuration files to object serialization and persistance.
+
+Name : SDL
+Arch : x86_64
+Version : 1.2.15
+Release : 14.el7
+Size : 467 k
+Repo : installed
+From repo : purkki-centos-base
+Summary : A cross-platform multimedia library
+URL : http://www.libsdl.org/
+License : LGPLv2+
+Description : Simple DirectMedia Layer (SDL) is a cross-platform multimedia
+ : library designed to provide fast access to the graphics frame
+ : buffer and audio device.
+
+Name : XStatic-Angular-common
+Arch : noarch
+Epoch : 1
+Version : 1.5.8.0
+Release : 1.el7
+Size : 3.1 M
+Repo : installed
+From repo : purkki-delorean
+Summary : Common files for XStatic-Angular (XStatic packaging standard)
+URL : http://angularjs.org
+License : MIT
+Description : Common files for XStatic-Angular (XStatic packaging standard)
+
+Name : acl
+Arch : x86_64
+Version : 2.2.51
+Release : 12.el7
+Size : 196 k
+Repo : installed
+Summary : Access control list utilities
+URL : http://acl.bestbits.at/
+License : GPLv2+
+Description : This package contains the getfacl and setfacl utilities needed for
+ : manipulating access control lists.
+
+Name : adwaita-cursor-theme
+Arch : noarch
+Version : 3.14.1
+Release : 1.el7
+Size : 1.9 M
+Repo : installed
+From repo : purkki-centos-base
+Summary : Adwaita cursor theme
+URL : http://www.gnome.org
+License : LGPLv3+ or CC-BY-SA
+Description : The adwaita-cursor-theme package contains a modern set of cursors
+ : originally designed for the GNOME desktop.
+
+Name : adwaita-icon-theme
+Arch : noarch
+Version : 3.14.1
+Release : 1.el7
+Size : 13 M
+Repo : installed
+From repo : purkki-centos-base
+Summary : Adwaita icon theme
+URL : http://www.gnome.org
+License : LGPLv3+ or CC-BY-SA
+Description : This package contains the Adwaita icon theme used by the GNOME
+ : desktop.
+
+Name : agg
+Arch : x86_64
+Version : 2.5
+Release : 18.el7
+Size : 410 k
+Repo : installed
+From repo : purkki-centos-base
+Summary : Anti-Grain Geometry graphical rendering engine
+URL : http://www.antigrain.com
+License : GPLv2+
+Description : A High Quality Rendering Engine for C++.
+
+Name : alsa-lib
+Arch : x86_64
+Version : 1.1.1
+Release : 1.el7
+Size : 1.2 M
+Repo : installed
+From repo : purkki-centos-base
+Summary : The Advanced Linux Sound Architecture (ALSA) library
+URL : http://www.alsa-project.org/
+License : LGPLv2+
+Description : The Advanced Linux Sound Architecture (ALSA) provides audio and
+ : MIDI functionality to the Linux operating system.
+ :
+ : This package includes the ALSA runtime libraries to simplify
+ : application programming and provide higher level functionality as
+ : well as support for the older OSS API, providing binary
+ : compatibility for most OSS programs.
+
+Name : ansible
+Arch : noarch
+Version : 2.3.0.0
+Release : 3.el7
+Size : 27 M
+Repo : installed
+From repo : purkki-delorean
+Summary : SSH-based configuration management, deployment, and task execution
+ : system
+URL : http://ansible.com
+License : GPLv3+
+Description :
+ : Ansible is a radically simple model-driven configuration
+ : management, multi-node deployment, and remote task execution
+ : system. Ansible works over SSH and does not require any software
+ : or daemons to be installed on remote nodes. Extension modules can
+ : be written in any language and are transferred to managed machines
+ : automatically.
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=wildcard-import
+import os
+import pytest
+
+from tools.test_data_yum import bash_yum_info, \
+ conntrack_tools_yum_info, \
+ pacemaker_yum_info
+from tools.yum import YumInfoParser
+from tools.yum_test_data import bash_expected, pacemaker_expected, \
+ yum_info_installed_header, \
+ yum_info_available_header, \
+ yum_info_available_header2
+from tools.yum_test_data import conntrack_tools_expected
+
+
+@pytest.mark.parametrize('yum_info, expected_output', [
+ (bash_yum_info, bash_expected),
+ (conntrack_tools_yum_info, conntrack_tools_expected),
+ (pacemaker_yum_info, pacemaker_expected)
+])
+def test_parse_package(yum_info, expected_output):
+ parsed = YumInfoParser().parse_package(yum_info)
+ expected = expected_output
+ assert parsed == expected
+
+
+def test_parse_installed():
+ fake_out = '\n'.join([yum_info_installed_header,
+ bash_yum_info,
+ conntrack_tools_yum_info])
+ parsed = YumInfoParser().parse_installed(fake_out)
+ expected = [bash_expected, conntrack_tools_expected]
+ assert parsed == expected
+
+
+@pytest.mark.parametrize('available_header', [
+ yum_info_available_header,
+ yum_info_available_header2
+])
+def test_parse_available(available_header):
+ fake_out = '\n'.join([available_header,
+ bash_yum_info,
+ conntrack_tools_yum_info])
+ parsed = YumInfoParser().parse_available(fake_out)
+ expected = [bash_expected, conntrack_tools_expected]
+ assert parsed == expected
+
+
+def test_parse_file():
+ test_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+ 'yum_info_installed.sample')
+ parsed = YumInfoParser().parse_file(test_file)
+ assert len(parsed) == 14
--- /dev/null
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=invalid-name,line-too-long
+
+yum_info_installed_header = """Loaded plugins: fastestmirror, priorities
+Loading mirror speeds from cached hostfile
+Installed Packages
+"""
+
+yum_info_available_header = """Added tmprepo repo from http://purkki/mirror/centos/snapshot/20170705-2/7/os/x86_64/
+Available Packages
+""" # noqa
+
+yum_info_available_header2 = """Available Packages
+"""
+
+bash_expected = {
+ 'Name': 'bash',
+ 'Arch': 'x86_64',
+ 'Version': '4.2.46',
+ 'Release': '21.el7_3',
+ 'Size': '3.5 M',
+ 'Repo': 'installed',
+ 'From repo': 'updates',
+ 'Summary': 'The GNU Bourne Again shell',
+ 'URL': 'http://www.gnu.org/software/bash',
+ 'License': 'GPLv3+',
+ 'Description': '\n'.join(
+ ['The GNU Bourne Again shell (Bash) is a shell or command language',
+ 'interpreter that is compatible with the Bourne shell (sh). Bash',
+ 'incorporates useful features from the Korn shell (ksh) and the C',
+ 'shell (csh). Most sh scripts can be run by bash without',
+ 'modification.'])
+}
+
+conntrack_tools_expected = {
+ 'Name': 'conntrack-tools',
+ 'Arch': 'x86_64',
+ 'Version': '1.4.4',
+ 'Release': '3.el7_3',
+ 'Size': '550 k',
+ 'Repo': 'installed',
+ 'From repo': 'centos-updates',
+ 'Summary': ' '.join(
+ ['Manipulate netfilter connection tracking table and run High',
+ 'Availability']),
+ 'URL': 'http://netfilter.org',
+ 'License': 'GPLv2',
+ 'Description': '\n'.join(
+ ['With conntrack-tools you can setup a High Availability cluster and',
+ 'synchronize conntrack state between multiple firewalls.',
+ '',
+ 'The conntrack-tools package contains two programs:',
+ '- conntrack: the command line interface to interact with the',
+ ' connection tracking system.',
+ '- conntrackd: the connection tracking userspace daemon that can be',
+ ' used to deploy highly available GNU/Linux firewalls and collect',
+ ' statistics of the firewall use.',
+ '',
+ 'conntrack is used to search, list, inspect and maintain the',
+ 'netfilter connection tracking subsystem of the Linux kernel.',
+ 'Using conntrack, you can dump a list of all (or a filtered',
+ 'selection of) currently tracked connections, delete connections',
+ 'from the state table, and even add new ones.',
+ 'In addition, you can also monitor connection tracking events, e.g.',
+ 'show an event message (one line) per newly established connection.'])
+}
+
+pacemaker_expected = {
+ 'Name': 'pacemaker',
+ 'Arch': 'x86_64',
+ 'Version': '1.1.15',
+ 'Release': '11.el7_3.5',
+ 'Size': '1.1 M',
+ 'Repo': 'installed',
+ 'From repo': 'purkki-centos-updates',
+ 'Summary': 'Scalable High-Availability cluster resource manager',
+ 'URL': 'http://www.clusterlabs.org',
+ 'License': 'GPLv2+ and LGPLv2+',
+ 'Description': '\n'.join(
+ ['Pacemaker is an advanced, scalable High-Availability cluster',
+ 'resource manager for Corosync, CMAN and/or Linux-HA.',
+ '',
+ 'It supports more than 16 node clusters with significant',
+ 'capabilities for managing resources and dependencies.',
+ '',
+ 'It will run scripts at initialization, when machines go up or',
+ 'down, when related resources fail and can be configured to',
+ 'periodically check resource health.',
+ '',
+ 'Available rpmbuild rebuild options:',
+ ' --with(out) : cman stonithd doc coverage profiling pre_release',
+ 'hardening'])
+}
--- /dev/null
+[tox]
+envlist = py27, pylint
+skipsdist=true
+
+[testenv]
+commands = py.test -v \
+ --basetemp={toxinidir}/.pytest-tmpdir \
+ --junitxml=junit.xml \
+ --pep8 \
+ --cov-config {toxinidir}/.coveragerc \
+ --cov-branch \
+ --cov-report term-missing \
+ --cov-report html:htmlcov \
+ --cov=. \
+ {posargs:.}
+
+setenv =
+ PYTHONPATH = {toxinidir}/tests/mocked_dependencies:{toxinidir}/src
+deps=-rrequirements.txt
+
+[pytest]
+cache_dir = .pytest-cache
+pep8maxlinelength = 100
+
+[testenv:pylint]
+basepython = python2.7
+commands = pylint --rcfile={toxinidir}/pylintrc {posargs:tools/}
+
+deps=pylint < 2.0
+ -rrequirements.txt
+
+[testenv:clean]
+deps=
+whitelist_externals = rm
+commands = rm -rf .coverage .pytest-cache .pytest-tmpdir junit.xml htmlcov
Code Review / ta / build-tools.git / commitdiff
