From 4ded4f2a805e9447be90751d7d4fb7e11552e545 Mon Sep 17 00:00:00 2001 From: Saku Chydenius Date: Mon, 18 Mar 2019 09:08:45 +0200 Subject: [PATCH] Add initial code Change-Id: I72d87e74c74defc97bd956c3b23de9a4e01acb28 Signed-off-by: Saku Chydenius --- .coveragerc | 3 + .gitignore | 10 + .gitreview | 6 + LICENSE | 202 ++++++++++ README | 8 + akraino_splash.png | Bin 0 -> 32435 bytes build_step_create_install_cd.sh | 133 ++++++ build_step_create_localrepo.sh | 22 + build_step_create_rpms.sh | 54 +++ build_step_create_yum_repo_files.sh | 36 ++ build_step_golden_image.sh | 50 +++ build_step_prepare.sh | 27 ++ create_golden_image.sh | 60 +++ create_mock_config.sh | 54 +++ create_rpmdata_in_docker.sh | 71 ++++ .../myproduct/cleanup.d/50-copy-build-details-out | 28 ++ .../myproduct/extra-data.d/01-copy-extra-data | 20 + dib_elements/myproduct/extra-data.d/collect_ecc.py | 128 ++++++ .../myproduct/finalise.d/01-remove-old-kernels | 23 ++ .../myproduct/finalise.d/99-collect-rpm-info | 30 ++ .../finalise.d/99-create-bonding-soft-dep | 22 + .../myproduct/finalise.d/99-fix-grub-console | 23 ++ .../finalise.d/99-generate-binary-checksum | 63 +++ .../99-remove-dhcp-all-interfaces-udev-rules | 24 ++ .../finalise.d/99-set-sshd-config-defaults | 24 ++ dib_elements/myproduct/install.d/50-set-rootpasswd | 28 ++ .../post-install.d/50-remove-local-repofile | 22 + .../post-install.d/98-collect-ecc-packages | 24 ++ .../post-install.d/99-validate-packages-to-install | 25 ++ .../pre-install.d/01-enable-yum-priorities | 24 ++ dib_elements/myproduct/root.d/50-local-repo | 31 ++ dib_elements/myproduct/root.d/51-rm-grub-defaults | 27 ++ docker-context/Dockerfile-buildtools | 20 + docker-context/Dockerfile-dib | 32 ++ docker-context/README | 31 ++ isolinux/isolinux.cfg | 48 +++ lib.sh | 233 +++++++++++ mock/logging.ini | 84 ++++ mock/mock.cfg.template | 89 +++++ mock/site-defaults.cfg | 160 ++++++++ nexus3_dl.sh | 52 +++ prepare_manifest.sh | 28 ++ pylintrc | 444 +++++++++++++++++++++ repo_summary.sh | 34 ++ requirements.txt | 7 + tools/__init__.py | 0 tools/buildconfig.py | 33 ++ tools/convert.py | 142 +++++++ tools/convert_test.py | 165 ++++++++ tools/executor.py | 107 +++++ tools/executor_test.py | 81 ++++ tools/io.py | 37 ++ tools/log.py | 29 ++ tools/package.py | 53 +++ tools/package_test.py | 44 ++ tools/releasereader.py | 31 ++ tools/releasereader_test.py | 34 ++ tools/repository.py | 53 +++ tools/rpm.py | 99 +++++ tools/rpm_info_installed.sample | 144 +++++++ tools/rpm_test.py | 50 +++ tools/rpm_test_data.py | 179 +++++++++ tools/script/__init__.py | 0 tools/script/ci_build_diff.py | 179 +++++++++ tools/script/ci_build_diff_test.py | 249 ++++++++++++ tools/script/ci_build_diff_test_data.py | 281 +++++++++++++ tools/script/create_rpm_data.py | 358 +++++++++++++++++ tools/script/create_rpm_data_test.py | 84 ++++ tools/script/create_rpm_data_test_data.py | 303 ++++++++++++++ tools/script/generate_repo_files.py | 74 ++++ tools/script/process_rpmdata.py | 97 +++++ tools/script/process_rpmdata_test.py | 58 +++ tools/script/read_build_config.py | 32 ++ tools/script/read_package_config.py | 44 ++ tools/statics.py | 26 ++ tools/test_data_rpm.py | 230 +++++++++++ tools/test_data_yum.py | 176 ++++++++ tools/utils.py | 31 ++ tools/yum.py | 215 ++++++++++ tools/yum_info_installed.sample | 223 +++++++++++ tools/yum_test.py | 67 ++++ tools/yum_test_data.py | 106 +++++ tox.ini | 35 ++ 83 files changed, 6713 insertions(+) create mode 100644 .coveragerc create mode 100644 .gitignore create mode 100644 .gitreview create mode 100644 LICENSE create mode 100644 README create mode 100644 akraino_splash.png create mode 100755 build_step_create_install_cd.sh create mode 100755 build_step_create_localrepo.sh create mode 100755 build_step_create_rpms.sh create mode 100755 build_step_create_yum_repo_files.sh create mode 100755 build_step_golden_image.sh create mode 100755 build_step_prepare.sh create mode 100755 create_golden_image.sh create mode 100755 create_mock_config.sh create mode 100755 create_rpmdata_in_docker.sh create mode 100755 dib_elements/myproduct/cleanup.d/50-copy-build-details-out create mode 100755 dib_elements/myproduct/extra-data.d/01-copy-extra-data create mode 100644 dib_elements/myproduct/extra-data.d/collect_ecc.py create mode 100755 dib_elements/myproduct/finalise.d/01-remove-old-kernels create mode 100755 dib_elements/myproduct/finalise.d/99-collect-rpm-info create mode 100755 dib_elements/myproduct/finalise.d/99-create-bonding-soft-dep create mode 100755 dib_elements/myproduct/finalise.d/99-fix-grub-console create mode 100755 dib_elements/myproduct/finalise.d/99-generate-binary-checksum create mode 100755 dib_elements/myproduct/finalise.d/99-remove-dhcp-all-interfaces-udev-rules create mode 100755 dib_elements/myproduct/finalise.d/99-set-sshd-config-defaults create mode 100755 dib_elements/myproduct/install.d/50-set-rootpasswd create mode 100755 dib_elements/myproduct/post-install.d/50-remove-local-repofile create mode 100755 dib_elements/myproduct/post-install.d/98-collect-ecc-packages create mode 100755 dib_elements/myproduct/post-install.d/99-validate-packages-to-install create mode 100755 dib_elements/myproduct/pre-install.d/01-enable-yum-priorities create mode 100755 dib_elements/myproduct/root.d/50-local-repo create mode 100755 dib_elements/myproduct/root.d/51-rm-grub-defaults create mode 100644 docker-context/Dockerfile-buildtools create mode 100644 docker-context/Dockerfile-dib create mode 100644 docker-context/README create mode 100644 isolinux/isolinux.cfg create mode 100644 lib.sh create mode 100644 mock/logging.ini create mode 100644 mock/mock.cfg.template create mode 100644 mock/site-defaults.cfg create mode 100755 nexus3_dl.sh create mode 100755 prepare_manifest.sh create mode 100644 pylintrc create mode 100755 repo_summary.sh create mode 100644 requirements.txt create mode 100644 tools/__init__.py create mode 100755 tools/buildconfig.py create mode 100644 tools/convert.py create mode 100755 tools/convert_test.py create mode 100755 tools/executor.py create mode 100755 tools/executor_test.py create mode 100755 tools/io.py create mode 100755 tools/log.py create mode 100755 tools/package.py create mode 100755 tools/package_test.py create mode 100755 tools/releasereader.py create mode 100755 tools/releasereader_test.py create mode 100755 tools/repository.py create mode 100755 tools/rpm.py create mode 100644 tools/rpm_info_installed.sample create mode 100755 tools/rpm_test.py create mode 100755 tools/rpm_test_data.py create mode 100644 tools/script/__init__.py create mode 100755 tools/script/ci_build_diff.py create mode 100755 tools/script/ci_build_diff_test.py create mode 100755 tools/script/ci_build_diff_test_data.py create mode 100755 tools/script/create_rpm_data.py create mode 100755 tools/script/create_rpm_data_test.py create mode 100755 tools/script/create_rpm_data_test_data.py create mode 100755 tools/script/generate_repo_files.py create mode 100755 tools/script/process_rpmdata.py create mode 100755 tools/script/process_rpmdata_test.py create mode 100755 tools/script/read_build_config.py create mode 100755 tools/script/read_package_config.py create mode 100755 tools/statics.py create mode 100755 tools/test_data_rpm.py create mode 100755 tools/test_data_yum.py create mode 100755 tools/utils.py create mode 100755 tools/yum.py create mode 100644 tools/yum_info_installed.sample create mode 100755 tools/yum_test.py create mode 100755 tools/yum_test_data.py create mode 100644 tox.ini diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..f3a5cd4 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + .tox/* diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e1d93d9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +.idea +*.pyc +.coverage +.coverage.* +.pytest-cache/ +.pytest-tmpdir/ +.tox/ +htmlcov/ +junit.xml +tmp_yum.conf diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000..0c6c889 --- /dev/null +++ b/.gitreview @@ -0,0 +1,6 @@ +[gerrit] +host=gerrit.akraino.org +port=29418 +project=ta/build-tools +defaultremote=origin + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README b/README new file mode 100644 index 0000000..4efc9eb --- /dev/null +++ b/README @@ -0,0 +1,8 @@ +# Unit tests and static analysis + +Execute with: + $ tox + +Execute only subset with verbose diff on assertion errors: + $ tox -e py27 -- -vv tools/script/process_rpmdata_test.py::test_components + diff --git a/akraino_splash.png b/akraino_splash.png new file mode 100644 index 0000000000000000000000000000000000000000..ed3e9bd110a72821cb773ca4d055c6eddd4542e8 GIT binary patch literal 32435 zcmeEtRa9I-w`Sw+8XSVt5JIrvZb2J&x8N4s-ARHIECdVg?(Xi|5Q4k=ocwq0!@SRZ znKk{uTEOB^RlClv&ni+`Q5pl41Qi4VVaUozKtLea01yaf6B!Zs3D?x>9}oy0!&+Qi zSyo(}O4-@L!rImx1mca2RoAy7Q^yx-aj|?$JFaTW8R}Xjj6sL-U4t{Xw%LDejoUt1 zmV23I&+O)8JauTC_e0Bbu9Bp(&-;T6{N9d+04m07tQDc!cS)#p%d=FBLbZ3n%6riT zhiUvD(yW`S*3D0RE;CQFyib}ePZ9Q-erqLv#fKqGFJ~p<*m4Pvj}T-@2no+2Ld?}v zA!pGqmWSn5LUXUv-4O5gHe8xpd&w?qkRm6ba-uY;`CEh*5f5`#s6O%q#_QoLrSO3c z#f`~c!+Vk5cV3N9(}f-vA-`B_i$CD4nFs`R)z1+r*zu!%cT(t&?HVYRDBCIBPiYq0YF+VQ} z)icEt_5<_43e&%#!d(iUFyN5A`Rc?0L*p4Gpm6h@tQybk^kGrXR?j!3eJ=KL$FlK$&+YzQm1pT(Be#0ugeK3zo{nnezX3giHs(9*JLWs?+occtm4762yB*5rNa>PYn8?Q}*eit>b#=9U$3!M_4F?6@c2T?-T5+?OL~ga)M$OAynk1zih&8q< zzbamF)vPF1cum2V96Q*@C-S*s`)bI+wCTDpto96_{UOE}f%~WF?k zc9NyoLgYkLI2n2+8r6sqitjW{Bq`&RyU4r0rMTQ8I5s{Sw`R?MJZR6oGVKkQKqcCE z!FbAXDd@{s*?@uZI{`rr=GCybxBm(}8JUvy?sWn!KM@uO_>_^5L^7x zeHU*Ak>^kMBtNwfrT0_G(B_MAr1$-)V=T|i?7O-nQ&S0spr(&>pCz-|`kTSS9WF!9 zo3xB1E<+%4Rm90k^PJ}$@v+cgEBse8D`bCe0dxl8qmnokPA41Apprrh$Lm?nn4l2I z1!Xri2nHrxd|)&Ai@@ELSB&j`(9BLz#$kN;2S&3`otk`AeNS^6nKL^g(uN;aDGJ+8 zHGT)k{|qqc>hvx7h|t)k=z7!837xfLC1NTmGC{}zhl-5{m=8S?K*{|L(Ik{S2bzP4s`H$h0;1lwO}EBmu*uah=sa6;{(!_&z;C# zh;VK$&rR|;&|=yX!84a|cNv&<3!dec=X$}4NEyUPtrLm`=M;Bs7;@jSvpNqdX0onf z+RMR$sLSjA=fkKKSN-6HQz_jB%Jll$Jd1mIoFOgy_LO_SotgxH`t~Ad2c&2Yo{xId z<~%2V46X3-WdrdB#8wk;`kw4lp)rW}o6P3WxMJz;qc(dX!P_G~&BSnn316xcWQHon1{sgBx5tIn;B@OEVr92jO$#9?A$xRd! zNbT=4{XSTLI?2q4$0%YIU@m8d>#0>BmOti74YHQ~rHY^>c^F6OmJeDh&aLYS&-x0g zKc$pf5U~+&<=K~@*d_`l7UT*ZnqmaQRsM1tz$#QDl#`6Ck+wJfD@DdnX&8+}K^8Jn z^OW6jW`^tVG!6&ioySQes^2{^W$dSj*isC=;o#Sz-AGZa`C<8ahmXIBu2|) z?&0n060i+MJ;*+6JM{)x)6%>#!$knsA}7MSYTh@IFRg~!J%&$B8$43`jU*ED^9LUi z1tBZZfBi&s;4qwAkXu%QvL+5AQ0ai|`_>LU;F{T-_^xH&nidrS)agMV+>xEx=ilM< z7R@sB0|r``E{Y!ihiI0vca~0gu0rO>GO3j| z?tYVi+?&uK@LDb%%gNnFD2P)KttT!jF3syzMiaD}|ay>*HyE0LI?ld6*d(IFxQO$i%|897fv{(yeCQo*BJ+)_h z!xt1x%AeAzX`Y+goWNrI;6;`ar?9o0rmaPvlZfD%9a5SoHpEEn@iuaB_M=4M_eCco z48F&V6gTEjYb{1I&ahhnV^X*BRk&+p}BIZrKtokL`tSeB5Fu=_7EVKXFHXArR@JGksI=jyv8)xb z_0F`n+jmcy?^8Y1{j!sO?`8Quw>|##@0dAlxoz<-e9g#YF+TVVt7`gQ;QYBv>r&)a z-`nC1DCID?7~hXeF!o#zQw*7r$ry&0B!2s=6@I+?m+AzKqCI(3BB6}V#U|}uOO60+ z!#DLhiv!BtoSz!MutiKtz+xRycAqfJ|19(N^okE<#2QHy?nlGohb7@u=!^?<h1rFYY!8{vqpKU4lM2!j$rheGw#;U33K9^i2r?U z)y#Z#*mWZ9^ z&Vt{&pjzmNXOb21)nEyjRvc7gcN>-`MFzCbp^btn&#Mf0jh;*~gRZy@F4HE=V)^(S z#kh+8j(DmIri;(LI<}sUIre`M*%~5DenM{$qJ zI0v0`8jX~a()MuVo6+?cI7qO$c75G?eyvd=tws3+HTl?Efu!COhJ~MHBZevBo*R2G zRn&SLt@!=L;tSNQU1tpuwDz)aUv;hZ`1zyt<16&*S;OUf#5_0xKMgRv=OU4|;)BTz zu@fvtMU~TGB*15*%m#XA-}pdS)lRM`=lxpA#mS1cstnWD2s5XLLEiGLbV&;~VZ*=G zQWMdURW8)FOGE>M$@VJ;ZF&ll?Fbz(ZG}tRhfLA ziy_IOi4YGKvVMyW%ELygU+;zU<#JPb9PfMa*?gSQ&L@RS6V$mI0vui+L7}cc1Avxl z^cfr`ZLk{Rw&D$SojW~H%=n(w3A7w1fRcec2ZG9<$wv)G^wo>#L-vGS0^T?-s6^RvC^j7mStjo-%@rM&); z&dv+d3rudTOEKaMZ=Jn+w+YNv;9}b1s=_+FC;u|H z(e#YPAy;iQqtTBI*lRJ=0%<|!I-jODpFiu<&rl;X34CH4G@G=w?Cp3d?75+x=pOOjhV=*h%M`dN)wPBu{v zv79WUUynVSq5Ty!Z+~1IZ&PxyJMH30=DC9laq(n(D4LRnlbfRjrPHA`NWJBKoWfXb zB^E6PvE9Qa;5ES@fH*cP;Z&YSfDy)j?0D9+lD9I3`mlbhK6QYZGQph|pmjUB}H z6%9D!cs{@#VxkG3JS=Jxnt%*25Z`ZI=8lZrGhY9hpF>`M9rGL=`)iz<|A0wjIQms~ zd(h!s=eEn5d&-@c`Y7q$)fh`ENp$sYML&~r=lYrEV3OnC9#USXJ(7zzc**ATmn?81 zUEuV_FZ2=g9Y{#8M&9~As30|yhHl!$NzXK;lU`xr<7|O3)?3Q7L9#tX9&cK!!DLZ= zXVwCks?J%ef(gmJV=GUn@GXGGZKNkgSTvN!%A-8gT2u6Z~LHB3_&+ed8L5zXM(9^*eKj`}V zOBWVwdI6lyJ(*T6JlJ{yNfkK|SZ$gbCiK!{tD=i+;@Hi*mIdWO&4umX)_ktH8&&dh z2sQRz_-^5M(;9@oe>2GFp!WUroN!47C&F<*SGe;fy;x=j82uUO z#er5w9%+xRm-1+e^*Ypo}LN2kfH~DO#rg(*a ztb&xehS2cDp}Wz_i}6*NbC>(Kqi`0;H|LNdms6+(jSpYtVM&98Ap zwl+l^p!o!9y10ok=%pF%w4Pp{CMS9;A*!*VE@?d{PmN=WEVU-{DHolBuC56ztQejn zubb`STySg)5_@eG+{?Ukyu+y@={OnrK-~vwVo4V#X=72Cf}*8?#%?dWt{uOlw)dkk z6Y5-a(#SbA9S9RoQ~zk7&-QK-1VD3a++*`0A~xYzq_^nc1BqZ)ns~Hq1sZ}8(~k`C zTFJe$*Tl~ICJNov1=eR=Emckql=F6{eVbvJje?4$o;jp#Hp`BG2YD?V=&6W2C4fWB zT8mvyV@yN3!ga?>Q9P#wg&#A6{QV$pPsNH_pM(ZygfaH#@*l%Z+$-62-q{mxVI53xnM1@W#~41v|TqOXp;+ig2n# zm4UJ%H2Kfo{u@0ZZj_PIr?+e&h!q==g?^3XzBbE;FCGFp0_YZLxXU!K#oE;ergsYv zWJOvdV)`Q=7uVb_sFTq^peO(Vu>m4$xU@Sx^QU`09C50n0K+nFy>sa;9y_|tR9o(- zuR8Zt2xg&bPeGgSqPL@9GosFkheeb1;C&Z|ispjhPz{S(6EyEvP-SuZdkpJGe1d`q z5+)tO^>_)#JmpGDjIWRckdP9+6aClDyyN4^kNB-ulolDy&-URh1@!NhAJZ$hJK4jy z-PYVPP^jUb$|wMBf!J=%?+2eVInTJXGFYwUp;3r=AylsqD4`-*a~#A#WA%~=)Y!%* z@brWT)UJiPJ3Es)kAhDbA78G1R38@`c9IjrJ|3?}m#=cS*lmyEbK4=}ZhCocTj87d z_Qp4Y;xJ_X?&#-6^<+{_zLVctp<*DmLNq&H1K^2L=ynHUE}bjN{^DG!iX<>_MzO2I zyXO`HnBsDa+!{srkb+3V@o^LHPiU}gL}*_7B{h;r!ugOV&cxY^<6{ZK# zEM(RdL!6X9{JH+P16I>a);i*erS?uZ7yJTuUA! zBVPtRwzjQYE^7E?@ARtIP=(aC_u-Qg)5!$guDg4?SA6V^?d1Qe;tL2$j(gnmnAa~` zRY;EN-{hbYG=UQU8Y`IK+9Q!kqc^|^J#*A04NFhg+QSgRBWd;HxgpaF2Gj%zn?fwi z-7P{gWc`d+@;No`Yn7&`mU>sZ@=UBPfOWCUmu!v9agvW68wlJEv;eQ_bi~G_*8|tc zZIB0h%jDtqA}?ZM{8^4LVwez2_TUKsIx5 zK+B)VHDsbihwDhql4=g~))$pq9ke(<+_!$>Ib7%&Y6`%>2D$zF>sPdl6W;Tt<^BNM ztKi|&iwVpAZQh3ix<57f%!Izkygy(c4Hp9fG00g;j5IWV1e{TN9Rf<#XwM#%eQMJ zV3AarjxZgSek0x`Cl$V*QYYMo!XuM%%gpV`<_ ze{BI-u_109%6;VFL#%s)xjZ{a;C4-1E`c7voj$tb$V(_bA7*EG@+gu{U+}9}7s}3k z|7^8Y&(6}3)`l7-E*Ggv8Dk0vQyK&-ZhK+_lgXF@Ppbv${HLd+J1d(LWc-H*8Rdt> z%Zrhg{;g<%;{I!uxbpP1@r{dXfb6TjK;Z?*y1Mz?>f_jrDLK?k3 zBMxk|g+Yqd`oafc{4+@7TN~eyx2=Mo4|hg70&P2Z#6;Zp_V#Q7g*T1wO0Ib}{&Ian z52&wICQxL+y)(1Ngv=zD4>8kygKV|Qz}MUhIKXqQx2V2EiOj(N{J`XM@j#ForheY8 zD`w@`bbTSUOvj>i`iB+mREh?fFk1QVG(|=JuihnB z63T&Xj1DTJ9R~7d-5s6LH@u#Ad|SEl>K&ppKpk&QZ}Dt2ds~H1P9#cyk~GJs)PxJKsH;hi%t_J|O3(V?9sH8m2%azN9KlPSIXF00uJ==E}M-|F z@XVoxfcR!h(?!~Ynw!fela89KegF72Bw%H?^M!~f$KQ@WyQ=>g6Q``Quz{P^uN=1g zt!WziI!bsh>rQuki=QMN2gm>m5vRlifsV_}w^Ho4r}}kPTH9L~+a-R`TJE_g_134( zvtf{yx93dnlomQ(7)(>|KTtfFMjw|CAyz2#4A!D7jS6}HfuhIKA)ba%zNcDays6wo zQrNrX;6i&G!(a0Ai|h7Ig~9Pl9eRLzbNbL`(vo(n8}5t!NGiP_$Hz#jhR|Gs7j1Pp zfOI=uOuDt+EDSj9^x{vW2;H)99$xMny2Z=FWy5!W3}Df$)#|mWsfm!a-dGiGx$#v- z-F5j$7A2wlCkR^yL0*-HBdwcXm_Pc~-xpJLA&-kc%xS-y_0)#d6!hV}sh7D@w9Z$$ zZjjdgfRx-1d0IzaiUzj{;T2}ue4})rZ6Ut@I^yIY!fm117vBAi5a^IZ-*ZswH&H=PmI1%RUo8aROC1NKYNK|4i&oBLKwb>qHVs zJI)|Im}|90Gq1JC`lcNNbA2n6MK-c$ic#wF{$h&f%POc4Z=$wgTPZ9IyWNXmH@|kr zJ)hdTyM3?q8zKz!2!}R68PTbU}^ca#>PGqdvBK0 zr=tSvnF6bQWhJ^Q62+U_MFTb01gGOiR=-p>Rv8;OfQY!+VH7n6&P+mBy>Bx zA16!B_+N8g76dgcEG1yjFcIY7|AGwWM@5VmU}ws{C7(!vGo%e}k|CT1V?E26i^{5F zVSV=P1J#FWgX_xaY+P|8s_)s}gV>b2IM1aRLyW4Bb7!}Nv&BSs$vm10k!*Z~_R_<` zmzOKw5%#EzRNMJMm();C!+P#S{e{U$q@Pl1zSOzUM?z4=z<+Pc;tmX$P#6OvSyUt} zA*qeHeeIS!Iqtv8PJ{&;czo66s8iTz0FC9$47N*`$?Q9V**8zK83bYkwDbbe;tRj1 z;W)<%{6yx{DaPbWZJeU9u0b-KT%Gx(x&xTnkD44Hw#hzM66tNHRLF9*gc5QfPB`4e z7x$jL6s(HVYmc+GbptXb+5{q#saFa*x@pjQWS?VB9^A9RJBL5j2sojTE)g?&y9rz+ z7{=m+m|d7b=FeesoHf76VycUA*(ft40Nv|2KF(-25XZx(y!j9#5tY$QK|W3j=SOM} zAVSjSeZ+UB!>^_zmPApQp-KvZ;C}YQiOHlCSS!w}%PVdjdggHN`ZJ9nQSe`4lrsHu3RWC=h*v=7KV{%{m;$&bz-VDq*(L3v{AQ-#E{qImoD&;6XVC^C(L)Vk$xh~N)jxH{eWwv(UKFhuJHfwEC964kvL_> z`f;nF+|=KORxA>@ZtBu%F^j`-)2(L#1kbh)j z<(f{aD@bM3gHrky8h#gt1vwz9Qu>UgmMhBK}A zvEyU>nOXG-33|inIb@m>qb$=v;A|7*s;`@Nir!-gH;0Tt5@n+XY}Lo;G}M#(KxGdW z{O(xW`FT8HgS~_1-FYyZ{y^8b;?dP3_1|42jl3CXr!WPHSadLXo7(HKFEuCUy}rkh zw=R2SJ05`81umueD_?aFr&dn_u}L^dgL9ZGoJBlnLT4!B-exeD9x#Y4xOzO>1Ckgb zeDm`vUsTaDr)J(kl3>86J!j_7JQo*6X7hKy?7nP5_nxbuByktRf;}MdF$(&Wz&zJ*+EM zfAM9H9Z?U z&`vaceclu~YO{3Zu&Nw}-r~bMV~2P1_daC(xepc0p`C;Cm}GZ1$t~pq1Y$V=rBZsf z;%^SqaRRi|-hvv#(kOjfeb#&ln)G068D7n!tbN$#0`WcEvP=bl$L6)IgrDSs0P-#Q zePb;A>%sn~QJV^G4+M^PO? z2UfT-tC@p{LQir5J=xL0gNeWB&-~B18ZP7tV{8H!TNbrnRy!T~YTE{?n{j)S=b%r@=cPM2br{}}Ruhz`#7$%X=86zbN86`1|PIS=y%n)=26^2+Rm)G9e3yB@w<)O zUcbYVYVL=k!1Qty(#mp<%L(OLp}H=QrpnG2?lG8mH-Zf~?_U0GQ=8zGo$}^2R^@e` znyf;3@Qn=jNUh`kB@>yy1G?1E9~P0?O)ojMCp@lVINS29RNVGoM%uh&k(+wlHWS*d zVj`-jk2<V$;iYnAWTKTcbzQ z;K?5kQA-o@quE;kWDH=eh%qqhDOY1ZThh(@w*oQW`W6C;L`g>KIze}In$y;;gy^u` z$M+1bymiBkrY5`4w?BLb$mqQ?Sp&D7R80`nGWq-7Xe|_*KY2u zt}eVtlasF53(QTyVDb2tZ-15$nK{x+RTY}cX>M|0$wN9D`C*R)Xd|1~O<&(fS^cq{ znLg=?8<@{2=^!{3^aBzLjRKg&bZC?ves6Y#(*kOL^H0k&g+ivfs~V(ptzoeQTVOCP zpV^3696)Tg-a7dBiM6L5qB={lvq?V3@@SV*AKv(I@*s^=TWd)m+ul5$udN%b{N3i_ z&%C?*ilaAVfx|{PG6qAv2|?XBIO+H5$mXez>&foP&ns&|5G#I1l5~K1ET234D(Sob zy|DeM!+$?}hr`qCdtcad>XPr=2|ZD(ACdDAu9={oGFPmig7|6rmm_ZrIAC72@9c(; ztYIm9ojyR$1?=_EHGQXcuo;~$&baR23~YhI*z?5PdXQV$-HOXzC!rUe%tt6U?!IQz z5#Ua7f`h3m+uue5*v~4#lMy|DG5hxM`;hD}g_$fN4eCvIzVo&$3eJ_)w>*EYx0>QG zpO|q2RhUKh`0Ba2jQw>zSJGxP5D1NdeWb!;H_mcW4UceqeGZ}Jnqp8 zJslIICZka78nC5&=3gOludi+c{8yJfLlpmpx-SoBf3WIlvRSD)zlJ2vap~mOCdkSG zoX?zBpO5?Kpri&*1m0C2Fq){zp7O{O3iPzDed2HBX5gLoX$`}BjQ^5jy|{gA>hH}@#=TKR4*;{jtH!r-pDLcrf9r;U|@rjS4!rVhfXl1C0wKNuTIc28k0iGDl-uC(N%f8=#RX zl$|TWg28HFEcG2iUO50ue{6ltxSS*>0s3ZfOO}WN92q8g<$jwJufk)B>NILO`R_D1 ztqd=T<RS@F+SmbCHZ=vFMWzVMRWMSsPqowMRPSuUV|YOKH_2`F4%ITs^% zM4RSNf6G0OCRZ$ZWmLJlO`23fP4bzt^Xu5!;%k|LqbFyCq}mQd)@+N;=i!kh70vSS zw0-fZh`AVa^ECAvxdJ`F-CQnmV|3K0TC4Wholl>wbrpDr&;(8!NPq)A5GZI)2^!GH zZAl+kwFz8LGYI+>Si86W?eSnOjk0$~V%cFX;ctNQf@MAWwZrl8QSb zMb6_bzF4c)zqO(V2_GHzIjXAmBuUq*b^AAa;yLvMe8XLE`PfPtxmd%TuGPdwt@Itu zAiD;t1Y0ZxXp6vEN`JSkH(9dos7IZ_JU{l?N@wg2eZG0vAPMFyDecV3H^Z@ zef>rA8K2gCwQsoLG&p*r79%~1a-y)j__13vS~2g)xzq?MtYOoO{93S8-mL^j9F+U6 z)l&fKkkp={%G$=y0M}CZnz(Tk3-p}+lEu%p$i06XM0NFVC7{lHv2UwV%g}_w^w?S_ z*J6_q1;SYMz+1;M@RB1BChK3n;Zo1^@TS+lZfn*0_vXS|RjD^yZ z6dqS5nNM?)L*RPWeVcQmTUDF)-X8)z*x^3uw06yEsjCGi1B_H-s2fO0eb2+Ws;~*4 zQP-Xw&W}b`qH{W5WR=&8dHa1>W9_n#7&Nld-bxT@)X$k)+NT*NHnR!iSXqNdN^(YA z|HFEVXJ#g~eIUz!nA3?SsTl^B;6@T)* znic(?=+n2>vqy5ZG|eVZC~1sc-kd12eltj~(E56a2Jr8$tilCj57k}ri69;YG@t-6 zS*iU=Gandtz*^I_Wn>xYf`H~*@&Tmy*V%)gENs@TGj3ft9F80l$>8KzvSDS-aR-Nq z(20>yyu12@p9sPfYRz>1g*S#vLE(%)vy^EGP&0A|wgX1CZJ+c(c6V?-tQ3ucY$A9f zl4*_Ya`ESGbs8#e>_nQsB?SZIm{JQuNr_C6*V22&8Rks@KMtO6M7mE0FsJGm`zMf` zuCLImQSb`1tKD+LCpoXs5hj-wS|4DIhN{n0z+q@h;9u6NKa+8>>gu=7$H&8T?_fZu zAw9gA@@%%$F_47I!1H~8Phr55TsRcWb?rNMuDNNpu*4-3;mGvpx*o zXuX9$-f5#zz_YTp2`YH^!Y3yMGyfwl9&oWI$KdNJiNjikLL*0v^?O^KVEIytQ6g(n| z=!>7qkgggqTr6$TWb?oj)561-XQRF4ObQq|7U4_{Q|`XmbmxECZ|EnH=Pn1)-2x3g?UNXb2kfzyD@AzW-_!B>@)u45C z0ZAGTGuS{e-;ozLaJpoXRGtZ-g2LHpqI(o9;hJ7CmTaX! zrM4oTa|q7EFF-|T#X;Z7j*02aJ1qIlferdG-F^Hyfby%?E4C&#lJf6+7sq@ZZLG52 z2kM4X3Iv!iGh&{;YycO8b8IH#b@Cw77?Nm;Qg*wFOO$@MDnwL&OxX=2Yq9svy7t@M zdv(9UOn$@7?lWmj7)osLt=b=KAQz&%scYc*uX+5*A0yV zYT7M$T*DD({>V}RLwUbS!7y)kD7jz++7)UJyrkR^<5Nj}1+yL3;^bNfK+d?E*ek34 zW94P#as#PkW|B-jCzwhj?SE7c@EGmT`~jvj69}4r^uge{vXP0zb8%@@W)xYYz%)Fv zVST0pg#CjtA4qJ;aeLlS@q=e7;Rt4jAf?cCu*&S`SY zVl}Q&k#j|75W;xejC#t2Y***745LZY4W4z@2?#*Z$)SddN^es~mNcw+Lp!t4(T##9 z794lJaw)V5@kggYP%2#x3I9c~=HwA6CGzg_xVQTyL5euR-oSm1Lp97g#3^d zCZss2O^LW_TF;Ep#@eA9Z(}72u>5$RyWCb=@e$?8Axk_vVhd!->7iL&gg`DKd`+n| zbm6q#JboM)h;Zb96F4n|DTFmpnwM;G{}LJ*7Z;XsXdxSUIYjG6gxXc>#P)}%q99QQ z&O?~6=jG-LIuBP>JB^>C?PD|8l!>vEbIDt-HYeckWTm|>*26+QbTpjD5&9ivH1x7E zv8l*i29Ja)pgh;2bZI&EI37K(rlN1&#_&B1yAF73&Ins2gWH!-tP^dsFN?yJYtysC zlj;1nre<)?dB!5&k@`O{{oi)S+Z ztGWd;JM>uFixP^j;Oox~|%F~8o? z1F^=yYLIF=+Pbi^s3;x@XD;>RT!@1AW0kQc<%85RA=hlnPq>_Xd`wB2##A{`E1m1d zRo}Fc&;dX@=x8I4Lqy}2WBM0#w53;Fzsc92?vkjPp^5$S3pQkI;b&sNiK9j>pAuuB z1TC61)5-}JRnDPeL?|W-ib$}-L&!wl3zm$cO*lN`V}$_!144^xXCL5|XPOnnNMj zVj>IIQ93jxLcFY&n6m&Cv)z}nG(-bfzti76UJfUW0`v6-KSFFr{x!#YE?YlG(4qU4 ziIph>S=@50P%`QF6K%wW;+S;CN=XwXAd}S*sy|tTv;bpKU zPEEoM&r<~rAL7;Y`2;EK@KA%@c?uz3b zKl~)wnTeSVgL5%Wh=F$6RfS!Hd7KhZcH8B-=p(W0g^Mc#*Xz2)`Bc1gAb>jU9mB$zuY_%o94qkP0INYS&0i%=Wk3HQ5REL_pt=ol-t-zk9rW)gRNBTwrDr$VtE4njE%^qTBC<%&0$l&?iDLglD!lV&k_9XJ&RZ z5*0FpUS<7-UzXCjFnCI>Wg=gB-v?$ZDRhC3PbWwY*8)3skQSGZhOqPNR#!D#NS&u9018L9BSIL%=f|Qf8qi^xi<-7@|LXX zY$;AyzqjmU)K)Hf%Hq|32;!;NSN8aZ3S8_-W@35b2SOylD~BN`mAl$pa`dR$d0XyM zOu$8#O>1Ydntf6r0LVx%I$Un<7c##;553{F^E3c|`kBD0t~bD(pL3XQhemt!Z3N%x z_y{cIZ-YN%1!H)Ef0Wj;PjrPOwEQuE~FCeTjkV(nNd3R@HJHmO?i^hNa@T(UH z%UYm?(g}~M0$#6q%^f>QA0;L_=W44g4b4v-K;rNUg9WVoFqP_n`(o_o@+EFm)!48X zx$IB$=h_08=F7F##pR~DpU~RpW@Zu!b$!U9p&o!f3-A3VH7pQFiUX$jT|XWUj&Ed7 zZqLe}poP=0g3UeA~2LdVtV3$2kB#Kp~7$eJV@RnFV8rc1C?y zaab)lhA8@uoltygYbj4cLgM;2HC&DwD&>I5oumDD7OP=MfHJDLj_$dHv~*8WI33-; zv=dvJ-i^(B1-y>*to}ONWCtSP@+Mr4FCg3jMoeu*;U{EQPk}oFC9HI^SRU334Wamk zw>ZD5Bue~FC7DKLBeRthX*9qW7;kB4Oh9_A0QpkpjX~+DWH>^GAqps5S*gm)&xihf z(S~ddKG84MkA>=inVbYN`qIpB|2iWCyd)90M83GGv4YMEkq*PEaex z;-fpq*Uk(myP6<^W_|(_;Xn77;~G~ExJ_F09?oc({j8>#iS~#ODu(Q#&*oDoI`ODL ziU`b$-Bf!-`06nhy~vR%qYo$^`VA~D4$ga!Mh^s)Vav(qhs^3VZ_;O5$EGt6X=(8j zU3a?7{_u0&MB~Lvw9tS_J3c9CCLsG(0Uw;YS=ED0mt5vApU%zw zg#={H91SC&dY^(EpC0ANy^1I3&en_CbWtD%0YW33W~-$%$>EGJPJ zBm`vdIlgYLNJ&Waf*KuF?*-OT!@s6XXEph!hhY}t3GbbTSHKYYtq$&-eJ?J)K-DE3 z7Plf<7NGFy=%B=^t=&ZO^>tk0q%Hp?(WuBgPVL_*X6lr+{T4htdf}0K`(hR_eGv%R_AVfimr~S!E zq70*34*`$*5;O{Rp-Qi*TNODoP7H@ovDC5M`*|E3F0pX&+z1bFboNKgEFoB8mGPOxXVFfgSj3h3Fr{h1gKl8uYwdLBUpuM}j${E{v z7#8-g&VwVGv3{Z_uH9)Av?eC51GnN~ZYza{kvPv?r;yTz8FyDFTgvFp!0WMDijV5E z;>#&R#H?T`l#U5?cP>X<4q~uj*;3$iLMLhK=Auk^raB?@Q;*gk!HHka5fXoan24)_xTYrQ17b^+=LiWp#_jTjQ-_Ze z8PRwwbY22m4s16OQjPyKCiJw4h556hqvR(Rq}XgT?``4v5`!i5ra8G*Q&JoC8E)-v z{5D%ccjbaJ1f!>O=M#_P^#w+VxYHl!(B)QX+`Yro-iO#8562CSMm@Cbd7Av*xe=c8 z$Ax&X?vIfoB)=y>#N<$5HmC2M z$Ugeu3O{UHo`(zjQP?bCv1z~Q{fY8Jm_IBs;{!0MVK;}e9!gI1_^T~%VV~xoh(|}Z z`2M%wd5nv_-EkRp&1SZtLD*+6D1u%;$a5b4<}kV$k0p^yG|7J$X-I?2Yp$yMNWEANiz z^Cf}`yAuCP)gS!6$2PB7`7K1OWeST8{-h5NEzA!?kf~072j3XO zmm}qm`QUwZG=WNAatdGDg=+Xt$WW)ZgMV1Z1CMau+z8&M3nKSh__KE}JCxfyl!CE0 zH}eQcooKri(2duXjd7l^WS2cvqm7p`kEdcmE|h=*>#gNQTFkcRH-0w_9Rs6g@CP;% z`LKXUY+?EKuH$9GaPuim<*85j5r@Lhb>OF$HW9>sjV!x=RT+aC@&_!JHf4uk8$AYOdT?dDigX0$@Qje|YFSlcY$rL+;-Y$|6X ziyd&HOQ|_7g<{d*9HGDi?^_A=X()EU%616E`OV83i2$3XBtZ@Yb;*|Et0S(mbC)oa z@#~~!7329|8);)=`jg1fr~cXth*kO0Bm-Q6w0T>~UI0S0&1;O;UIbZ~dJv)|tA{r}x;?c7Xt zG1b+5`aSP?>@@mOtQaGuMS*c)L9=DjRv*NlZL{b%#N9~WR^J{Mf#2%Y`VyeLzMZUl z`r$5i@SX3E9V@jlS8RkQ9FQr4dNSgmO+$M;)=v68e0-CAlkYbE8ZbEKxSWlTKkXH_ z6feb*nUZ8%bi1Jydn%vhv;Fp<``vgp=KZEE85KLFl$GiMe%K4W0wp$%ZnbR&Vq3VHrHg548LM(oVP4-I3nIeo!EqN(tN1)0=v02KL(S$t9!%`B z)TQulYBHp4v=a+L-LK&Ee5Dn6?AJ@VW{djc>B++b-k*HI%3e5$D7IerD`f-^oszwX z$_5Kghvf@TokHYS@T)y8`{AFr8LEwj8v9O8S*UPJ|1>!6nYvIe2sV*?B__t?ve}L- zFI3aS-ut^duGNHV={fl;NVni|qGYD(v){QJ~ z0n_U432V~F<~apZsYI*-6Gl|r@2k`z9iQV`C@aMYWR9g7&}tqY2AaJJ_Io!|*|4bc zA+^a*Sp2!D`{n0>ZxAiLR_*oiC)M7ai|r0s2*`Jz0jrw6sDD^22{Xb{+_wX#y1j<* z?UlW2=r?2Lyg&@`*B*)* zp*86Fi^y^6?ZvZu%kpa1t@$r$CPR(V`eG3#Z0O$7RF&1$L-6rKHF#{wP{o#(rZS|r zukh|?AfeW6RCtvw=aj zXoB0%B&2(3f!h%sxX>Qkcd8CguaZN-{tQlgb997mflGAA7C4ueb=y@Le(^b@Aymg3 z-f#pH3-t!nZGro|JUqsS+p&baKI;9_j3&DooXk@Kx#QYxKEexu%{Z%#MWT#TyMQZF6Km1((s5~l>hzude8<=TJpLdtGFCxhTBfv)NH135i)d1 zmklyGlm{fqG5Bsk;Z9pvAm(G1+1uhW^qrKIX&+uld`w}BlqiD;oLm{Z`iqjkXNw!n z`VJ?(VkXZkq>nTxWgWK=U&RO@RmS#6o=;lKSTkZqvY( z$a}VBZ9$HcoJ){wC@S=+A$rh@KZG-$H)MPi|+jZ!R7Zy#c9(L%WEoLs74& zA=Q!JI71UVi(Stw%q&OfwwatbP|~e&qN)rYZR=C8(0#;j9NFmHuC4`xV45u8A>In) z4)R*a_K;;}W@Mz!gEt$pl$*Osk)E=%HBPUHu7cgjMvowzLZW2S_IzS;KdtuUjt)7f zdqtZ?^%b-=`c<5(5;eE`L04Y819vq$JJO|9wQr&ea_$6SqpBSjM@n<^0_SRPPm0$h z>NBanmE;?72N!%0Hy-dV0shX`>&HQIRM3oeg`e2_-&GMqrFd*%5{dIM@ZsSK+*XQp zr*6|HgRLvNJdjq-_y8@u%<`DBreX{m-k+yHt zf;S#}ld+fn)NLFbQ67v0ZrdXhu~WCLKZ>oFKqr_1=N+3{FP?n$5nYf}6i%_2@DQp> z0}y)GQ?&xyGcNquJ*S5rsODAAW8aX&< zs{C^1!6EDE{roD^g^6l@k<(JbKfg+RV)sHuwxBJcZ+LiD`+GD56UhtKsjXvO^A|6+ zq9Q)6E_St_`n0N8i;L>zW8XmZ>N1Uv;Nf}_415^pD3_F!gd|5PQC+JhZc#CPkDC_Q z!g)Rb`^RG->#u^AvLe%Up_A7r8Ad`W#$Vxg5^$?;mzC4QljutQtdzLl+fn*EMTaix zC_2JSAq)yVpOb@d)l7`=| zuj4(=ypWsf$hVXC{5)CdgoQEDyuPGvh@qkZ1#5GLtR+kF4ITp@PhaP(sje0q+sRN8 zdLmaG!;@p_29%}Q3%=Va`bMCUmK2iHFSDkAD=YaUpd%qM(PlU{8yBW8;^y$vNvQ%l z`3hpZ7-!Qu>PAWA5j}kt$ahhr{q0o9OEmX~j@94BAFh*R%{Pt49GP=be;T3|@F;GN z#bWL@H<0u(^zXKFb92dWvhk^Kant`6=2IeSo;~u-0|6t%lTci{+~A`lACb3JPw>q>wXG-CF}JKn!k2_tte=naZh0SZq0>iWafaAs zcm_Fmd6E0%t|RehR{XtH7CF7Fh_JC&1GBWWbRxgJ)uRo?>mRt?<48#;MQmmJZL0Wa#-Tm_!i(8|Sm$U(d5Hs`aJhB6NWSy2IEbju;op`%8csVw%) zgeV2gUmWZZB))J&6=`W%rC>`DPXQTOY%M(0t1FGP98ynFr95?{YzV}80m5ssjR=L; zZGU)3Xwv;SL@(R(8|WibQ$aVXmiOEqoN&xiTkV_izdeNa@NoOhtsEh=St(`%Y5^C# z$~kTyjsrIqbWJ-(Q8?c^t!uQgg`WM#ku56qI^o6C@`s+Fw6q4AS0lDv2q%GP(85;#k#Lp`O-gaO^8*!1l0* zIDjLD;`91g*x!d^i1n@HmX~L6*}pGVCLy^vJ6?6gHT3#B8Q-=U?R*l7cqX(>!7jk!=AcC$QK(&kRoR1H@Wiz3F0T?XBPF-h zKjNe<%n^RxQbq~xVPeWiPq$pM98*x}evy?0k|1a$&{SGSLeD*vX^Zp5c5;L0TKIBG z(GrwM`isI13#OBP7G$l(o=1C%i#Hi4_^tQ`L!Y$fqq^c#uzkXk@d{tGWFeswx8z#x zw+M7;@g%?9H<@B)PA%t6%+Q<}OJ&<@zmXSBzLya&@~CuDx(8WnD6$n#w&QtDw2+w& zq6g1sd6}=?oh+gP`spGtJ{{W63%Yiz9!SuEW;lGZ{L1YuKDBbbXLw?21Gmn#W)}$W z4Qq*lqv!`v#xnd#@?K~B7tsRk{o~2egVJq#cpqOPauq)+(~Eri1oxFx^||!!2~=B* zz>1)+6IM>GqroA}PY{j}S^mnz)XXnV;W#%(Y1Un7AAKIf@S(i?{G`+0u^AKuT3f{q zcO)@g?R4#$H6Zzf8~oSoLz6OGOUugSrmtb)bszt0F%DVg-~8^hys(R=_}&iG5|age z93vakQgtO8?9)7PBA`0D&j7|!<9BFr-~LJ{I8g)^XVDdAY<&{8d%LH6z2AOv;^UD& zC&EJ`GKe_yJ9EaxweUQ6@p%u0;7(?1W9$4Yx9aj8uTL`aj;1CD9}UgvpUyBhGQ=F! zV>^ck9xM9#1nfTGFTBDe^r21iNh4uIf`UtSO*@6f?*5l?DM&C&M@~i-4r~!CcFh`V zDFkQt%2;H=-t-w7+9@RbqHPy4SZl@WRCEI|hun^Rz%DNi>pxb1|H|hfj)h%U2QNNHFL|TsMNde z?LByNXGL889MG;~@XnJF&g3BGFrg4hJNbR3AyICH%%`tdj(>VZ178r+wl@PV&ENYO z$@PQJ%Jtp)C?ND&C-)wQFh3^a`3+v6+8QZh1USS1h^n0JZJXWE$VmX@&IGD&B~BM1 z@5_oWEZ5dn4>ERMa{aqlf8LMLMZr;u7N?`d$tgbfndn0kXVt~BTy78dN zdExhw4Q?gZXfD_CbX_=pYnNanbaXd*D;GW@{y4IvvpN5RXv%tC+H+O}-uyfxfB9H6 z<`)#)ySTpPpH)Vfj$c0a0_cz%tGMCr%=O{nU{=N4$zgP?e=di#vUatAh?7c2>Da-p zf2CDOWTS#$a%S!X+Ub7y^!NVpeaftF0q&c*56lE!AB&LSHyhAMiG@pP> z1q}tUwdE9I9Zz@+7a8$@@W(w(CS_!FCt$)Srf)z^{HW@747mB?2l+uE?Ogo~StuF< zRs;UG!|C&Zj|SE9@={tU?-rYXk$ljsg-4~x2u{Fnb@0VPNx+_mD(wSX4f;A(_pQte zQ+rji8G-1x0RG)8dR5})?V{9#AknrNfAbD83#u>8@57w?*q>2pXMkir@Sq2_xFO$4 zbk}T%e=FRY-rjaF)Y!S?ayWa3?(G26>ftC~*y5fV zeH+O4%UZkhnub)&jrd34p21kFEljN|S3<)1vZ8%?IW;S5I7tR|CzG0LbZn~AI|`BL z9r;Nr!k$@fEiI@0V`Q((!-t5qemC3nW%@RbcZXQP{%J$9uB-_&-0`px#1s?4vdfuH3a&+XBU&hp_#@F2918r(M zyc3Hh=NNXIoM1JW-&4eqttGR<%~I6=;F@}W7;n(~F<$(etYhWi;YR{2lT@fyNw`od zFOiL?Dq87?n;y@uHv!;2(7ewaAkZSyYu|Yr8qP_0E~GiL6K%PSl|JKFItyI0FGFA;G5-*`FqAQJ;VpA$8E&oCqw-C#Q_f=29OR#wJp-9;nS)ukyh0pNXfIl-B& zmEoKJDQ($K%G#PLtWb(_slckrnpl;*Ey_^jj^P?&_+6g73pKeoPEzp&agnKGxQO$_k);(T^IEvm#j0(VJOmU|laSx62u1lqfNgwnNRoo{to8W3gk+Rc?_bJNA@L!wPFfvLtyT!$~c5of_pqnPhnb)+4$z(GJ~l6G;S_A_uSX$+r)^d+iIY%d4;lYcCZ?eWK$ z=w>xV_*p8DenWT|K_Av#l`DVI;rL=Gay1{GA>>bbiQ0O5UH27ci!&XSLYP!NEC?$# zG^{W`P!k&;AAfMSKfX^zWocu{N{@&%85y?n^IB^9;*M+$_kVbTYU{ccr&Ty+7pAtx z$51S5YsOmL9V@jDaG_wRrJE1QP?Q}0d(9abK_G*h;<6TamNdJ;z1XK?0lv1eHOP#xvjgEqIyNgN@&-J*b*GH@13yO;30lbL7+#iLiwS6r+Y;%986~l9@L_7dl{1tqq{n9gZ zJ~+HmFEcGfa^DVJsRzQFe1)7{WE*xwAC*5Yp$2Ix9oyRVM?CtN_8QuX4*Zz6S(2>Ma<5-s*l}s;h|&yq=RK zEwKaM@;Iy%m|kBP3%YF&RH|=ozc*U`{CtlX*j@MbC|vpEgqW43_r6z^ z-wr87b9-ih`1y1(u&>MSF0khKMa8MfOX!+e!ynFXWT$6!ch`xed0Ref`8&{sjK1|n zQ_LH%FU-vsXe`ENBD2EHgmvdUKWF&z=w_va33GE_6mP~GG0xrKNx6{`bQruZtL*>s z=T8E0Bt#S^lpNZkj%d~*PI~jTqsDoit^H=u22h#~|AhapBao3OteS`(__kmS*?S@M z9+eQe{j0aeY0mS_y$Li9CsT4Z9Zv1#b#qN_*HPuL+>QY-kpX~EpU0kwd=Cy)^5uhi zFR`QX>v$CAr(mZCd>K zEQS93S|!qp{^KjehV-8#7)L4;TgE*I2C=V&2E6!!n(s=Qzpe>5A4^3p-l){mZmnga zz&%5!y?jsPaIALh2LwRpTQ@ zitfAss#P#%r<5f`V&tQv&ET|hBJxScH<_6OEj1hSZ+xvM2Y4$cN(ANfGSLcnv!beT zXC08pY@Zl1nw(Jbyllw1>ShE?3Bk40L-B^)Z;y|G?)My6Fivg$PCPvBJ`~dp{B|ZG zM8Vy=B^I@P_OAzt-1oce1_M=O^5iNOdN6@pz|eU;3rgY&5}ykm_C&5Q9={zkL^CF>IC22+)^YwEBh{MLtIBhVAp z#+LZJ&M*L{$Xs^uxusM@UKH#CJQLApgm7`BVCp)eIu}>tm^#$X6l{jCx-}yX{CgS# zz2;7Tl9Df43JWs~saZv-piu!`-hqpV8-Q*+z+Sn!LKqKm%O?#>HXUqT#<>+L)~c(0 zxnm=w_=lMJ+0uceC}`LZ7sqN{F^ik8MC4vx_%Xf&-Da9??P`AH;-A@`kVqCz!|>DR z;R-!v++Hz=@Lp$TKAw+#IUT|Xyq105ukH%f@r;ceusQ54(i|H4`fjU8{C&ib|4D0h z#2pkLe5lvYcG6R$shSWYK%m@@iBUHK_3JPCn{q_q@$#0UhogOc698a7gRn6DpH3N< z9wL)EV;c+ljp_<+v&(OAPJnc{{YS~Kr4nr~!;natlUDseGqU(Fze-Vo<7YE1#S6l9 ztDl1ZBRTB23ftU*fXNXIme`&9Z?Xq}Dl011JWTcO?|Hcb#c)#s8_Ei9nJ{g%t?p-(g z+x`#0YhtbYHiaikB&YV8xF@IKnL$h&_9W-(73R%dIBI*_YO!Oj2oGx)_}Z< zO2(mwsgaG%Sb(RN7SLG@%`BFns3@y(yU^J$=<+2RCk9d9a+{AN2*9SL?hPpVi-~Ct zuj(K?Zd23741J6v$L5 z3-jjvS_B9e7zD`Ej|uXPX|W2bwfP)zpT_|mhr>=rT!jCaM4-(866hQ+?PP>2;krq-NaOL{1i%}T* zQAA?o$uR!%EFCvw!*jh=_P&NS>e!5@o1eG3c#35L-ZwGxPan|x{ypqIyF&|TM;hjI zBCQU}j-{9iyqPtW>=+}?pBIeOZJ5d5Gxt)k=>=hpHh)l&E?SK>pP#=sP*PH!KmwwX zCZMe+aW8C~Q$qLMF8@cUm+P~7cS^(4jzYFPr*R8h{N4J8yR2fWHwk=P+3?~>pH@7o zX`8{m_|0(3&RWc@Vnb`1MO}<*7vYviOR@D%WqRJKaz>|eG$L*<}WbKY^ zQ@d3kC`&ahy||{tTz;tg_5e~*7X)sL^+6~FQHO)`&ikV#av7x~%b_CXm=Hlvwf+QS z|BIQk&4E7{jfE|sB3LSQ>Me}?GSF^Tpsh9p<4_3yix#+k)qX9FGtI$36`&9Xw1Xz(TWGBnvEsV0U~# z2}R#{}jEko1`dyKg~_Z^9$Fp14ImX>c`#hjf2(IzZpOQYrWeCR52F;Or$qN*v4-m zQd$GoYirH#?=%2^MGI(g2GzzLfe#kk#$;c(f~K0<%J{Qs)8Tva3Q6&db<108arM@3~}>*qEF$& zhYr=vAY`BxqCrjQ13|$Z7m=IqLpnBybbK+fW2m9X({WMB;C#$Xb?vR0?w`5uOaQKV zzk_;adNnAD1Q(+3<0uVdq@Yw;`CB%1(o#7y`7_X9IRqkxJR9grt-YF5kAjz=pom5P z{f~fJu6DajEU=oAHa0c~KRbs1b`AfvQZWI{Y;B!$J_rj!LP8ZP3CkDTZ=8<$PkJ@f zz?!7LKE&@;nVR*Lh@fcW6VyNV^0Pn1ntqH=!&mDsQK?%FlmC%}YO6^d@Ob>esflwn zkso1t1z)7Ib@lg7`!H<5E-!`^m+PpK%Pqcu`BAdhRT&;OIZJ!K%<)B5Q1UHOCb zrog*l8GgHdOt8+_rBDIYvtjDHy9@?;dZUG$RggbK^1b0Dmb4M?rU{t_@admN>W*J;)iVXwa z2SAdVYK4LjD~OH^jn~}~9?lrocjC-yIy1}3#De;bg~hnPOxlJxu6{xHI5!2?bB89n z)K*h7*=aQlOS{3Zq5zM;;aXkHNu4O*@y8}qN?f=@a=FQ7p6UI1^;<-%uKZX!UMm*- zGnrRNu?!PbRYgS~Tghxj^gF1q zHI51c+EV7zkOXq)5eStuP3!)MyMzmmfs0o#oWIePQ1z3OFzm~$j~niHSuIY$rnUU4n@_|; z^-FiQX4+vWf9}(-!6_LcgOsSkz~ljv-*K~TPPsF@WNUv#8T|TimW!@tigdk1#AxYs zz9awBP!8R;Qx;K0akd<9{` z0LY1>zdM>^NT2Qb_1QZGR-N~xZu8_QuxsP)bnMLk$Z{;}__>ubH&ae(wxde6#7}S53VKGS?PbwRo zo*0EpjsXw)(_DFsfdzN5nI*|W_U=57UG1uK{|SOrLEh$PxgQd4WUNwKxiKgyv~&gC zCSaYAs3__-IiD9y>UR2@SlkU+lF26_Ufm5MBqJ4j)^aH->Cw(-QKDfsR<-ryK(v)Q z4&-1RZ*RiHG>at*Q9xB-_3*uSR(x+)!5HRO23dl*7?<<~=TEr1met)E<|yE*g<+}- zK74E4dR|6(e%Phc?y|56ty}N_5X^oQx*zC*{RBqkxLRYSjxZgp`PL>W;#@xEu zp^}wl)$*}bBwSyGflUH{$Ec`<7qM)2KmluRd|yi2dC(d*x@)a(nZ9lQgO|Upz_l6P zJ)vVGnrs!rbeg0Bw6mi&)y9MI0KZJ=3sFbtp0ul!fEq2Yn4a+g(e3mZ#uOU)c<}kq z(GlIYt{!^;t~O*Qe^a29c*}$_3^2bLLAr^EqeA*V4RQkA7<2ma5$mtbOZvH4l&f4Nz;jH@g3$!FjGQyKsdgwJbD z3<{fI;>*tX;+KL-N37b~86qfl)FUAe!QA^_MFY*z6xZ>%@S&QTPV%4AjM;TgZcJoV z(50p`WN?0l1c>v^Et-WBL!T!Ng?chGZ|&-Tp6fM?8wo>1K}|HCYHbWq>67;mj`QgNYl`wu?+hOqVFp%| zcHceSoq=wW!!gx$DRxNag`IC~iT$y|Q)(6kr>CaE^pbt((BvKiz&MU>tJ=+f>~GTq^GNZJZfthjN{ox{ z>os@LpGmk(o=C+#-c5==VUzo~;h%Om3e~#Kyn*WwLaB0uoS{E#u1DdG^8ajHL1v;R z3ukRnA4V@P?~VKM_BUR@w-tD?fEZ$UxZvBAgA|a;%B${lnRR+z5`@~$k{0y??8N|M zP6RZ@o^uUYlWT)A8gw1Ks~HMssB0VI&vSEIg`1P|UDJvv>QHN0X%<#zhQBxC0sfC< z!1myRGhNi5S4@BGEWp9bP_p_3DG?hfLh9e5tT#?>%C-SBJTlRJ;2fHmk%1W-@$X*2 zKe}#M{t{T29@K~t3RP14^F-^fKPfO(`+EMi7OGFcpSQI$DI}7@laSe|r+arS9E`fp z0`k1$^gFb@`}r1xjYQ>iY6rjJZ;Q@XwHf=7f$c?<(k;kFGtn@CY(S9)I}5)TL}U5y zi6gMK(Iw^?_ho=W6LXU0^ckAp=`T&>v_`$%jV^*L7d1p)ir@*wooS)+i&p?Os1*gE zJAHB{jHvyk+uN7z<&48!sMTsk2yF=j?g zme74v;ybx8(#vCdw;7+LR-bJXP&{_JU5M143uYk*jXR4iEF}H@Lx8$GKKi*6bD?@UI(HSr z^MPIp37Ykn@lw<)fc=ltpi0Y89?=irT?>hNPh!13_=*MqQyRxoEp;akJ~=TpHx6%9 ztarN4a_i<%xPdhcO4FVxL&%gAkRcg8m|{Xg3Q4sqqa}W_Y@2csq8X3wy$Ej|CUKuii!{N~^wetol;uR+`a z3A`H_04TuCI-xT4eLo7jso^Xv&GOZ1IX#I1hUVE)vj9dC_i0yz%~!|Gac=_^H{!>q zrd$sXS-CY5F0N9-AK$1RWRdTxlo%q1va*r`S+!808cZO?{=>udR5aXV{!e(u*GoOa z?8CjSyuzTcqBRDuoZLOSS)h`j{{qFmu0yStV3&#k1KX4YTm4?UXc9 zaIKi^{h%lLV4%pIJkJt|@DdK~w40Wr6vJaeemRWygWx^*?ExFmXE#+$55L<<2Nk+G zdKdo1WqIA*%p2@_o~;I!04>C(_OPLy(4ie$|D?5xZXl6l5P%betPt~g$5^cHtpPJCepU)^i$To@kW9*%+D734Qg1SideIFRv^O#54Xkv8)c z-$H^r_c5o-nPz_UtHXh@w&3oZsvxLE%epT(6yjSwS=yYzR(;|3cIpxd1OJXexfcu` z;V4Lm*t6izL08FT%wbNNZ$C1C2h~rMa3@uIv8l5?u_>Yd&gMs;TF;g*VvUhy(LkA<( z*^)N8-PuN;>aKx3;YIf694L~TKB#ayq3*<*Y zV-6jdX~^c4)p9tv#9r(yWzOJnM>BYKB{%R6;9Ll3hMhnlpmpoJ%fys$Ny0OAAPYex z@5J#`CL+6$zrhuSMNHlP()D85-e$#`*4_pUpb$wU@eaCrC{VmJ9?$K2okJrHxNu1D=@o;jsx zbPr*<;-a2~@LCS~I^<^9klVPaAw;zxB@F;PQCv4m6zGAu&s_US|N`4&wb&=Ds>PZ6y6!)06kIrxJIpwNr` z^faT*jU6yi4MIcquU|4~$i8AEfN6x6QicZMmUre&dL%^74XiWNqJ= zP%JFWjU{ZDaYKn$bec#=NQg0cYAlw}>pF`1fepQ1XJ?%~0|H768R+PcKNiOi9Hp*q z-K|?epSzHWOMhwVbZG>X2XV2tIgz!usoaQMJ^Mynzn4`E+*=dHx)T@RW~M5vSHpV000+2u*$@F-7$Ev4a(`flMly{N_%e#P)Eq!yV@s;P zWC?DZYjWf~&EY0TrO1f}gmmwjww{b}dgUxG1Rz#5l;p|3xOn-5iJW#1=v+5=BRr|!ofe<79MW1{v~e6fQ+PIE_k|^l;2!LR?es)O!)fP5alGk!8KJ|yV8&Ij z8H7z+qV4>uESNKakA!rc7)L<)oN9)&q5=}2Q-uDCcIdvMo$H{H;P78t55B35UT5_7U}C9hMF5JDD7OZ>-w z_7kD~!{atm1uR88!`k>-1EJyJ2j&gLw^?70J&vvH!aivs=_9h_8iW=bca4TOxVwAR z{({?6Z9F;7LG#=9Rom(cxU~e7uBug^uSF?SLKjPd&kt2yO7b=~T%9-T@>5NQ>;P89 zkHpCBDZ|-!zE!3>vLQ{-e2XONNz5_BVYT@Fg9GMnLmVBj&arOfnRS*A|F`loJk{xS zGC5k~-Dgl=U%yGh7$)H($6P#47zux;HuTrp^k*k~oh#S*T>7c3xh&c3ndmzeNEHe4 zG;ytum>UXJLXO=45cn}31{9oCNCoKlB^C(sH2HE8U;;4(4)aItfAi9HUnNG@ek>}U zdw9WysIWsHbG`_nsQvk9+lK;LGhN>oqpayopQ`m*%zm?`JVW3R9~ex6P=s{wx-7)$ zRnPosNC+}*7d#K;L-JqeE5BW}f>qDo9oMlOQ&EKjFsF}GZ$#Pz$) za1edFW@G7J`Wi{iE{+-+kz)MXYhB7%Ei3EI3Mk)UuurN+XQwo!l{HOy4l6*=x}ebU z&E%ikHJ0ZqjPlK;jZRsQ&*7OP<(8>G#oG6LC_Qa!F8Z7OHFNmckF(bOcR+FR0I8G@ z{R3LD9G7X5_si^E;s5M=d6K)Hom=Xat8Y9i(iUl~dp;wN7zeRuZ|A7Z?#~?UuO+)n zpckC{ab#)~xlJLYcrh`8kCxxRWjY5$M>6p7Aj(aXU>o)LC)%x@prd=Z$k_Kxi8K0$xuk8o`%l zB8St6@zW&zR7LNZ3Kty^>y2HJW~>;N=YOeS1d6KZl^n<+-I0SIg0_ z6thuQKY|ic>}0#3={R*T)RdyROW463O^e_Y{>A*3k=fQsxKGI;+XjMOTLn59Vi3M- z3U%y{xwusp{BdNWtEb))i|$J^Nt|xlU8Q~zu9knmSMVzVD6~-evx#|mF_O}Mgz#WU za@q6*43WFUr*7}?z;H=<#?ogqFxu~@?H$o?<^FlT|IC9N0S{0EiK+x63|+_7`P7ap zhx>~5HN}!}xf%@+rz92j;VYyjwYFv=6lex;IRlVwIL>t85z<%xv@3L?(rc!vvCu@A znQ483C_!BaaTdr@_b$T+P;Y)~Ik^;LLPCu$_EcLXCJ^tF`TSXT%+H|YC()|7Sd`kk z+>oCIL<4l*2a8>Bz#9DXtsA=v1uIl<+yUtJXivAK(!lgUL2>2y=TxTR8v*@8zr06* zpG|+!`8t_f!0jAYkBwiM9tY*JZmPl)8puXKE{tI~=ZSiu6lC-TmmgoiL#&z+L%W1p z%yh&*8U}VH8F51fY=9Ug;i@?~lBQDgSCR9lPz7Zk8bP|~r|;C*Xb6Dpu8ter-w)SF zf*=8G+NIuBhW_{Up92#p+W*0W|He80bA|u2*ZJQ&=l>bj|HeT4H_rM02jc&l2mdcu d*^$Q^#TUgRGnzd%CMe*eD61+{C1n)yzW|K5S55!` literal 0 HcmV?d00001 diff --git a/build_step_create_install_cd.sh b/build_step_create_install_cd.sh new file mode 100755 index 0000000..40de1cc --- /dev/null +++ b/build_step_create_install_cd.sh @@ -0,0 +1,133 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh +_read_manifest_vars + +tmp=$WORKTMP/install_cd +iso_build_dir=$tmp/build + +input_image="$WORKTMP/goldenimage/${GOLDEN_IMAGE_NAME}" +output_image_path="$1" +[[ $output_image_path =~ ^/ ]] || output_image_path=$(pwd)/$output_image_path +output_bootcd_path="$2" +[[ $output_bootcd_path =~ ^/ ]] || output_bootcd_path=$(pwd)/$output_bootcd_path +mkdir -p $tmp +rm -rf $iso_build_dir +mkdir -p $iso_build_dir + +reposnap_base=$(_read_build_config DEFAULT centos_reposnap_base) +release_version=$PRODUCT_RELEASE_LABEL +reposnap_base_dir="${reposnap_base}/os/x86_64/" +iso_image_label=$(_read_build_config DEFAULT iso_image_label) +cd_efi_dir="${reposnap_base_dir}/EFI" +cd_images_dir="${reposnap_base_dir}/images" +cd_isolinux_dir="${reposnap_base_dir}/isolinux" + +remove_extra_slashes_from_url() { + echo $1 | sed -re 's#([^:])//+#\1/#g' +} + +get_nexus() { + $scriptdir/nexus3_dl.sh \ + $nexus_url \ + $(basename $nexus_reposnaps) \ + ${reposnap_base#$nexus_reposnaps/}/os/x86_64 $@ +} + +wget_dir() { + local url=$1 + echo $url | grep -q /$ || _abort "wget path '$url' must end with slash for recursive wget" + # if any extra slashes within path, it messes up the --cut-dirs count + url=$(remove_extra_slashes_from_url $url) + # count cut length in case url depth changes + cut_dirs=$(echo $url | sed -re 's|.*://[^/]+/(.+)|\1|' -e 's|/$||' | grep -o / | wc -l) + wget -N -r --no-host-directories --no-verbose --cut-dirs=${cut_dirs} --reject index.html* --no-parent $url +} + +pushd $iso_build_dir + +# Get files needed for generating CD image. +if echo $reposnap_base_dir | grep -E "https?://nexus3"; then + nexus_url=$(_read_build_config DEFAULT nexus_url) + nexus_reposnaps=$(_read_build_config DEFAULT nexus_reposnaps) + get_nexus "EFI/BOOT" "EFI/BOOT/fonts" + get_nexus "images:*efiboot.img" "images/pxeboot" + get_nexus "isolinux" +else + wget_dir ${cd_efi_dir}/ + wget_dir ${cd_images_dir}/ + rm -rf images/boot.iso + sync + wget_dir ${cd_isolinux_dir}/ +fi +chmod +w -R isolinux/ EFI/ images/ + +if [ -e $scriptdir/isolinux/isolinux.cfg ]; then + cp $scriptdir/isolinux/isolinux.cfg isolinux/isolinux.cfg +else + sed -i "s/^timeout.*/timeout 100/" isolinux/isolinux.cfg + sed -i "s/^ - Press.*/Beginning the cloud installation process/" isolinux/boot.msg + sed -i "s/^#menu hidden/menu hidden/" isolinux/isolinux.cfg + sed -i "s/menu default//" isolinux/isolinux.cfg + sed -i "/^label linux/amenu default" isolinux/isolinux.cfg + sed -i "/append initrd/ s/$/ console=tty0 console=ttyS1,115200/" isolinux/isolinux.cfg +fi +cp -f $scriptdir/akraino_splash.png isolinux/splash.png + +popd + +pushd $tmp + + # Copy latest kernel and initrd-provisioning from boot dir +virt-copy-out -a $input_image /boot/ ./ +chmod u+w boot/ +rm -f $iso_build_dir/isolinux/vmlinuz $iso_build_dir/isolinux/initrd.img +KVER=`ls -lrt boot/vmlinuz-* |grep -v rescue |tail -n1 |awk -F 'boot/vmlinuz-' '{print $2}'` +cp -fp boot/vmlinuz-${KVER} $iso_build_dir/isolinux/vmlinuz +cp -fp boot/initrd-provisioning.img $iso_build_dir/isolinux/initrd.img +rm -rf boot/ + +echo "Generating boot iso" +_run_cmd genisoimage -U -r -v -T -J -joliet-long \ + -V "${release_version}" -A "${release_version}" -P ${iso_image_label} \ + -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 \ + -boot-info-table -eltorito-alt-boot -e images/efiboot.img -no-emul-boot \ + -o boot.iso $iso_build_dir +_publish_image $tmp/boot.iso $output_bootcd_path + +cp -f ${input_image} $iso_build_dir/ + +# Keep the placeholder +mkdir -p $iso_build_dir/rpms + +echo "Generating product iso" +_run_cmd genisoimage -U -r -v -T -J -joliet-long \ + -V "${release_version}" -A "${release_version}" -P ${iso_image_label} \ + -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 \ + -boot-info-table -eltorito-alt-boot -e images/efiboot.img -no-emul-boot \ + -o release.iso $iso_build_dir +_run_cmd isohybrid $tmp/release.iso +_publish_image $tmp/release.iso $output_image_path + +echo "Clean up to preserve workspace footprint" +rm -f $iso_build_dir/$(basename ${input_image}) +rm -rf $iso_build_dir/rpms + +popd diff --git a/build_step_create_localrepo.sh b/build_step_create_localrepo.sh new file mode 100755 index 0000000..4bb22a0 --- /dev/null +++ b/build_step_create_localrepo.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh + +_run_cmd_as_step _create_localrepo diff --git a/build_step_create_rpms.sh b/build_step_create_rpms.sh new file mode 100755 index 0000000..3d45388 --- /dev/null +++ b/build_step_create_rpms.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh + +_get_projects_all() { repo list -g torpm -n | sort; } + +_get_project_dirs() { + for i in $@; do + repo info -l $i | grep "^Mount path: " | awk '{print $3}' | tr '\n' ' ' + done +} + +build_rpms() +{ + local work=$1 + shift + rm -rf $work + mkdir -p $work + + CENTOS_SOURCES="$(_read_build_config rpm centos_sources)" \ + $RPM_BUILDER/makebuild.py \ + -m $RPM_BUILDER_SETTINGS \ + -w $work \ + $@ #-v --nowipe +} + +# build one or all projects +if [ -n "$1" ]; then + projects_to_build=$@ +else + projects_to_build="$MANIFEST_PATH $(_get_project_dirs $(_get_projects_all))" + _run_cmd $LIBDIR/prepare_manifest.sh +fi + +rpmwork=$WORKTMP/rpms +build_rpms $rpmwork $projects_to_build +_add_rpms_to_repos_from_workdir $rpmwork diff --git a/build_step_create_yum_repo_files.sh b/build_step_create_yum_repo_files.sh new file mode 100755 index 0000000..c08ae0c --- /dev/null +++ b/build_step_create_yum_repo_files.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh + +config_ini=${1:-$BUILD_CONFIG_INI} +output_dir=${2:-$REPO_FILES} +[ -f "$config_ini" ] || _abort "Config INI '$config_ini' not found" + +gen() { + PYTHONPATH=$LIBDIR python $LIBDIR/tools/script/generate_repo_files.py \ + --config-ini $config_ini \ + --output-dir $output_dir \ + $1 +} + +mkdir -p $output_dir + +gen localrepo +gen repositories +gen baseimage-repositories diff --git a/build_step_golden_image.sh b/build_step_golden_image.sh new file mode 100755 index 0000000..2dc1a37 --- /dev/null +++ b/build_step_golden_image.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh + +output_image_path="$1" +[[ $output_image_path =~ ^/ ]] || output_image_path=$(pwd)/$output_image_path +rpm_info_output_dir=$2 +[[ $rpm_info_output_dir =~ ^/ ]] || rpm_info_output_dir=$(pwd)/$rpm_info_output_dir + +docker_dib_image=dib:2.0 +_load_docker_image $docker_dib_image + +docker run \ + --rm \ + --privileged \ + -v /dev:/dev \ + -v $WORK:/work \ + $docker_dib_image \ + $(realpath --relative-to $WORK $scriptdir)/create_golden_image.sh + +_publish_image ${TMP_GOLDEN_IMAGE}.qcow2 $output_image_path/${GOLDEN_IMAGE_NAME} + +input_dir=$WORKTMP/rpmdata +mkdir $input_dir +cp -r \ + $BUILD_CONFIG_INI $RPMLISTS/rpm_info_installed $RPMLISTS/yum_info_installed $RPMLISTS/crypto_rpms.json $RPMLISTS/boms \ + $input_dir + +$LIBDIR/create_rpmdata_in_docker.sh \ + $input_dir \ + $RPMLISTS \ + -v + diff --git a/build_step_prepare.sh b/build_step_prepare.sh new file mode 100755 index 0000000..57d66cc --- /dev/null +++ b/build_step_prepare.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh +_read_manifest_vars + +_run_cmd_as_step _initialize_work_dirs + +_run_cmd_as_step $LIBDIR/repo_summary.sh + +_run_cmd_as_step $LIBDIR/create_mock_config.sh diff --git a/create_golden_image.sh b/create_golden_image.sh new file mode 100755 index 0000000..19dad38 --- /dev/null +++ b/create_golden_image.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +set -e +set -o pipefail + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh + +BASE_IMAGE_URL=${BASE_IMAGE_URL:-$(_read_build_config DEFAULT base_image)} +CENTOS_SNAP=${CENTOS_SNAP:-$(_read_build_config DEFAULT centos_reposnap)} + +BASE_IMAGE_NAME=`echo $BASE_IMAGE_URL | awk -F "/" '{print $NF}'` +BASE_IMAGE_SIZE="8GiB" + +wget_args="" +[ -n "$GOLDEN_BASE_IMAGE_FETCH_USER" ] && wget_args="$wget_args --http-user=$GOLDEN_BASE_IMAGE_FETCH_USER" +[ -n "$GOLDEN_BASE_IMAGE_FETCH_PASSWORD" ] && wget_args="$wget_args --http-password=$GOLDEN_BASE_IMAGE_FETCH_PASSWORD" + +fetch_image() { + sourceurl=$1 + echo "Download $sourceurl to $WORKTMP/base-img" + mkdir -pv $WORKTMP/base-img + pushd $WORKTMP/base-img + _run_cmd wget --no-check-certificate --no-verbose -N \ + --auth-no-challenge $wget_args \ + $sourceurl + popd +} + +fetch_image $BASE_IMAGE_URL +cp $MANIFEST_PATH/packages.yaml $scriptdir/dib_elements/myproduct/package-installs.yaml + +DIB_DEBUG_TRACE=1 \ + FS_TYPE=xfs \ + PACKAGES_TO_INSTALL="$(_get_package_list install)" \ + PACKAGES_TO_UNINSTALL="$(_get_package_list uninstall)" \ + DIB_RPMLISTS=$RPMLISTS \ + DIB_CHECKSUM=$CHECKSUM_DIR \ + DIB_LOCAL_REPO=$REPO_DIR \ + DIB_DISTRIBUTION_MIRROR=$CENTOS_SNAP \ + DIB_YUM_REPO_CONF="${REPO_FILES}/repositories.repo ${REPO_FILES}/localrepo.repo" \ + DIB_LOCAL_IMAGE=$WORKTMP/base-img/$BASE_IMAGE_NAME \ + ELEMENTS_PATH=$scriptdir/dib_elements/ \ + /usr/bin/disk-image-create --root-label img-rootfs --image-size $BASE_IMAGE_SIZE vm centos7 selinux-permissive myproduct -o $TMP_GOLDEN_IMAGE + +rm -rf $WORKTMP/base-img diff --git a/create_mock_config.sh b/create_mock_config.sh new file mode 100755 index 0000000..701cbc1 --- /dev/null +++ b/create_mock_config.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh + +config_ini=${1:-$BUILD_CONFIG_INI} +output_repo_files_dir=${2:-$REPO_FILES} +output_mock_dir=${2:-$(dirname $RPM_BUILDER_SETTINGS)} +[ -f "$config_ini" ] || _abort "Config INI '$config_ini' not found" + +# Create .repo files +$LIBDIR/build_step_create_yum_repo_files.sh $config_ini $output_repo_files_dir + +# Create mock config +mkdir -p $output_mock_dir +cp $LIBDIR/mock/logging.ini $output_mock_dir/ +cp $LIBDIR/mock/site-defaults.cfg $output_mock_dir/ +mock_cfg=$output_mock_dir/mock.cfg +sed -e "/#REPOSITORIES#/r $output_repo_files_dir/repositories.repo" $LIBDIR/mock/mock.cfg.template > $mock_cfg +sed -i \ + -e "s/#RPM_PACKAGER#/\"$(_read_build_config $config_ini rpm packager)\"/" \ + -e "s/#RPM_VENDOR#/\"$(_read_build_config $config_ini rpm vendor)\"/" \ + -e "s/#RPM_LICENSE#/\"$(_read_build_config $config_ini rpm license)\"/" \ + -e "s/#RPM_RELEASE_ID#/\"$(_read_build_config $config_ini rpm release_id)\"/" \ + $mock_cfg + +docker_sock=/var/run/docker.sock +if [ -S "$docker_sock" ]; then + sed -i "/#ADDITIONAL_CONFIG_OPTS#/a config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('$docker_sock', '$docker_sock'))" $mock_cfg +fi +if [ -n "$DOCKER_HOST" ]; then + sed -i "/#ADDITIONAL_CONFIG_OPTS#/a config_opts['environment']['DOCKER_HOST'] = '${DOCKER_HOST}:2375'" $mock_cfg +fi + +# HACK ?? +# include kernels in order for yum development group to install +sed -i -e 's/exclude=kernel/#exclude=kernel/' $mock_cfg + +echo "Wrote mock configuration to: $output_mock_dir" diff --git a/create_rpmdata_in_docker.sh b/create_rpmdata_in_docker.sh new file mode 100755 index 0000000..84d2472 --- /dev/null +++ b/create_rpmdata_in_docker.sh @@ -0,0 +1,71 @@ +#!/bin/sh +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh + +docker_image=buildtools:2.0 +_load_docker_image $docker_image + +function _resolve_abs_path() { + if ! echo $1 | grep -q "^/"; then + echo $(pwd)/$1 + else + echo $1 + fi +} + +input_dir=$(_resolve_abs_path ${1:?ERROR, please give input dir as argument}) +output_dir=$(_resolve_abs_path ${2:?ERROR, please give output dir as argument}) +shift; shift + +# Run! +input_mp=/input +output_mp=/output +docker run \ + --rm \ + -e PYTHONPATH=/work \ + -e BUILD_URL -e JENKINS_USERNAME -e JENKINS_TOKEN -e WORKSPACE \ + -v $scriptdir:/work \ + -v $input_dir:$input_mp \ + -v $output_dir:$output_mp \ + -w /work \ + $docker_image \ + python tools/script/create_rpm_data.py \ + --build-config-path $input_mp/build_config.ini \ + --yum-info-path $input_mp/yum_info_installed \ + --rpm-info-path $input_mp/rpm_info_installed \ + --crypto-info-path $input_mp/crypto_rpms.json \ + --boms-path $input_mp/boms \ + --output-json $output_mp/rpmdata.json \ + --output-csv $output_mp/rpmdata.csv \ + --output-ms-csv $output_mp/rpmdata-ms.csv \ + --output-rpmlist $output_mp/rpmlist \ + $* + +docker run \ + --rm \ + -e PYTHONPATH=/work \ + -v $scriptdir:/work \ + -v $output_dir:$output_mp \ + -w /work \ + $docker_image \ + python tools/script/process_rpmdata.py \ + --rpmdata-path $output_mp/rpmdata.json \ + --output-components $output_mp/components.json \ + --output-components-csv $output_mp/components-ms.csv \ + diff --git a/dib_elements/myproduct/cleanup.d/50-copy-build-details-out b/dib_elements/myproduct/cleanup.d/50-copy-build-details-out new file mode 100755 index 0000000..7732c3c --- /dev/null +++ b/dib_elements/myproduct/cleanup.d/50-copy-build-details-out @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +[ -n "$TARGET_ROOT" ] +mv $TARGET_ROOT/root/yum_info_installed $DIB_RPMLISTS +mv $TARGET_ROOT/root/rpm_info_installed $DIB_RPMLISTS +mv $TARGET_ROOT/root/boms $DIB_RPMLISTS +mv $TARGET_ROOT/root/binary_checksum.md5 $DIB_CHECKSUM +mv $TARGET_ROOT/root/binary_checksum.sha256 $DIB_CHECKSUM +mv $TARGET_ROOT/crypto_rpms.json $DIB_RPMLISTS diff --git a/dib_elements/myproduct/extra-data.d/01-copy-extra-data b/dib_elements/myproduct/extra-data.d/01-copy-extra-data new file mode 100755 index 0000000..175af75 --- /dev/null +++ b/dib_elements/myproduct/extra-data.d/01-copy-extra-data @@ -0,0 +1,20 @@ +#!/bin/sh +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +SCRIPT_DIR="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" + +cp -rf ${SCRIPT_DIR}/* $TMP_HOOKS_PATH/ diff --git a/dib_elements/myproduct/extra-data.d/collect_ecc.py b/dib_elements/myproduct/extra-data.d/collect_ecc.py new file mode 100644 index 0000000..56f0d2a --- /dev/null +++ b/dib_elements/myproduct/extra-data.d/collect_ecc.py @@ -0,0 +1,128 @@ +#!/usr/bin/python +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +import json +import sys + +# following libs are found from openssl, libgcrypto, openssh with rpm -qR +# in the future, the libs need to be updated with exact version number + +input_capabilities = [] +output_rpms = [] +input_packages = [ + 'openssl', + 'openssh', + 'p11-kit', + 'openssl-libs', + 'libgcrypt', + 'libgcrypt.so', + 'python2-cryptography', + 'sshpass', + 'm2crypto', + 'erlang-crypto', + 'openssh-clients', + 'python2-passlib', + 'python-paramiko', + 'python-keyring', + 'python2-asn1crypto', + 'python2-cryptography', + 'python2-pyasn1', + 'xstatic-jsencrypt-common', + 'krb5-libs' + ] +crypto_capabilities = [ + 'rtld(', + 'libcrypt.so' + ] + + +def output_rpm_command(lib, command): + command.append(lib) + try: + output = subprocess.check_output(command) + except: + output = "" + return output + + +def build_capability_list_dynamically(): + map_dict = {} + command = ['rpm', + '-qa', + '--queryformat', + '[%{=NAME}-%{VERSION}-%{RELEASE}.%{ARCH} %{PROVIDES}\n]'] + output = subprocess.check_output(command) + for item in output.splitlines(): + items = item.split(' ') + if items: + if items[0] in map_dict: + capa_list = map_dict[items[0]] + capa_list.append(items[1]) + map_dict[items[0]] = capa_list + else: + map_dict[items[0]] = [items[1]] + for rpms, caps in map_dict.items(): + for cap in caps: + for item in crypto_capabilities: + if cap.startswith(item): + if cap not in input_capabilities: + input_capabilities.append(cap) + command = ['rpm', '-q', '--whatprovides'] + output = output_rpm_command(cap, command) + name = output.strip('\n') + result = filter(lambda rpm: rpm['name'] == name, output_rpms) + if not result: + output_rpms.append({'name': name, 'provides': [cap]}) + else: + sources_list = result[0]['provides'] + if cap not in sources_list: + sources_list.append(cap) + + +def capability_dependencies_with_whatrequires(): + for capability in input_capabilities: + command = ['rpm', '-q', '--whatrequires'] + output = output_rpm_command(capability, command) + if output: + for item in output.splitlines(): + result = filter(lambda rpm: rpm['name'] == item, output_rpms) + if not result: + output_rpms.append({'name': item, 'requires': [capability]}) + else: + sources_list = result[0]['requires'] + if capability not in sources_list: + sources_list.append(capability) + + +def package_dependencies_with_provides(): + for package in input_packages: + command = ['rpm', '-q'] + name = output_rpm_command(package, command) + name = name.strip('\n') + output_rpms.append({'name': name, 'requires': []}) + command = ['rpm', '-ql', '--provides'] + output = output_rpm_command(package, command) + for item in output.splitlines(): + input_capabilities.append(item) + + +if __name__ == '__main__': + build_capability_list_dynamically() + package_dependencies_with_provides() + capability_dependencies_with_whatrequires() + rpms_json = json.dumps(output_rpms, sort_keys=True, indent=4) + with open(sys.argv[1], "w") as f: + f.write(rpms_json) diff --git a/dib_elements/myproduct/finalise.d/01-remove-old-kernels b/dib_elements/myproduct/finalise.d/01-remove-old-kernels new file mode 100755 index 0000000..c207fb6 --- /dev/null +++ b/dib_elements/myproduct/finalise.d/01-remove-old-kernels @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +rpm -qa kernel | sort -V | head -n -1 | xargs rpm -e || true + diff --git a/dib_elements/myproduct/finalise.d/99-collect-rpm-info b/dib_elements/myproduct/finalise.d/99-collect-rpm-info new file mode 100755 index 0000000..f3cc6b8 --- /dev/null +++ b/dib_elements/myproduct/finalise.d/99-collect-rpm-info @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +yum info installed > /root/yum_info_installed +rpm -qai --queryformat "Obsoletes : [%{OBSOLETES},]\n" > /root/rpm_info_installed + +# Collect bills of material, RPM subcomponent lists +boms_dir=/root/boms +mkdir $boms_dir +for bom_path in $(rpm -qal | egrep "/[^/]+-bom.json$" || :); do + cp $bom_path $boms_dir/$(rpm -qf $bom_path) +done diff --git a/dib_elements/myproduct/finalise.d/99-create-bonding-soft-dep b/dib_elements/myproduct/finalise.d/99-create-bonding-soft-dep new file mode 100755 index 0000000..f1dc245 --- /dev/null +++ b/dib_elements/myproduct/finalise.d/99-create-bonding-soft-dep @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +echo "softdep bonding pre: mlx5_core ixgbe" > /etc/modprobe.d/bonding.conf diff --git a/dib_elements/myproduct/finalise.d/99-fix-grub-console b/dib_elements/myproduct/finalise.d/99-fix-grub-console new file mode 100755 index 0000000..951e39f --- /dev/null +++ b/dib_elements/myproduct/finalise.d/99-fix-grub-console @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +sed -i -e "s/console=ttyS0,115200/console=ttyS1,115200/g" /etc/default/grub +grub2-mkconfig -o /boot/grub2/grub.cfg diff --git a/dib_elements/myproduct/finalise.d/99-generate-binary-checksum b/dib_elements/myproduct/finalise.d/99-generate-binary-checksum new file mode 100755 index 0000000..1668ceb --- /dev/null +++ b/dib_elements/myproduct/finalise.d/99-generate-binary-checksum @@ -0,0 +1,63 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +md5sum $(find / ! -path "/tmp/*" -executable -type f) > /root/binary_checksum.md5 +sha256sum $(find / ! -path "/tmp/*" -executable -type f) > /root/binary_checksum.sha256 + +# Remove checksum that has been modified during deployment +changed_checksums=( "resolv.conf" "mariadb.sg" "rabbitmq-server.sg" "audit.rules" "influxdb.conf" "influxdb-1" ) +for checksum in ${changed_checksums[@]} +do + sed -i "/${checksum}/d" /root/binary_checksum.md5 + sed -i "/${checksum}/d" /root/binary_checksum.sha256 +done + +# Add checksums that will be added during deployment +checksums_to_add=(\ +"/etc/openstack-dashboard/enabled/_2200_ironic.py" \ +"/etc/openstack-dashboard/enabled/_2200_ironic.pyc" \ +"/etc/openstack-dashboard/enabled/_2200_ironic.pyo" \ +"/etc/openstack-dashboard/local_settings" \ +"/etc/openstack-dashboard/nova_policy.json" \ +"/etc/openstack-dashboard/neutron_policy.json" \ +"/etc/openstack-dashboard/keystone_policy.json" \ +"/etc/openstack-dashboard/glance_policy.json" \ +"/etc/openstack-dashboard/cinder_policy.json" \ +"/etc/openstack-dashboard/cinder_policy.d/consistencygroup.yaml" \ +"/etc/openstack-dashboard/nova_policy.d/api-extensions.yaml" \ +) +for f in ${checksums_to_add[@]}; do + if test -f ${f}; then + md5sum ${f} >> /root/binary_checksum.md5 + sha256sum ${f} >> /root/binary_checksum.sha256 + fi +done + +# Include docker images +docker_images=$( find /var/lib/crf-images/docker-images/infra/ -type f ) || : +if [ -n "${docker_images}" ]; then + md5sum ${docker_images} >> /root/binary_checksum.md5 + sha256sum ${docker_images} >> /root/binary_checksum.sha256 +fi + +# Sort the checksum files +sort -k2 -o /root/binary_checksum.md5 /root/binary_checksum.md5 +sort -k2 -o /root/binary_checksum.sha256 /root/binary_checksum.sha256 diff --git a/dib_elements/myproduct/finalise.d/99-remove-dhcp-all-interfaces-udev-rules b/dib_elements/myproduct/finalise.d/99-remove-dhcp-all-interfaces-udev-rules new file mode 100755 index 0000000..f8382d4 --- /dev/null +++ b/dib_elements/myproduct/finalise.d/99-remove-dhcp-all-interfaces-udev-rules @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +rm -rf /etc/udev/rules.d/99-dhcp-all-interfaces.rules +# below file is coming from the base image we use +rm -rf /etc/sysconfig/network-scripts/ifcfg-eth0 diff --git a/dib_elements/myproduct/finalise.d/99-set-sshd-config-defaults b/dib_elements/myproduct/finalise.d/99-set-sshd-config-defaults new file mode 100755 index 0000000..103ddc8 --- /dev/null +++ b/dib_elements/myproduct/finalise.d/99-set-sshd-config-defaults @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +sed -i 's/^#*UseDNS .*/UseDNS no/' /etc/ssh/sshd_config +sed -i 's/^#*GSSAPIAuthentication .*/GSSAPIAuthentication no/' /etc/ssh/sshd_config + diff --git a/dib_elements/myproduct/install.d/50-set-rootpasswd b/dib_elements/myproduct/install.d/50-set-rootpasswd new file mode 100755 index 0000000..4a7ea58 --- /dev/null +++ b/dib_elements/myproduct/install.d/50-set-rootpasswd @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script should be removed as soon as possible. +# It is hardcoding root password to root. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +echo "root:root" | chpasswd + +sed -i 's/.*PermitRootLogin yes/PermitRootLogin yes/' /etc/ssh/sshd_config +sed -i 's/.*PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config diff --git a/dib_elements/myproduct/post-install.d/50-remove-local-repofile b/dib_elements/myproduct/post-install.d/50-remove-local-repofile new file mode 100755 index 0000000..aad0d37 --- /dev/null +++ b/dib_elements/myproduct/post-install.d/50-remove-local-repofile @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +rm -rf /etc/yum.repos.d/localrepo.repo diff --git a/dib_elements/myproduct/post-install.d/98-collect-ecc-packages b/dib_elements/myproduct/post-install.d/98-collect-ecc-packages new file mode 100755 index 0000000..69fccd9 --- /dev/null +++ b/dib_elements/myproduct/post-install.d/98-collect-ecc-packages @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +cp -f /tmp/in_target.d/collect_ecc.py . +python collect_ecc.py crypto_rpms.json +rm -f collect_ecc.py diff --git a/dib_elements/myproduct/post-install.d/99-validate-packages-to-install b/dib_elements/myproduct/post-install.d/99-validate-packages-to-install new file mode 100755 index 0000000..028330b --- /dev/null +++ b/dib_elements/myproduct/post-install.d/99-validate-packages-to-install @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" + +rpm -q --queryformat '' $PACKAGES_TO_INSTALL + diff --git a/dib_elements/myproduct/pre-install.d/01-enable-yum-priorities b/dib_elements/myproduct/pre-install.d/01-enable-yum-priorities new file mode 100755 index 0000000..6c472d2 --- /dev/null +++ b/dib_elements/myproduct/pre-install.d/01-enable-yum-priorities @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +pkgs="yum-plugin-priorities" +yum install -y $pkgs +rpm -q --queryformat '' $pkgs diff --git a/dib_elements/myproduct/root.d/50-local-repo b/dib_elements/myproduct/root.d/50-local-repo new file mode 100755 index 0000000..bb9bd64 --- /dev/null +++ b/dib_elements/myproduct/root.d/50-local-repo @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +[ -n "$TARGET_ROOT" ] + +# Remove all the repo file content in the target root. +for file in `ls $TMP_MOUNT_PATH/etc/yum.repos.d/*.repo`; do + sudo echo > $file +done + +# mount the local repo dir on target loop device +mkdir -p $TARGET_ROOT/$DIB_LOCAL_REPO +sudo mount --bind $DIB_LOCAL_REPO $TARGET_ROOT/$DIB_LOCAL_REPO diff --git a/dib_elements/myproduct/root.d/51-rm-grub-defaults b/dib_elements/myproduct/root.d/51-rm-grub-defaults new file mode 100755 index 0000000..313bb70 --- /dev/null +++ b/dib_elements/myproduct/root.d/51-rm-grub-defaults @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +[ -n "$TARGET_ROOT" ] + +# Empty the grub defaults file coming from the base image. +# Native DIB element running in the finalise stage is appending to the original file. +# This was resulting in duplicate entries. +echo > $TMP_MOUNT_PATH/etc/default/grub diff --git a/docker-context/Dockerfile-buildtools b/docker-context/Dockerfile-buildtools new file mode 100644 index 0000000..65f4302 --- /dev/null +++ b/docker-context/Dockerfile-buildtools @@ -0,0 +1,20 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM fedora:27 + +RUN dnf install -y python wget yum-utils && \ + pip install requests +WORKDIR /work + diff --git a/docker-context/Dockerfile-dib b/docker-context/Dockerfile-dib new file mode 100644 index 0000000..fa1359f --- /dev/null +++ b/docker-context/Dockerfile-dib @@ -0,0 +1,32 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM centos:7.5.1804 +RUN \ + yum-config-manager --add-repo http://mirror.centos.org/centos/7/cloud/x86_64/openstack-pike/ && \ + yum-config-manager --add-repo https://trunk.rdoproject.org/centos7-pike/57/c7/57c7be250c919c04b51361d4d42e95818cfec5a5_15fc9723 && \ + yum install --nogpgcheck -y diskimage-builder \ + git \ + python \ + wget \ + which \ + findutils \ + systemd-udev \ + PyYAML \ + parted \ + sudo \ + e2fsprogs \ + xfsprogs + +WORKDIR /work diff --git a/docker-context/README b/docker-context/README new file mode 100644 index 0000000..a701673 --- /dev/null +++ b/docker-context/README @@ -0,0 +1,31 @@ +Rules +----- + +- CI must pull images from the intranet and not from the internet +- CI must use always versioned tags, not the "latest" + + +How to use? +----------- + +e.g. + $ curl -L http:///dib:1.2.tar | docker load + + +How to develop? +--------------- + +#. Create local image + + .. code-block:: + + $ docker build --rm -f Dockerfile-dib -t dib: . + e.g. + $ docker build --rm -f Dockerfile-dib -t dib:1.2 .``` + +#. Upload to file server + + .. code-block:: + + $ docker save dib:1.2 > dib:1.2.tar + $ scp ./dib\:1.2.tar @:/var/www// diff --git a/isolinux/isolinux.cfg b/isolinux/isolinux.cfg new file mode 100644 index 0000000..bbd96eb --- /dev/null +++ b/isolinux/isolinux.cfg @@ -0,0 +1,48 @@ +default vesamenu.c32 +timeout 100 + +display boot.msg + +# Clear the screen when exiting the menu, instead of leaving the menu displayed. +# For vesamenu, this means the graphical background is still displayed without +# the menu itself for as long as the screen remains in graphics mode. +menu clear +menu background splash.png +menu title Cloud +menu vshift 8 +menu rows 18 +menu margin 8 +menu hidden + +menu color border 0 #ffffffff #ee000000 std +menu color title 0 #ffffffff #ee000000 std +menu color disabled 0 #ffffffff #ee000000 std +menu color sel 0 #ffffffff #85000000 std +menu color unsel 0 #ffffffff #ee000000 std +menu color pwdheader 0 #ff000000 #99ffffff rev +menu color pwdborder 0 #ff000000 #99ffffff rev +menu color pwdentry 0 #ff000000 #99ffffff rev +menu color hotkey 0 #ff00ff00 #ee000000 std +menu color hotsel 0 #ffffffff #85000000 std +menu color cmdmark 0 #ffffffff #ee000000 std +menu color cmdline 0 #ffffffff #ee000000 std + +# Do not display the actual menu unless the user presses a key. All that is displayed is a timeout message. + +menu tabmsg Press Tab for full configuration options on menu items. + +menu separator # insert an empty line +menu separator # insert an empty line + +label linux +menu default + menu label ^Install Cloud + kernel vmlinuz + append initrd=initrd.img inst.stage2=hd:LABEL=CentOS\x207\x20x86_64 quiet console=tty0 console=ttyS1,115200 + +label check + menu label Test this ^media & install Cloud + kernel vmlinuz + append initrd=initrd.img inst.stage2=hd:LABEL=CentOS\x207\x20x86_64 rd.live.check quiet console=tty0 console=ttyS1,115200 + +menu end diff --git a/lib.sh b/lib.sh new file mode 100644 index 0000000..671d041 --- /dev/null +++ b/lib.sh @@ -0,0 +1,233 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o pipefail +set -e + +LIBDIR="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" + +PUBLISH_RESULTS="${PUBLISH_RESULTS:-false}" +VIRT_CUSTOMIZE_MEM="${VIRT_CUSTOMIZE_MEM:-}" +VIRT_CUSTOMIZE_SMP="${VIRT_CUSTOMIZE_SMP:-}" +PARALLEL_BUILD_TIMEOUT="${PARALLEL_BUILD_TIMEOUT:-0}" +ENABLE_GOLDEN_IMAGE_ROOT_PASSWORD="${ENABLE_GOLDEN_IMAGE_ROOT_PASSWORD:-true}" +GOLDEN_BASE_IMAGE_TAR_URL=${GOLDEN_BASE_IMAGE_TAR_URL:-} +GOLDEN_BASE_IMAGE_FETCH_USER=${GOLDEN_BASE_IMAGE_FETCH_USER:-} +GOLDEN_BASE_IMAGE_FETCH_PASSWORD=${GOLDEN_BASE_IMAGE_FETCH_PASSWORD:-} + +WORK=$(dirname $(dirname $LIBDIR)) +RPM_BUILDER=$(find $WORK -maxdepth 2 -type d -name rpmbuilder) + +WORKTMP=$WORK/tmp +WORKLOGS=$WORKTMP/logs +DURATION_LOG=$WORKLOGS/durations.log +MANIFEST_PATH=$WORK/.repo/manifests +BUILD_CONFIG_INI=$WORK/.repo/manifests/build_config.ini +GOLDEN_IMAGE_NAME=guest-image.img +TMP_GOLDEN_IMAGE=$WORKTMP/$GOLDEN_IMAGE_NAME + +WORKRESULTS=$WORK/results +REPO_FILES=$WORKRESULTS/repo_files +REPO_DIR=$WORKRESULTS/repo +SRC_REPO_DIR=$WORKRESULTS/src_repo +RPMLISTS=$WORKRESULTS/rpmlists +CHECKSUM_DIR=$WORKRESULTS/bin_checksum +RESULT_IMAGES_DIR=$WORKRESULTS/images +RPM_BUILDER_SETTINGS=$WORKTMP/mocksettings/mock.cfg + +function _read_build_config() +{ + local config_ini=$BUILD_CONFIG_INI + if [[ -f "$1" ]] && [[ $1 == *.ini ]]; then + config_ini=$1 + shift + fi + PYTHONPATH=$LIBDIR $LIBDIR/tools/script/read_build_config.py $config_ini $@ +} + +function _read_manifest_vars() +{ + PRODUCT_RELEASE_BUILD_ID="${BUILD_NUMBER:?0}" + PRODUCT_RELEASE_LABEL="$(_read_build_config DEFAULT product_release_label)" +} + +function _initialize_work_dirs() +{ + rm -rf $WORKRESULTS + mkdir -p $WORKRESULTS $REPO_FILES $REPO_DIR $RPMLISTS $CHECKSUM_DIR + # dont clear tmp, can be used for caching + mkdir -p $WORKTMP + rm -rf $WORKLOGS + mkdir -p $WORKLOGS +} + +function _log() +{ + echo "$(date) $@" +} + +function _info() +{ + _log INFO: $@ +} + +function _header() +{ + _info "##################################################################" + _info "# $@" + _info "##################################################################" +} + + +function _divider() +{ + _info "------------------------------------------------------------------" +} + + +function _step() +{ + _header "STEP START: $@" +} + + +function _abort() +{ + _header "ERROR: $@" + exit 1 +} + + +function _success() +{ + _header "STEP OK: $@" +} + + +function _run_cmd() +{ + _info "[cmd-start]: $@" + stamp_start=$(date +%s) + time $@ 2>&1 || _abort "Command failed: $@" + stamp_end=$(date +%s) + echo "$((stamp_end - stamp_start)) $@" >> $DURATION_LOG.unsorted + sort -nr $DURATION_LOG.unsorted > $DURATION_LOG + _log "[cmd-end]: $@" +} + + +function _run_cmd_as_step() +{ + if [ $# -eq 1 -a -f $1 ]; then + step="$(basename $1)" + else + step="$@" + fi + _step $step + _run_cmd $@ + _success $step +} + + +function _add_rpms_to_repo() +{ + local repo_dir=$1 + local rpm_dir=$2 + mkdir -p $repo_dir + cp -f $(repomanage --keep=1 --new $rpm_dir) $repo_dir/ +} + +function _create_localrepo() +{ + pushd $REPO_DIR + _run_cmd createrepo --workers=8 --update . + popd + pushd $SRC_REPO_DIR + _run_cmd createrepo --workers=8 --update . + popd +} + +function _add_rpms_to_repos_from_workdir() +{ + _add_rpms_to_repo $REPO_DIR $1/buildrepository/mock/rpm + _add_rpms_to_repo $SRC_REPO_DIR $1/buildrepository/mock/srpm + #find $1/ -name '*.tar.gz' | xargs rm -f + true +} + +function _publish_results() +{ + local from=$1 + local to=$2 + mkdir -p $(dirname $to) + mv -f $from $to +} + +function _publish_image() +{ + _publish_results $1 $2 + _create_checksum $2 +} + +function _create_checksum() +{ + _create_md5_checksum $1 + _create_sha256_checksum $1 +} + +function _create_sha256_checksum() +{ + pushd $(dirname $1) + time sha256sum $(basename $1) > $(basename $1).sha256 + popd +} + +function _create_md5_checksum() +{ + pushd $(dirname $1) + time md5sum $(basename $1) > $(basename $1).md5 + popd +} + +function _is_true() +{ + # e.g. for Jenkins boolean parameters + [ "$1" == "true" ] +} + +function _join_array() +{ + local IFS="$1" + shift + echo "$*" +} + +function _get_package_list() +{ + PYTHONPATH=$LIBDIR $LIBDIR/tools/script/read_package_config.py $@ +} + +function _load_docker_image() +{ + local docker_image=$1 + local docker_image_url="$(_read_build_config DEFAULT docker_images)/${docker_image}.tar" + if docker inspect ${docker_image} &> /dev/null; then + echo "Using already built ${docker_image} image" + else + echo "Loading ${docker_image} image" + curl -L $docker_image_url | docker load + fi +} + diff --git a/mock/logging.ini b/mock/logging.ini new file mode 100644 index 0000000..8186ead --- /dev/null +++ b/mock/logging.ini @@ -0,0 +1,84 @@ +[formatters] +keys: detailed,simple,unadorned,state + +[handlers] +keys: simple_console,detailed_console,unadorned_console,simple_console_warnings_only + +[loggers] +keys: root,build,state,mockbuild + +[formatter_state] +format: %(asctime)s - %(message)s + +[formatter_unadorned] +format: %(message)s + +[formatter_simple] +format: %(levelname)s: %(message)s + +;useful for debugging: +[formatter_detailed] +format: %(levelname)s %(filename)s:%(lineno)d: %(message)s + +[handler_unadorned_console] +class: StreamHandler +args: [] +formatter: unadorned +level: INFO + +[handler_simple_console] +class: StreamHandler +args: [] +formatter: simple +level: INFO + +[handler_simple_console_warnings_only] +class: StreamHandler +args: [] +formatter: simple +level: WARNING + +[handler_detailed_console] +class: StreamHandler +args: [] +formatter: detailed +level: WARNING + +; usually dont want to set a level for loggers +; this way all handlers get all messages, and messages can be filtered +; at the handler level +; +; all these loggers default to a console output handler +; +[logger_root] +level: NOTSET +handlers: simple_console + +; mockbuild logger normally has no output +; catches stuff like mockbuild.trace_decorator and mockbuild.util +; dont normally want to propagate to root logger, either +[logger_mockbuild] +level: NOTSET +handlers: +qualname: mockbuild +propagate: 1 + +[logger_state] +level: NOTSET +; unadorned_console only outputs INFO or above +handlers: unadorned_console +qualname: mockbuild.Root.state +propagate: 0 + +[logger_build] +level: NOTSET +handlers: simple_console_warnings_only +qualname: mockbuild.Root.build +propagate: 0 + +; the following is a list mock logger qualnames used within the code: +; +; qualname: mockbuild.util +; qualname: mockbuild.uid +; qualname: mockbuild.trace_decorator + diff --git a/mock/mock.cfg.template b/mock/mock.cfg.template new file mode 100644 index 0000000..e2db78b --- /dev/null +++ b/mock/mock.cfg.template @@ -0,0 +1,89 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Root name to be used for chroot and caching, must differ between products +config_opts['root'] = 'akrainolite' + +config_opts['target_arch'] = 'x86_64' +config_opts['legal_host_arches'] = ('x86_64',) +config_opts['dist'] = 'el7' # only useful for --resultdir variable subst +config_opts['chroot_setup_cmd'] = 'install createrepo yum-utils bison byacc cscope ctags cvs diffstat doxygen flex gcc gcc-c++ gcc-gfortran gettext git indent intltool libtool patch patchutils rcs redhat-rpm-config rpm-build subversion swig systemtap sudo' +config_opts['plugin_conf']['yum_cache_enable'] = False +config_opts['plugin_conf']['ccache_enable'] = False +config_opts['plugin_conf']['ccache_opts']['max_cache_size'] = '1G' +config_opts['plugin_conf']['ccache_opts']['dir'] = "/dev/shm/ccache.lcc-epel7" +config_opts['rpmbuild_networking'] = True +config_opts['cleanup_on_success'] = True +config_opts['cleanup_on_failure'] = False +config_opts['exclude_from_homedir_cleanup'] = ('build/SOURCES', '.bash_history', '.bashrc', 'build/RPMS', ) +#ADDITIONAL_CONFIG_OPTS# + +# Common RPM directive values +config_opts['macros']['%packager'] = #RPM_PACKAGER# +config_opts['macros']['%dist'] = ".el7.centos%{_platform_product}" +config_opts['macros']['%_platform_product'] = #RPM_RELEASE_ID# +config_opts['macros']['%_platform_dist'] = ".el7.centos" +config_opts['macros']['%_platform_vendor'] = #RPM_VENDOR# +config_opts['macros']['%_platform_license'] = #RPM_LICENSE# +config_opts['macros']['%_platform_licence'] = "%{_platform_license}" + +# Product specific macros +config_opts['macros']['%_playbooks_path'] = "/opt/openstack-ansible/playbooks" +config_opts['macros']['%_inventory_path'] = "/opt/openstack-ansible/inventory" +config_opts['macros']['%_roles_path'] = "/etc/ansible/roles" +config_opts['macros']['%_installation_root_path'] = "/etc/lcm/playbooks/installation" +config_opts['macros']['%_bootstrapping_path'] = "%{_installation_root_path}/bootstrapping" +config_opts['macros']['%_provisioning_path'] = "%{_installation_root_path}/provisioning" +config_opts['macros']['%_postconfig_path'] = "%{_installation_root_path}/postconfig" +config_opts['macros']['%_finalize_path'] = "%{_installation_root_path}/finalize" +config_opts['macros']['%_ansible_filter_plugins_path'] = "%{_roles_path}/plugins/filter" +config_opts['macros']['%_caas_path'] = "/var/lib/caas" +config_opts['macros']['%_caas_container_tar_path'] = "%{_caas_path}/images" +config_opts['macros']['%_caas_manifest_path'] = "%{_caas_path}/manifests" +config_opts['macros']['%_caas_chart_path'] = "${_caas_path}/chart/" +config_opts['macros']['%_platform_bin_path'] = "/usr/local/bin" +config_opts['macros']['%_platform_lib_path'] = "/usr/local/lib" +config_opts['macros']['%_platform_etc_path'] = "/etc" +config_opts['macros']['%_platform_share_path'] = "/share" +config_opts['macros']['%_platform_man_path'] = "%{_platform_share_path}/man" +config_opts['macros']['%_platform_doc_path'] = "%{_platform_share_path}/doc" +config_opts['macros']['%_platform_var_path'] = "/var" +config_opts['macros']['%_platform_python'] = "/python2.7" +config_opts['macros']['%_platform_python_site_packages_path'] = "%{_platform_lib_path}%{_platform_python}/site-packages" +config_opts['macros']['%_platform_ocf_resource_path'] = "/usr/lib/ocf/resource.d" +config_opts['macros']['%_python_site_packages_path'] = "/usr/lib/python2.7/site-packages" +config_opts['macros']['%_secrets_path'] = "/etc/required-secrets" + +# Compilation +#config_opts['macros']['%_smp_mflags'] = "-j6" +#config_opts['macros']['%_smp_ncpus_max'] = 0 + +# Yum configuration +config_opts['yum.conf'] = """ +[main] +cachedir=/var/cache/yum +keepcache=1 +debuglevel=2 +reposdir=/dev/null +logfile=/var/log/yum.log +retries=20 +obsoletes=1 +gpgcheck=0 +assumeyes=1 +syslog_ident=mock +syslog_device= + +#REPOSITORIES# + +""" diff --git a/mock/site-defaults.cfg b/mock/site-defaults.cfg new file mode 100644 index 0000000..9fff7af --- /dev/null +++ b/mock/site-defaults.cfg @@ -0,0 +1,160 @@ +# mock defaults +# vim:tw=0:ts=4:sw=4:et: +# +# This config file is for site-specific default values that apply across all +# configurations. Options specified in this config file can be overridden in +# the individual mock config files. +# +# The site-defaults.cfg delivered by default has NO options set. Only set +# options here if you want to override the defaults. +# +# Entries in this file follow the same format as other mock config files. +# config_opts['foo'] = bar + +############################################################################# +# +# Things that we recommend you set in site-defaults.cfg: +# +# config_opts['basedir'] = '/var/lib/mock/' +# config_opts['cache_topdir'] = '/var/cache/mock' +# Note: the path pointed to by basedir and cache_topdir must be owned +# by group 'mock' and must have mode: g+rws +# config_opts['rpmbuild_timeout'] = 0 +# config_opts['use_host_resolv'] = True + +# You can configure log format to pull from logging.ini formats of these names: +# config_opts['build_log_fmt_name'] = "unadorned" +# config_opts['root_log_fmt_name'] = "detailed" +# config_opts['state_log_fmt_name'] = "state" +# +# mock will normally set up a minimal chroot /dev. +# If you want to use a pre-configured /dev, disable this and use the bind-mount +# plugin to mount your special /dev +# config_opts['internal_dev_setup'] = True +# +# internal_setarch defaults to 'True' if the python 'ctypes' package is +# available. It is in the python std lib on >= python 2.5. On older versions, +# it is available as an addon. On systems w/o ctypes, it will default to 'False' +# config_opts['internal_setarch'] = False +# +# the cleanup_on_* options allow you to automatically clean and remove the +# mock build directory, but only take effect if --resultdir is used. +# config_opts provides fine-grained control. cmdline only has big hammer +# +# config_opts['cleanup_on_success'] = 1 +# config_opts['cleanup_on_failure'] = 1 + +# if you want mock to automatically run createrepo on the rpms in your +# resultdir. +# config_opts['createrepo_on_rpms'] = False +# config_opts['createrepo_command'] = '/usr/bin/createrepo -d -q -x *.src.rpm' + +# if you want mock to backup the contents of a result dir before clean +# config_opts['backup_on_clean'] = False +# config_opts('backup_base_dir'] = config_opts['basedir'] + "backup" + + +############################################################################# +# +# plugin related. Below are the defaults. Change to suit your site +# policy. site-defaults.cfg is a good place to do this. +# +# NOTE: Some of the caching options can theoretically affect build +# reproducability. Change with care. +# +# config_opts['plugin_conf']['package_state_enable'] = True +# config_opts['plugin_conf']['ccache_enable'] = True +# config_opts['plugin_conf']['ccache_opts']['max_cache_size'] = '4G' +# config_opts['plugin_conf']['ccache_opts']['compress'] = None +# config_opts['plugin_conf']['ccache_opts']['dir'] = "%(cache_topdir)s/%(root)s/ccache/" +# config_opts['plugin_conf']['yum_cache_enable'] = True +# config_opts['plugin_conf']['yum_cache_opts']['max_age_days'] = 30 +# config_opts['plugin_conf']['yum_cache_opts']['dir'] = "%(cache_topdir)s/%(root)s/yum_cache/" +# config_opts['plugin_conf']['root_cache_enable'] = True +# config_opts['plugin_conf']['root_cache_opts']['max_age_days'] = 15 +# config_opts['plugin_conf']['root_cache_opts']['dir'] = "%(cache_topdir)s/%(root)s/root_cache/" +# config_opts['plugin_conf']['root_cache_opts']['compress_program'] = "pigz" +# config_opts['plugin_conf']['root_cache_opts']['extension'] = ".gz" +# config_opts['plugin_conf']['root_cache_opts']['exclude_dirs'] = ["./proc", "./sys", "./dev", +# "./tmp/ccache", "./var/cache/yum" ] +# +# bind mount plugin is enabled by default but has no configured directories to +# mount +# config_opts['plugin_conf']['bind_mount_enable'] = True +# config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/host/path', '/bind/mount/path/in/chroot/' )) +# +# config_opts['plugin_conf']['tmpfs_enable'] = False +# config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 1024 +# config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = '512m' +# config_opts['plugin_conf']['tmpfs_opts']['mode'] = '0755' +# config_opts['plugin_conf']['chroot_scan_enable'] = False +# config_opts['plugin_conf']['chroot_scan_opts'] = [ "core(\.\d+)?", "\.log$",] + +############################################################################# +# +# environment for chroot +# +# config_opts['environment']['TERM'] = 'vt100' +# config_opts['environment']['SHELL'] = '/bin/bash' +# config_opts['environment']['HOME'] = '/builddir' +# config_opts['environment']['HOSTNAME'] = 'mock' +# config_opts['environment']['PATH'] = '/usr/bin:/bin:/usr/sbin:/sbin' +# config_opts['environment']['PROMPT_COMMAND'] = 'echo -n ""' +# config_opts['environment']['LANG'] = os.environ.setdefault('LANG', 'en_US.UTF-8') +# config_opts['environment']['TZ'] = os.environ.setdefault('TZ', 'EST5EDT') + +############################################################################# +# +# Things that you can change, but we dont recommend it: +# config_opts['chroothome'] = '/builddir' +# config_opts['clean'] = True + +############################################################################# +# +# Things that must be adjusted if SCM integration is used: +# +# config_opts['scm'] = True +# config_opts['scm_opts']['method'] = 'git' +# config_opts['scm_opts']['cvs_get'] = 'cvs -d /srv/cvs co SCM_BRN SCM_PKG' +# config_opts['scm_opts']['git_get'] = 'git clone SCM_BRN git://localhost/SCM_PKG.git SCM_PKG' +# config_opts['scm_opts']['svn_get'] = 'svn co file:///srv/svn/SCM_PKG/SCM_BRN SCM_PKG' +# config_opts['scm_opts']['spec'] = 'SCM_PKG.spec' +# config_opts['scm_opts']['ext_src_dir'] = '/dev/null' +# config_opts['scm_opts']['write_tar'] = True +# config_opts['scm_opts']['git_timestamps'] = True + +# These options are also recognized but usually defined in cmd line +# with --scm-option package= --scm-option branch= +# config_opts['scm_opts']['package'] = 'mypkg' +# config_opts['scm_opts']['branch'] = 'master' + +############################################################################# +# +# Things that are best suited for individual chroot config files: +# +# MUST SET (in individual chroot cfg file): +# config_opts['root'] = 'name-of-yum-build-dir' +# config_opts['target_arch'] = 'i386' +# config_opts['yum.conf'] = '' +# config_opts['yum_common_opts'] = [] +# +# CAN SET, defaults usually work ok: +# config_opts['chroot_setup_cmd'] = 'install buildsys-build' +# config_opts['log_config_file'] = 'logging.ini' +# config_opts['more_buildreqs']['srpm_name-version-release'] = 'dependencies' +# config_opts['macros']['%Add_your_macro_name_here'] = "add macro value here" +# config_opts['files']['path/name/no/leading/slash'] = "put file contents here." +# config_opts['chrootuid'] = os.getuid() + +# If you change chrootgid, you must also change "mock" to the correct group +# name in this line of the mock PAM config: +# auth sufficient pam_succeed_if.so user ingroup mock use_uid quiet +# config_opts['chrootgid'] = grp.getgrnam("mock")[2] + +# config_opts['useradd'] = '/usr/sbin/useradd -m -u %(uid)s -g %(gid)s -d %(home)s -n %(user)s' # Fedora/RedHat +# +# Security related +# config_opts['no_root_shells'] = False +# +# Proxy settings (https_proxy, ftp_proxy, and no_proxy can also be set) +# config_opts['http_proxy'] = 'http://localhost:3128' diff --git a/nexus3_dl.sh b/nexus3_dl.sh new file mode 100755 index 0000000..03dd7ed --- /dev/null +++ b/nexus3_dl.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +NEXUS_URL=$1 +NEXUS_REPOSITORY=$2 +NEXUS_REPOSITORY_BASE_PATH=$3 +shift;shift;shift +NEXUS_REPOSITORY_SEARCH_PATTERNS=$@ + +_abort() { + echo "ERROR: $@" + exit 1 +} + +_search_group() { + local params="" + [ -n "$1" ] && params+="&group=$1" + [ -n "${2:-}" ] && params+="&name=$2" + curl "${NEXUS_URL}/service/rest/v1/search?repository=${NEXUS_REPOSITORY}${params}" +} + +for pat in $NEXUS_REPOSITORY_SEARCH_PATTERNS; do + search_group="/$NEXUS_REPOSITORY_BASE_PATH/$(echo $pat | cut -d':' -f1)" + search_name="" + if echo $pat | grep ':'; then + search_name="$(echo $pat | cut -d':' -f2)" + fi + resp=$(_search_group $search_group $search_name) + if [ "$(echo $resp | jq -r '.continuationToken')" != "null" ]; then + _abort "Pagination not implemented" + fi + for url in $(echo $resp | jq -r '.items[].assets[].downloadUrl'); do + to=${url#$NEXUS_URL/repository/$NEXUS_REPOSITORY/$NEXUS_REPOSITORY_BASE_PATH/} + mkdir -p $(dirname $to) + echo "Fetch $url" + curl $url > $to + done +done diff --git a/prepare_manifest.sh b/prepare_manifest.sh new file mode 100755 index 0000000..e34536d --- /dev/null +++ b/prepare_manifest.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh +_read_manifest_vars + +pushd $MANIFEST_PATH +sed -i -e "s/^Version: .*/Version: $PRODUCT_RELEASE_BUILD_ID/" rpmbuild.spec +cat > product-release <?$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module +max-module-lines=1000 + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of statements in function / method body +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[IMPORTS] + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/repo_summary.sh b/repo_summary.sh new file mode 100755 index 0000000..e533389 --- /dev/null +++ b/repo_summary.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +scriptdir="$(dirname $(readlink -f ${BASH_SOURCE[0]}))" +source $scriptdir/lib.sh + +_run_repo_cmd() +{ + _divider + _run_cmd $@ +} + +pushd $WORK/.repo/repo +_run_repo_cmd git rev-parse --short HEAD +popd +echo q | _run_repo_cmd repo info +_run_repo_cmd repo status -o +echo q | _run_repo_cmd repo forall -p -c git rev-parse HEAD + +# Store build manifest with project HEADS to a file +repo manifest -o $WORKRESULTS/manifest.xml +repo manifest -r -o $WORKRESULTS/manifest_revisions.xml diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..41944d1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +requests +pyyaml +pytest +pytest-cov +pytest-flakes +pytest-pep8 +mock diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/buildconfig.py b/tools/buildconfig.py new file mode 100755 index 0000000..a43fa92 --- /dev/null +++ b/tools/buildconfig.py @@ -0,0 +1,33 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ConfigParser + +from tools.statics import BUILD_CONFIG_PATH + + +class BuildConfigParser(ConfigParser.ConfigParser): + def __init__(self, ini_file=BUILD_CONFIG_PATH): + ConfigParser.ConfigParser.__init__(self) + self.ini_file = ini_file + self.optionxform = str + self.read(self.ini_file) + + def items(self, section): # pylint: disable=arguments-differ + defaults = self.defaults() + resultlist = [] + for item in ConfigParser.ConfigParser.items(self, section): + if item[0] not in defaults: + resultlist.append(item) + return resultlist diff --git a/tools/convert.py b/tools/convert.py new file mode 100644 index 0000000..31fa30d --- /dev/null +++ b/tools/convert.py @@ -0,0 +1,142 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import re + + +def to_json(data): + return json.dumps(data, sort_keys=True, indent=4) + + +class CsvConverter(object): + def __init__(self, data, preferred_field_order=None, escape_newlines=True): + self.data = data + self.preferred_field_order = preferred_field_order + self.escape_newlines = escape_newlines + self.csv_data = None + self._convert() + + def __str__(self): + return self.convert() + + def convert(self): + return self._render(CsvFormatter(self.csv_data)) + + def convert_to_ms_excel(self, text_fields=None): + """ + CSV that Microsoft Excel can read well. + + :param text_fields: list of columns to mark as text + NOTE: must not be used for fields that can contain comma(,) or + semicolon(;) as field will be split from these + :return: + """ + return self._render(CsvMSFormatter(self.csv_data, text_fields=text_fields)) + + def _convert(self): + if not isinstance(self.data, list): + raise Exception('Input data given is NOT a list') + if not self.data: + self.csv_data = [] + return + if not isinstance(self.data[0], dict): + raise Exception('First data element is NOT a dict') + headers = [] + possible_fields = list(set([key for i in self.data for key in i.keys()])) + if self.preferred_field_order is not None: + for preferred_field in self.preferred_field_order: + if preferred_field in possible_fields: + headers.append(preferred_field) + possible_fields.remove(preferred_field) + headers += sorted(possible_fields) + self.csv_data = [headers] + for obj in self.data: + row_data = [] + for header in headers: + field = obj.get(header) + if isinstance(field, (list, dict)): + x = json.dumps(field, sort_keys=True) + elif isinstance(field, unicode): + x = field.encode('utf-8') + else: + x = str(field) + row_data.append(x) + self.csv_data.append(row_data) + + def _render(self, formatter): + return formatter.format(self.escape_newlines) + + +class CsvFormatter(object): + def __init__(self, csv_data): + self.csv_data = csv_data + + def format(self, escape_newlines=True): + f_file = [] + for record in self.csv_data: + f_record = [] + for field in record: + f_field = self._field_formatter(field, escape_newlines) + f_record.append(f_field) + f_file.append(','.join(self._record_formatter(f_record))) + return '\r\n'.join(self._file_formatter(f_file)) + + @staticmethod + def _file_formatter(_file): + return _file + + @staticmethod + def _record_formatter(record): + return ['"{}"'.format(i) for i in record] + + @staticmethod + def _field_formatter(field, escape_newlines): + out = field.replace('"', '""') + if escape_newlines: + out = out.replace('\n', '\\n') + return out + + +class CsvMSFormatter(CsvFormatter): + max_cell_size = 32000 + + def __init__(self, csv_data, text_fields=None): + super(CsvMSFormatter, self).__init__(csv_data) + self.text_fields = text_fields + + def _file_formatter(self, _file): + return ['sep=,'] + super(CsvMSFormatter, self)._file_formatter(_file) + + def _record_formatter(self, record): + record = super(CsvMSFormatter, self)._record_formatter(record) + if self.text_fields: + formatted_record = [] + for index, field in enumerate(record): + heading = self.csv_data[0][index] + if heading in self.text_fields: + formatted_field = '=' + field + else: + formatted_field = field + formatted_record.append(formatted_field) + record = formatted_record + return record + + def _field_formatter(self, field, escape_newlines): + field = super(CsvMSFormatter, self)._field_formatter(field, escape_newlines) + if len(field) > self.max_cell_size: + field = field[:self.max_cell_size / 2] + "..." + field[-self.max_cell_size / 2:] + if not re.match(r'^-\d+$', field) and re.match(r'^-.*$', field): + return r'\{}'.format(field) + return field diff --git a/tools/convert_test.py b/tools/convert_test.py new file mode 100755 index 0000000..cdfe7dc --- /dev/null +++ b/tools/convert_test.py @@ -0,0 +1,165 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=invalid-name +from collections import OrderedDict + +import pytest + +from tools.convert import to_json, CsvConverter + + +class TestJson(object): + @staticmethod + def test_json(): + _input = dict(b=1, a=2) + expected = '\n'.join(['{', + ' "a": 2, ', + ' "b": 1', + '}']) + assert to_json(_input) == expected + + +class TestCsv(object): + @staticmethod + @pytest.mark.parametrize('_input, expected', [ + ([], ''), + ([{}], '\r\n'), + ([dict(a=1)], '\r\n'.join(['"a"', + '"1"'])), + ([dict(a=1), dict(a=2)], '\r\n'.join(['"a"', + '"1"', + '"2"'])), + ([dict(a=1, b=2), dict(a=3, b=4)], '\r\n'.join(['"a","b"', + '"1","2"', + '"3","4"'])), + ([dict(x=OrderedDict({'b': 1, 'a': 2})), + dict(x={'a': 2, 'b': 1})], '\r\n'.join(['"x"', + '"{""a"": 2, ""b"": 1}"', + '"{""a"": 2, ""b"": 1}"'])) + + ]) + def test_csv(_input, expected): + assert str(CsvConverter(_input)) == expected + + @staticmethod + def test_str(): + _input = [dict(a=1)] + assert str(CsvConverter(_input)) == CsvConverter(_input).convert() + + @staticmethod + @pytest.mark.parametrize('_input, exception_re', [ + ('boom', r'NOT a list'), + ([['boom']], r'NOT a dict') + ]) + def test_to_csv_fail(_input, exception_re): + with pytest.raises(Exception, match=exception_re): + str(CsvConverter(_input)) + + @staticmethod + def test_newlines_are_escaped(): + """ + ...in order to get properly formatted file when written + """ + _input = [dict(a='1"2')] + expected = '\r\n'.join(['"a"', + '"1""2"']) + assert CsvConverter(_input).convert() == expected + + @staticmethod + def test_missing_fields_are_filled_with_none(): + _input = [dict(a='1"2')] + expected = '\r\n'.join(['"a"', + '"1""2"']) + assert CsvConverter(_input).convert() == expected + + @staticmethod + def test_double_quote_is_escaped_with_double_quote(): + """ + RFC 4180 + """ + _input = [dict(a='1"2')] + expected = '\r\n'.join(['"a"', + '"1""2"']) + assert CsvConverter(_input).convert() == expected + + @staticmethod + @pytest.mark.parametrize('_input, expected', [ + ([dict(a=1, b=2)], '\r\n'.join(['"b","a"', '"2","1"'])), + ([dict(a=1, c=2)], '\r\n'.join(['"a","c"', '"1","2"'])) + ]) + def test_csv_preferred_order(_input, expected): + assert CsvConverter(_input, preferred_field_order=['b', 'a']).convert() == expected + + @staticmethod + @pytest.mark.parametrize('_input, expected', [ + ([dict(a='Copyright \xc2\xa9 2014')], '\r\n'.join(['"a"', + '"Copyright \xc2\xa9 2014"'])), + ([dict(a=u'Copyright \xa9 2014')], '\r\n'.join(['"a"', + '"Copyright \xc2\xa9 2014"'])), + ]) + def test_unicode_input(_input, expected): + assert str(CsvConverter(_input)) == expected + + +class TestCsvMSExcel(object): + @staticmethod + def test_ms_excel_format(): + """ + MS Excel treats CSV files with 'sep=,' as the first line to get automatically columnized + """ + _input = [dict(a=1, b=2)] + expected = '\r\n'.join(['sep=,', + '"a","b"', + '"1","2"']) + assert CsvConverter(_input).convert_to_ms_excel() == expected + + @staticmethod + def test_text_fields(): + """ + MS Excel CSV fields prefixed with '=' will be treated as equations to string. + This makes it possible to e.g. to have all RPM version strings treated equally instead + of making some of them treated as generic and some of them as integers. + """ + _input = [dict(a=1, b=2)] + expected = '\r\n'.join(['sep=,', + '"a",="b"', + '"1",="2"']) + assert CsvConverter(_input).convert_to_ms_excel(text_fields=['b']) == expected + + @staticmethod + def test_field_with_beginning_minus_is_prefixed(): + """ + MS Excel CSV fields beginning with '-' are treated as an equation even they would be + essentially just strings. Make sure to escape the beginning signs with something in order + not to get field braking equation. + """ + _input = [dict(a=-1), dict(a="-2a")] + expected = '\r\n'.join(['sep=,', + '"a"', + '"-1"', + r'"\-2a"']) + assert CsvConverter(_input).convert_to_ms_excel() == expected + + @staticmethod + def test_too_big_cell_is_truncated(): + """ + MS Excel has ~32k character limit per cell. BOM information can easily exceed this. + """ + _input = [dict(a='1' * 32000), dict(a='2' * 32001)] + expected = '\r\n'.join(['sep=,', + '"a"', + '"{}"'.format('1' * 32000), + '"{}...{}"'.format('2' * 16000, '2' * 16000)]) + assert CsvConverter(_input).convert_to_ms_excel() == expected diff --git a/tools/executor.py b/tools/executor.py new file mode 100755 index 0000000..f2a428c --- /dev/null +++ b/tools/executor.py @@ -0,0 +1,107 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import logging +import shlex +import subprocess + + +class ExecutionError(Exception): + def __init__(self, msg, result): + super(ExecutionError, self).__init__(msg) + self.result = result + + +class Result(object): + def __init__(self, status, stdout, stderr): + self.status = status + self.stdout = stdout + self.stderr = stderr + + @property + def str_status(self): + return 'Status:"{}"'.format(self.status) + + @property + def str_stdout(self): + return 'Stdout:"{}"'.format(self.stdout) + + @property + def str_stderr(self): + return 'Stderr:"{}"'.format(self.stderr) + + def __str__(self): + return '\n'.join([self.str_status, self.str_stdout, self.str_stderr]) + + +def run(*args, **kwargs): + return Executor().run(*args, **kwargs) + + +class Executor(object): + def __init__(self, shell=False, chdir=None): + self.shell = shell + self.chdir = chdir + + def run(self, cmd, raise_on_error=True, raise_on_stderr=True, retries=0): + result = self._run(cmd, retries) + if raise_on_error and result.status != 0: + raise ExecutionError('Command "{}" execution status NOT zero: {}'.format( + cmd, str(result)), result.status) + if raise_on_stderr and result.stderr: + raise ExecutionError('Command "{}" execution stderr not empty: {}'.format( + cmd, str(result)), result.status) + return result + + @staticmethod + def _log_result(result): + logging.debug('Result: %s', str(result)) + + def _run(self, cmd, retries): + logstr = 'Executing command: "{}"'.format(cmd) + cwd = os.getcwd() + if self.chdir is not None: + os.chdir(self.chdir) + logstr += ' in dir {}'.format(self.chdir) + logging.debug(logstr) + result = self._run_with_retries(cmd, retries) + if self.chdir is not None: + os.chdir(cwd) + return result + + def _run_with_retries(self, cmd, retries): + result = self._execute(cmd) + while result.status != 0 and retries > 0: + logging.debug('Retrying, retries left: %s', retries) + retries -= 1 + result = self._execute(cmd) + self._log_result(result) + if result.status == 0: + break + return result + + def _execute(self, cmd): + if not self.shell: + if isinstance(cmd, list): + cmd_args = cmd[:] + else: + cmd_args = shlex.split(cmd) + else: + cmd_args = cmd + process = subprocess.Popen(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=self.shell) + stdout, stderr = process.communicate() + result = Result(process.returncode, stdout, stderr) + return result diff --git a/tools/executor_test.py b/tools/executor_test.py new file mode 100755 index 0000000..5b389c7 --- /dev/null +++ b/tools/executor_test.py @@ -0,0 +1,81 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import mock + +from tools.executor import Executor, ExecutionError +from tools.executor import Result + + +@mock.patch.object(Executor, '_execute') +def test_success(mocky): + mocky.side_effect = [Result(0, 'fake-stdout', '')] + Executor().run('test-cmd') + mocky.assert_called_once_with('test-cmd') + + +@mock.patch.object(Executor, '_execute') +def test_fail_returncode(mocky): + mocky.side_effect = [Result(1, 'fake-stdout', '')] + with pytest.raises(ExecutionError, match=r'execution status NOT zero'): + Executor().run('test-cmd') + mocky.assert_called_once_with('test-cmd') + + +@mock.patch.object(Executor, '_execute') +def test_fail_stderr(mocky): + mocky.side_effect = [Result(0, 'fake-stdout', 'fake-stderr')] + with pytest.raises(ExecutionError, match=r'stderr not empty'): + Executor().run('test-cmd') + mocky.assert_called_once_with('test-cmd') + + +@mock.patch.object(Executor, '_execute') +def test_success_ignore_returncode(mocky): + mocky.side_effect = [Result(1, 'fake-stdout', '')] + Executor().run('test-cmd', raise_on_error=False) + mocky.assert_called_once_with('test-cmd') + + +@mock.patch.object(Executor, '_execute') +def test_success_ignore_stderr(mocky): + mocky.side_effect = [Result(0, 'fake-stdout', 'fake-stderr')] + Executor().run('test-cmd', raise_on_stderr=False) + mocky.assert_called_once_with('test-cmd') + + +@mock.patch.object(Executor, '_execute') +def test_no_retry_on_success(mocky): + mocky.side_effect = [Result(0, 'fake-stdout', '')] + Executor().run('test-cmd', retries=1) + mocky.assert_called_once_with('test-cmd') + + +@mock.patch.object(Executor, '_execute') +def test_retry_on_fail(mocky): + mocky.side_effect = [Result(1, 'fake-stdout', ''), + Result(0, 'fake-stdout', '')] + Executor().run('test-cmd', retries=1) + expected = [mock.call('test-cmd'), + mock.call('test-cmd')] + assert mocky.mock_calls == expected + + +@mock.patch.object(Executor, '_execute') +def test_error_on_retry_exceeded(mocky): + mocky.side_effect = [Result(1, 'fake-stdout', ''), + Result(1, 'fake-stdout', '')] + with pytest.raises(ExecutionError): + Executor().run('test-cmd', retries=1) diff --git a/tools/io.py b/tools/io.py new file mode 100755 index 0000000..c09f9ce --- /dev/null +++ b/tools/io.py @@ -0,0 +1,37 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging + +from tools.convert import to_json + + +def write_to(file_path, data): + with open(file_path, 'w') as f: + f.write(data) + logging.debug('Wrote: {}'.format(file_path)) + + +def read_from(file_path): + with open(file_path, 'r') as f: + return f.read() + + +def write_json(file_path, data): + write_to(file_path, to_json(data)) + + +def read_json(file_path): + return json.loads(read_from(file_path)) diff --git a/tools/log.py b/tools/log.py new file mode 100755 index 0000000..42ce383 --- /dev/null +++ b/tools/log.py @@ -0,0 +1,29 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys + + +def set_logging(debug=True, timestamps=False): + _format = '{}%(levelname)s:%(message)s' + if timestamps: + _format = _format.format('%(asctime)-15s:') + else: + _format = _format.format('') + if debug: + level = logging.DEBUG + else: + level = logging.INFO + logging.basicConfig(stream=sys.stdout, level=level, format=_format) diff --git a/tools/package.py b/tools/package.py new file mode 100755 index 0000000..1a42af9 --- /dev/null +++ b/tools/package.py @@ -0,0 +1,53 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml + +from tools.statics import PACKAGES_CONFIG_PATH + + +class PackageConfigReader(object): + def __init__(self, config_file=PACKAGES_CONFIG_PATH): + self.config_file = config_file + self.config = None + self._read_file() + + def _read_file(self): + with open(self.config_file, 'r') as fp: + self.config = yaml.load(fp) + + def get(self, package_operation_type): + return self._get_packages_by_type(package_operation_type) + + def _get_packages_by_type(self, package_operation_type): + if self.config is None: + return [] + if package_operation_type == 'install': + return self._get_installed_packages() + elif package_operation_type == 'uninstall': + return self._get_uninstalled_packages() + raise Exception('Never here') + + def _get_uninstalled_packages(self): + return sorted([p for p in self.config if not self._is_install_package(p)]) + + def _get_installed_packages(self): + return sorted([p for p in self.config if self._is_install_package(p)]) + + def _is_install_package(self, package): + if self.config[package] is None: + return True + if self.config[package].get('uninstall') is True: + return False + return True diff --git a/tools/package_test.py b/tools/package_test.py new file mode 100755 index 0000000..2225d53 --- /dev/null +++ b/tools/package_test.py @@ -0,0 +1,44 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +from tools.package import PackageConfigReader + + +def test_packages(): + with mock.patch('tools.package.open', mock.mock_open(read_data=FAKE_PACKAGES_YAML)) as _: + config = PackageConfigReader('test-config') + assert config.get('install') == ['install-pkg-1', + 'install-pkg-2'] + assert config.get('uninstall') == ['remove-pkg-1', + 'remove-pkg-2'] + + +def test_empty(): + with mock.patch('tools.package.open', mock.mock_open(read_data="")) as _: + config = PackageConfigReader('test-config') + assert config.get('install') == [] + + +FAKE_PACKAGES_YAML = """ +remove-pkg-1: + uninstall: True +remove-pkg-2: + uninstall: True + +install-pkg-1: + uninstall: False +install-pkg-2: +""" diff --git a/tools/releasereader.py b/tools/releasereader.py new file mode 100755 index 0000000..982504a --- /dev/null +++ b/tools/releasereader.py @@ -0,0 +1,31 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +from tools.executor import Executor + + +class ReleaseReader(object): + def __init__(self, manifest_dir): + self.manifest_dir = manifest_dir + + def read_current_release(self): + return Executor(chdir=self.manifest_dir).run( + ["git", "describe", "--tags", "--abbrev=0"]).stdout + + def get_next_release_label(self): + current = self.read_current_release() + match = re.search(r'^(.+[\.-])(\d+)$', current) + return '{}{}'.format(match.group(1), (int(match.group(2)) + 1)) diff --git a/tools/releasereader_test.py b/tools/releasereader_test.py new file mode 100755 index 0000000..273b680 --- /dev/null +++ b/tools/releasereader_test.py @@ -0,0 +1,34 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import mock + +from tools.releasereader import ReleaseReader +from tools.executor import Result + + +@pytest.mark.parametrize('curr_release, next_release', [ + ('ABC_D19-0', 'ABC_D19-1'), + ('ABC_D19-19', 'ABC_D19-20'), + ('ABC_D19A-1', 'ABC_D19A-2'), + ('ABC_D19A-1.0', 'ABC_D19A-1.1'), + ('ABC_D20-0', 'ABC_D20-1') +]) +@mock.patch('tools.releasereader.Executor') +def test_next_release(mock_exec, curr_release, next_release): + mock_exec.return_value.run.return_value = Result(0, curr_release + '\n', '') + assert ReleaseReader('test-path').get_next_release_label() == next_release + mock_exec.assert_called_once_with(chdir='test-path') + mock_exec.return_value.run.assert_called_once_with(['git', 'describe', '--tags', '--abbrev=0']) diff --git a/tools/repository.py b/tools/repository.py new file mode 100755 index 0000000..19000e9 --- /dev/null +++ b/tools/repository.py @@ -0,0 +1,53 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from tools.buildconfig import BuildConfigParser +from tools.statics import BUILD_CONFIG_PATH +from tools.utils import validate_environ + + +class RepositoryConfig(object): + def __init__(self, ini_file=BUILD_CONFIG_PATH): + self.config = BuildConfigParser(ini_file=ini_file) + + def read_section(self, section): + repositories = [] + for repo_name, repo_value in self.config.items(section): + parts = repo_value.split('#') + repodata = dict(name=repo_name, baseurl=parts[0]) + for p in parts[1:]: + key, value = p.split('=', 1) + repodata[key] = value + repositories.append(repodata) + return repositories + + def read_sections(self, sections): + repositories = [] + for s in sections: + repositories += self.read_section(s) + return repositories + + @classmethod + def get_localrepo(cls, remote=False): + dirname = 'repo' + if remote: + validate_environ(['BUILD_URL']) + baseurl = os.path.join(os.environ['BUILD_URL'], 'artifact/results', dirname) + else: + validate_environ(['WORKSPACE']) + baseurl = 'file://' + \ + os.path.join(os.environ['WORKSPACE'], 'results', dirname) + return dict(name='localrepo', baseurl=baseurl) diff --git a/tools/rpm.py b/tools/rpm.py new file mode 100755 index 0000000..efd7365 --- /dev/null +++ b/tools/rpm.py @@ -0,0 +1,99 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + + +class RpmData(dict): + + @property + def name(self): + return self['Name'] + + @property + def epoch(self): + return self.get('Epoch', '0') + + @property + def version(self): + return self['Version'] + + @property + def release(self): + return self['Release'] + + @property + def arch(self): + return self['Architecture'] + + @property + def vendor(self): + return self.get('Vendor', '') + + @property + def license(self): + return self.get('License', '') + + def __str__(self): + return '{}-{}-{}.{}'.format(self.name, + self.version, + self.release, + self.arch) + + def is_same_package_as(self, other): + for attr in ['name', 'epoch', 'version', 'release', 'arch']: + if getattr(self, attr) != getattr(other, attr): + return False + return True + + +class RpmInfoParser(object): + """ + Parse 'rpm -qi' output + """ + + def parse_file(self, rpm_info_installed_file_path): + with open(rpm_info_installed_file_path, 'r') as f: + return self.parse_multiple(f.read()) + + def parse_multiple(self, rpm_info_output_multiple): + packages = [] + package_index = -1 + for line in rpm_info_output_multiple.splitlines(): + if re.match(r'^Name\s+:.*', line): + packages.append(line) + package_index += 1 + else: + packages[package_index] += '\n' + line + return [self.parse_package(pkg) for pkg in packages] + + @staticmethod + def parse_package(rpm_info_output): + result = RpmData() + current_key = None + colon_location = rpm_info_output.splitlines()[0].find(':') + matcher = re.compile(r'^([A-Z][A-Za-z0-9 ]{{{}}}):( ?| .+)$'.format(colon_location - 1)) + for line in rpm_info_output.splitlines(): + match = matcher.match(line) + if match: + parsed_key = match.group(1).rstrip() + parsed_value = match.group(2).strip() + result[parsed_key] = parsed_value + current_key = parsed_key + else: + if not result[current_key]: + result[current_key] = line + else: + result[current_key] = result[current_key] + '\n' + line + return result diff --git a/tools/rpm_info_installed.sample b/tools/rpm_info_installed.sample new file mode 100644 index 0000000..a6e2c63 --- /dev/null +++ b/tools/rpm_info_installed.sample @@ -0,0 +1,144 @@ +Name : python-futures +Version : 3.0.3 +Release : 1.el7 +Architecture: noarch +Install Date: Wed Feb 7 13:49:03 2018 +Group : Development/Libraries +Size : 88366 +License : BSD +Signature : RSA/SHA1, Fri Sep 1 15:07:30 2017, Key ID f9b9fee7764429e6 +Source RPM : python-futures-3.0.3-1.el7.src.rpm +Build Date : Tue Jul 28 12:27:11 2015 +Build Host : c1bj.rdu2.centos.org +Relocations : (not relocatable) +Packager : CBS +Vendor : CentOS +URL : https://github.com/agronholm/pythonfutures +Summary : Backport of the concurrent.futures package from Python 3.2 +Description : +The concurrent.futures module provides a high-level interface for +asynchronously executing callables. +Name : libgnome-keyring +Version : 3.8.0 +Release : 3.el7 +Architecture: x86_64 +Install Date: Wed Feb 7 13:50:17 2018 +Group : System Environment/Libraries +Size : 303457 +License : GPLv2+ and LGPLv2+ +Signature : RSA/SHA256, Fri Jul 4 02:48:34 2014, Key ID 24c6a8a7f4a80eb5 +Source RPM : libgnome-keyring-3.8.0-3.el7.src.rpm +Build Date : Tue Jun 10 08:58:52 2014 +Build Host : worker1.bsys.centos.org +Relocations : (not relocatable) +Packager : CentOS BuildSystem +Vendor : CentOS +URL : http://live.gnome.org/GnomeKeyring +Summary : Framework for managing passwords and other secrets +Description : +gnome-keyring is a program that keep password and other secrets for +users. The library libgnome-keyring is used by applications to integrate +with the gnome-keyring system. +Name : python-ldap3 +Version : 0.9.8.6 +Release : 2.el7 +Architecture: noarch +Install Date: Wed Feb 7 13:52:25 2018 +Group : Unspecified +Size : 3120182 +License : LGPLv2+ +Signature : RSA/SHA1, Fri Sep 1 15:07:38 2017, Key ID f9b9fee7764429e6 +Source RPM : python-ldap3-0.9.8.6-2.el7.src.rpm +Build Date : Fri Nov 20 16:28:01 2015 +Build Host : c1bk.rdu2.centos.org +Relocations : (not relocatable) +Packager : CBS +Vendor : CentOS +URL : https://pypi.python.org/pypi/ldap3/0.9.8.6 +Summary : Strictly RFC 4511 conforming LDAP V3 pure Python client +Description : +python-ldap3 is a strictly RFC 4511 conforming LDAP V3 pure Python client. +The same codebase works with Python, Python 3, PyPy and PyPy3. +Name : python-dracclient +Version : 1.3.1 +Release : 0.20170926083913.8f368ed.el7.centos +Architecture: noarch +Install Date: Wed Feb 7 13:49:30 2018 +Group : Unspecified +Size : 1325637 +License : ASL 2.0 +Signature : (none) +Source RPM : python-dracclient-1.3.1-0.20170926083913.8f368ed.el7.centos.src.rpm +Build Date : Tue Sep 26 08:39:32 2017 +Build Host : trunk-primary.rdoproject.org.rdocloud +Relocations : (not relocatable) +URL : http://github.com/openstack/python-dracclient +Summary : Library for managing machines with Dell iDRAC cards. +Description : +Library for managing machines with Dell iDRAC cards. +Name : dialog +Version : 1.2 +Release : 4.20130523.el7 +Architecture: x86_64 +Install Date: Wed Feb 7 13:49:48 2018 +Group : Applications/System +Size : 517263 +License : LGPLv2 +Signature : RSA/SHA256, Fri Jul 4 01:08:16 2014, Key ID 24c6a8a7f4a80eb5 +Source RPM : dialog-1.2-4.20130523.el7.src.rpm +Build Date : Tue Jun 10 01:38:26 2014 +Build Host : worker1.bsys.centos.org +Relocations : (not relocatable) +Packager : CentOS BuildSystem +Vendor : CentOS +URL : http://invisible-island.net/dialog/dialog.html +Summary : A utility for creating TTY dialog boxes +Description : +Dialog is a utility that allows you to show dialog boxes (containing +questions or messages) in TTY (text mode) interfaces. Dialog is called +from within a shell script. The following dialog boxes are implemented: +yes/no, menu, input, message, text, info, checklist, radiolist, and +gauge. + +Install dialog if you would like to create TTY dialog boxes. +Name : filesystem +Version : 3.2 +Release : 21.el7 +Architecture: x86_64 +Install Date: Wed Feb 7 13:44:24 2018 +Group : System Environment/Base +Size : 0 +License : Public Domain +Signature : RSA/SHA256, Sun Nov 20 17:43:00 2016, Key ID 24c6a8a7f4a80eb5 +Source RPM : filesystem-3.2-21.el7.src.rpm +Build Date : Sat Nov 5 15:39:14 2016 +Build Host : worker1.bsys.centos.org +Relocations : (not relocatable) +Packager : CentOS BuildSystem +Vendor : CentOS +URL : https://fedorahosted.org/filesystem +Summary : The basic directory layout for a Linux system +Description : +The filesystem package is one of the basic packages that is installed +on a Linux system. Filesystem contains the basic directory layout +for a Linux operating system, including the correct permissions for +the directories. +Name : python2-tenacity +Version : 4.4.0 +Release : 1.el7 +Architecture: noarch +Install Date: Wed Feb 7 13:49:04 2018 +Group : Unspecified +Size : 159111 +License : ASL 2.0 +Signature : RSA/SHA1, Fri Sep 1 15:11:29 2017, Key ID f9b9fee7764429e6 +Source RPM : python-tenacity-4.4.0-1.el7.src.rpm +Build Date : Sat Aug 5 13:47:41 2017 +Build Host : c1bd.rdu2.centos.org +Relocations : (not relocatable) +Packager : CBS +Vendor : CentOS +URL : https://github.com/jd/tenacity +Summary : Tenacity is a general purpose retrying library +Description : + Tenacity is a general purpose retrying library \ No newline at end of file diff --git a/tools/rpm_test.py b/tools/rpm_test.py new file mode 100755 index 0000000..d875df3 --- /dev/null +++ b/tools/rpm_test.py @@ -0,0 +1,50 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +import pytest + +from tools.rpm import RpmInfoParser +from tools.rpm_test_data import \ + bash_expected, conntrack_tools_expected, cpp_expected, usbredir_expected, perl_compress_expected +from tools.test_data_rpm import \ + bash_rpm_info, cpp_rpm_info, conntrack_tools_rpm_info, usbredir_rpm_info, perl_compress_rpm_info + + +@pytest.mark.parametrize('rpm_info, expected_output', [ + (bash_rpm_info, bash_expected), + (conntrack_tools_rpm_info, conntrack_tools_expected), + (cpp_rpm_info, cpp_expected), + (usbredir_rpm_info, usbredir_expected), + (perl_compress_rpm_info, perl_compress_expected) +]) +def test_parse_package(rpm_info, expected_output): + parsed = RpmInfoParser().parse_package(rpm_info) + assert parsed == expected_output + + +def test_parse_multiple(): + parsed = RpmInfoParser().parse_multiple('\n'.join([bash_rpm_info, conntrack_tools_rpm_info])) + assert parsed == [bash_expected, conntrack_tools_expected] + + +def test_parse_file(): + test_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'rpm_info_installed.sample') + parsed = RpmInfoParser().parse_file(test_file) + with open(test_file, 'r') as f: + expected_rpms = re.findall(r'^Name\s+:.*$', f.read(), re.MULTILINE) + assert len(parsed) == len(expected_rpms) diff --git a/tools/rpm_test_data.py b/tools/rpm_test_data.py new file mode 100755 index 0000000..d89daca --- /dev/null +++ b/tools/rpm_test_data.py @@ -0,0 +1,179 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=invalid-name,line-too-long + + +bash_expected = { + 'Name': 'bash', + 'Version': '4.2.46', + 'Release': '21.el7_3', + 'Architecture': 'x86_64', + 'Install Date': 'Thu 11 Jan 2018 12:32:51 PM EET', + 'Group': 'System Environment/Shells', + 'Size': '3663714', + 'License': 'GPLv3+', + 'Signature': 'RSA/SHA256, Wed 07 Dec 2016 02:11:28 AM EET, Key ID 24c6a8a7f4a80eb5', + 'Source RPM': 'bash-4.2.46-21.el7_3.src.rpm', + 'Build Date': 'Wed 07 Dec 2016 01:21:54 AM EET', + 'Build Host': 'c1bm.rdu2.centos.org', + 'Relocations': '(not relocatable)', + 'Packager': 'CentOS BuildSystem ', + 'Vendor': 'CentOS', + 'URL': 'http://www.gnu.org/software/bash', + 'Summary': 'The GNU Bourne Again shell', + 'Description': '\n'.join( + ['The GNU Bourne Again shell (Bash) is a shell or command language', + 'interpreter that is compatible with the Bourne shell (sh). Bash', + 'incorporates useful features from the Korn shell (ksh) and the C shell', + '(csh). Most sh scripts can be run by bash without modification.']) +} + +conntrack_tools_expected = { + 'Name': 'conntrack-tools', + 'Version': '1.4.4', + 'Release': '3.el7_3', + 'Architecture': 'x86_64', + 'Install Date': 'Thu 11 Jan 2018 12:39:20 PM EET', + 'Group': 'System Environment/Base', + 'Size': '562826', + 'License': 'GPLv2', + 'Signature': 'RSA/SHA256, Thu 29 Jun 2017 03:36:05 PM EEST, Key ID 24c6a8a7f4a80eb5', + 'Source RPM': 'conntrack-tools-1.4.4-3.el7_3.src.rpm', + 'Build Date': 'Thu 29 Jun 2017 03:18:42 AM EEST', + 'Build Host': 'c1bm.rdu2.centos.org', + 'Relocations': '(not relocatable)', + 'Packager': 'CentOS BuildSystem ', + 'Vendor': 'CentOS', + 'URL': 'http://netfilter.org', + 'Summary': 'Manipulate netfilter connection tracking table and run High Availability', + 'Description': '\n'.join([ + 'With conntrack-tools you can setup a High Availability cluster and', + 'synchronize conntrack state between multiple firewalls.', + '', + 'The conntrack-tools package contains two programs:', + '- conntrack: the command line interface to interact with the connection', + ' tracking system.', + '- conntrackd: the connection tracking userspace daemon that can be used to', + ' deploy highly available GNU/Linux firewalls and collect', + ' statistics of the firewall use.', + '', + 'conntrack is used to search, list, inspect and maintain the netfilter', + 'connection tracking subsystem of the Linux kernel.', + 'Using conntrack, you can dump a list of all (or a filtered selection of)', + 'currently tracked connections, delete connections from the state table,', + 'and even add new ones.', + 'In addition, you can also monitor connection tracking events, e.g.', + 'show an event message (one line) per newly established connection.' + ]) +} + +cpp_expected = { + 'Name': 'cpp', + 'Version': '4.8.5', + 'Release': '11.el7', + 'Architecture': 'x86_64', + 'Install Date': 'Thu 11 Jan 2018 12:37:55 PM EET', + 'Group': 'Development/Languages', + 'Size': '15632501', + 'License': 'GPLv3+ and GPLv3+ with exceptions and GPLv2+ with exceptions and LGPLv2+ and BSD', + 'Signature': 'RSA/SHA256, Sun 20 Nov 2016 07:27:00 PM EET, Key ID 24c6a8a7f4a80eb5', + 'Source RPM': 'gcc-4.8.5-11.el7.src.rpm', + 'Build Date': 'Fri 04 Nov 2016 06:01:22 PM EET', + 'Build Host': 'worker1.bsys.centos.org', + 'Relocations': '(not relocatable)', + 'Packager': 'CentOS BuildSystem ', + 'Vendor': 'CentOS', + 'URL': 'http://gcc.gnu.org', + 'Summary': 'The C Preprocessor', + 'Description': + '\n'.join(['Cpp is the GNU C-Compatible Compiler Preprocessor.', + 'Cpp is a macro processor which is used automatically', + 'by the C compiler to transform your program before actual', + 'compilation. It is called a macro processor because it allows', + 'you to define macros, abbreviations for longer', + 'constructs.', + '', + 'The C preprocessor provides four separate functionalities: the', + 'inclusion of header files (files of declarations that can be', + 'substituted into your program); macro expansion (you can define macros,', + 'and the C preprocessor will replace the macros with their definitions', + 'throughout the program); conditional compilation (using special', + 'preprocessing directives, you can include or exclude parts of the', + 'program according to various conditions); and line control (if you use', + 'a program to combine or rearrange source files into an intermediate', + 'file which is then compiled, you can use line control to inform the', + 'compiler about where each source line originated).', + '', + 'You should install this package if you are a C programmer and you use', + 'macros.']), +} + +usbredir_expected = { + 'Name': 'usbredir', + 'Version': '0.7.1', + 'Release': '1.el7', + 'Architecture': 'x86_64', + 'Install Date': 'Wed Feb 7 13:49:24 2018', + 'Group': 'System Environment/Libraries', + 'Size': '108319', + 'License': 'LGPLv2+', + 'Signature': 'RSA/SHA256, Sun Nov 20 20:56:49 2016, Key ID 24c6a8a7f4a80eb5', + 'Source RPM': 'usbredir-0.7.1-1.el7.src.rpm', + 'Build Date': 'Sat Nov 5 18:33:15 2016', + 'Build Host': 'worker1.bsys.centos.org', + 'Relocations': '(not relocatable)', + 'Packager': 'CentOS BuildSystem ', + 'Vendor': 'CentOS', + 'URL': 'http://spice-space.org/page/UsbRedir', + 'Summary': 'USB network redirection protocol libraries', + 'Description': '\n'.join([ + 'The usbredir libraries allow USB devices to be used on remote and/or virtual', + 'hosts over TCP. The following libraries are provided:', + '', + 'usbredirparser:', + 'A library containing the parser for the usbredir protocol', + '', + 'usbredirhost:', + 'A library implementing the USB host side of a usbredir connection.', + 'All that an application wishing to implement a USB host needs to do is:', + '* Provide a libusb device handle for the device', + '* Provide write and read callbacks for the actual transport of usbredir data', + '* Monitor for usbredir and libusb read/write events and call their handlers']) +} + +perl_compress_expected = { + 'Name': 'perl-Compress-Raw-Zlib', + 'Epoch': '1', + 'Version': '2.061', + 'Release': '4.el7', + 'Architecture': 'x86_64', + 'Install Date': 'Sat Jan 26 20:05:50 2019', + 'Group': 'Development/Libraries', + 'Size': '139803', + 'License': 'GPL+ or Artistic', + 'Signature': 'RSA/SHA256, Fri Jul 4 04:15:33 2014, Key ID 24c6a8a7f4a80eb5', + 'Source RPM': 'perl-Compress-Raw-Zlib-2.061-4.el7.src.rpm', + 'Build Date': 'Tue Jun 10 01:12:08 2014', + 'Build Host': 'worker1.bsys.centos.org', + 'Relocations': '(not relocatable)', + 'Packager': 'CentOS BuildSystem ', + 'Vendor': 'CentOS', + 'URL': 'http://search.cpan.org/dist/Compress-Raw-Zlib/', + 'Summary': 'Low-level interface to the zlib compression library', + 'Description': '\n'.join([ + 'The Compress::Raw::Zlib module provides a Perl interface to the zlib', + 'compression library, which is used by IO::Compress::Zlib.']), + 'Obsoletes': '' +} diff --git a/tools/script/__init__.py b/tools/script/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/script/ci_build_diff.py b/tools/script/ci_build_diff.py new file mode 100755 index 0000000..c9dc567 --- /dev/null +++ b/tools/script/ci_build_diff.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys +import argparse +from operator import itemgetter +from pprint import pformat + +from tools.convert import CsvConverter +from tools.io import read_json, write_to, write_json +from tools.log import set_logging + + +class SwComponent(dict): + + @property + def name(self): + return self['Name'] + + @property + def version(self): + return self['Version'] + + @property + def foss_id(self): + return self.name, self.version, self.get('Release') + + def __str__(self): + return '{}:{}({})'.format(self.name, + self.version, + self['Source RPM']) + + +class BuildDiffReader(object): + + def __init__(self): + self.changes = {} + self.summary = {} + + @staticmethod + def get_component_names(data): + return [i['Name'] for i in data] + + @staticmethod + def get_components(name, components): + out = [] + for component in components: + if component['Name'] == name: + out.append(SwComponent(component)) + return sorted(out, key=itemgetter('Name', 'Version', 'Source RPM')) + + def read(self, json_old, json_new): + old_build = read_json(json_old) + new_build = read_json(json_new) + self.summary['input'] = {} + self.summary['input']['from'] = len(old_build) + self.summary['input']['to'] = len(new_build) + self.changes = self.read_build_diff(old_build, new_build) + self.summary['output'] = self._generate_summary(self.changes) + + @staticmethod + def _generate_summary(changes): + summary = {} + summary['counts'] = changes['counts'] + summary['added'] = {name: [str(c) for c in compos] for name, compos in + changes['added'].items()} + summary['removed'] = {name: [str(c) for c in compos] for name, compos in + changes['removed'].items()} + summary['changed'] = {name: {'old': [str(c) for c in change['old']], + 'new': [str(c) for c in change['new']]} for name, change in + changes['changed'].items()} + return summary + + def read_build_diff(self, old_build, new_build): + old_names = self.get_component_names(old_build) + logging.debug('Old names: {}'.format(old_names)) + new_names = self.get_component_names(new_build) + logging.debug('New names: {}'.format(new_names)) + added = {n: self.get_components(n, new_build) for n in set(new_names) - set(old_names)} + self._mark('[MARK] added', [j for i in added.values() for j in i]) + removed = {n: self.get_components(n, old_build) for n in set(old_names) - set(new_names)} + self._mark('[MARK] removed', [j for i in removed.values() for j in i]) + changed = {} + for n in set(old_names) & set(new_names): + old_components = self.get_components(n, old_build) + new_components = self.get_components(n, new_build) + if sorted([i.foss_id for i in old_components]) != \ + sorted([i.foss_id for i in new_components]): + changed[n] = {'old': old_components, 'new': new_components} + self._mark('[MARK] changed old', changed[n]['old']) + self._mark('[MARK] changed new', changed[n]['new']) + return dict(counts=dict(added=len(added), + changed=len(changed), + removed=len(removed)), + added=added, + removed=removed, + changed=changed) + + @staticmethod + def _mark(title, components): + logging.debug( + '[MARK] {}: {}'.format(title, pformat([i.foss_id for i in components]))) + + @staticmethod + def _get_csv_cells(name, old_components, new_components): + cells = dict(name=name) + if old_components: + cells.update(dict(old_components='\n'.join([str(i) for i in old_components]), + old_srpms='\n'.join([i['Source RPM'] for i in old_components]), + old_licenses='\n'.join( + [i.get('License', 'Unknown') for i in old_components]))) + if new_components: + cells.update(dict(new_components='\n'.join([str(i) for i in new_components]), + new_srpms='\n'.join([i['Source RPM'] for i in new_components]), + new_licenses='\n'.join( + [i.get('License', 'Unknown') for i in new_components]))) + return cells + + def write_csv(self, path): + data = [] + for name, components in self.changes['added'].items(): + data += [self._get_csv_cells(name, [], components)] + + for name, components in self.changes['removed'].items(): + data += [self._get_csv_cells(name, components, [])] + + for name, components in self.changes['changed'].items(): + data += [self._get_csv_cells(name, components['old'], components['new'])] + + csv = CsvConverter(sorted(data, key=itemgetter('name')), + preferred_field_order=['name', + 'old_components', 'old_srpms', 'old_licenses', + 'new_components', 'new_srpms', 'new_licenses'], + escape_newlines=False) + write_to(path, csv.convert_to_ms_excel()) + + +def parse(args): + parser = argparse.ArgumentParser(description='Outputs RPM changes between two CI builds') + parser.add_argument('--verbose', '-v', action='store_true', + help='More verbose logging') + parser.add_argument('components_json_1', + help='Components json file path (CI build artifact)') + parser.add_argument('components_json_2', + help='Components json file path (CI build artifact)') + parser.add_argument('--output-json', + help='output to json file') + parser.add_argument('--output-csv', + help='output to $MS csv file') + return parser.parse_args(args) + + +def main(input_args): + args = parse(input_args) + set_logging(debug=args.verbose) + x = BuildDiffReader() + x.read(args.components_json_1, args.components_json_2) + logging.info('----- SUMMARY ------\n{}'.format(pformat(x.summary))) + if args.output_json: + write_json(args.output_json, x.changes) + if args.output_csv: + x.write_csv(args.output_csv) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/tools/script/ci_build_diff_test.py b/tools/script/ci_build_diff_test.py new file mode 100755 index 0000000..81a1e76 --- /dev/null +++ b/tools/script/ci_build_diff_test.py @@ -0,0 +1,249 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long,invalid-name +import json + +from tools.script.ci_build_diff import main +from tools.script.ci_build_diff_test_data import component_added, component_changed_old, \ + component_changed_new, component_removed, caas_grafana1, caas_grafana2, caas_grafana2_sub, \ + caas_grafana1_sub, caas_grafana1_v2, caas_grafana1_v2sub, caas_grafana1_r2, \ + caas_grafana1_r2sub, grafana_v1, grafana_v2, caas_grafana3, caas_grafana3_sub, caas_abc1_sub, \ + caas_abc1, caas_abc1_r2, caas_abc1_sub_r2, abc1_v2, abc1, abc2, abc3, \ + caas_grafana1_sub_new_field + + +def test_no_output_args(tmpdir): + input_old, input_new = _gen_input_json(tmpdir) + main([str(input_old), str(input_new)]) + + +def test_no_changes(tmpdir): + input_old, _ = _gen_input_json(tmpdir) + output_csv = tmpdir.join('diff.csv') + output_json = tmpdir.join('diff.json') + main([str(input_old), str(input_old), + '--output-csv', str(output_csv), + '--output-json', str(output_json)]) + assert output_csv.read() == 'sep=,' + assert json.loads(output_json.read()) == {"added": {}, + "changed": {}, + "removed": {}, + "counts": { + "added": 0, + "changed": 0, + "removed": 0 + }} + + +def test_no_changes_if_json_field_is_added(tmpdir): + """ + Make sure diff is ok e.g. when new build has new field in json + """ + _assert_json_out(tmpdir, + [caas_grafana1_sub], + [caas_grafana1_sub_new_field], + {"added": {}, + "changed": {}, + "removed": {}, + "counts": { + "added": 0, + "changed": 0, + "removed": 0 + }}) + + +def test_multiple_rpms(tmpdir): + _assert_json_out(tmpdir, + [component_changed_old, component_removed], + [component_changed_new, component_added], + {"added": {component_added['Name']: [component_added]}, + "changed": {component_changed_old['Name']: {'old': [component_changed_old], + 'new': [component_changed_new]}}, + "removed": {component_removed['Name']: [component_removed]}, + "counts": { + "added": 1, + "changed": 1, + "removed": 1 + }}) + + +def test_component_name_change(tmpdir): + """ + Sub-component does not change although RPM containing it changes + """ + _assert_json_out(tmpdir, + [caas_grafana1, caas_grafana1_sub], + [caas_grafana2, caas_grafana2_sub], + {"added": {caas_grafana2['Name']: [caas_grafana2]}, + "changed": {}, + "removed": {caas_grafana1['Name']: [caas_grafana1]}, + "counts": { + "added": 1, + "changed": 0, + "removed": 1 + }}) + + +def test_component_version_change(tmpdir): + _assert_json_out(tmpdir, + [caas_grafana1, caas_grafana1_sub], + [caas_grafana1_v2, caas_grafana1_v2sub], + {"added": {}, + "changed": {caas_grafana1['Name']: {'old': [caas_grafana1], + 'new': [caas_grafana1_v2]}, + caas_grafana1_sub['Name']: {'old': [caas_grafana1_sub], + 'new': [caas_grafana1_v2sub]}}, + "removed": {}, + "counts": { + "added": 0, + "changed": 2, + "removed": 0 + }}) + + +def test_component_release_change(tmpdir): + _assert_json_out(tmpdir, + [caas_grafana1, caas_grafana1_sub], + [caas_grafana1_r2, caas_grafana1_r2sub], + {"added": {}, + "changed": {caas_grafana1['Name']: {'old': [caas_grafana1], + 'new': [caas_grafana1_r2]}}, + "removed": {}, + "counts": { + "added": 0, + "changed": 1, + "removed": 0 + }}) + + +def test_same_name_component_added(tmpdir): + _assert_json_out(tmpdir, + [caas_grafana1, caas_grafana1_sub], + [caas_grafana1, caas_grafana1_sub, grafana_v1], + {"added": {}, + "changed": {grafana_v1['Name']: {'old': [caas_grafana1_sub], + 'new': [caas_grafana1_sub, grafana_v1]}}, + "removed": {}, + "counts": { + "added": 0, + "changed": 1, + "removed": 0 + }}) + + +def test_same_name_component_removed(tmpdir): + _assert_json_out(tmpdir, + [caas_grafana1, caas_grafana1_sub, grafana_v1], + [caas_grafana1, caas_grafana1_sub], + {"added": {}, + "changed": {grafana_v1['Name']: {'old': [caas_grafana1_sub, grafana_v1], + 'new': [caas_grafana1_sub]}}, + "removed": {}, + "counts": { + "added": 0, + "changed": 1, + "removed": 0 + }}) + + +def test_same_name_component_changed(tmpdir): + _assert_json_out(tmpdir, + [caas_grafana1, caas_grafana1_sub, grafana_v1], + [caas_grafana1, caas_grafana1_sub, grafana_v2], + {"added": {}, + "changed": {grafana_v1['Name']: {'old': [caas_grafana1_sub, grafana_v1], + 'new': [caas_grafana1_sub, grafana_v2]}}, + "removed": {}, + "counts": { + "added": 0, + "changed": 1, + "removed": 0 + }}) + + +def test_epic(tmpdir): + _assert_json_out(tmpdir, + [caas_grafana1, caas_grafana1_sub, + caas_grafana2, caas_grafana2_sub, + grafana_v1], + [caas_grafana1_r2, caas_grafana1_r2sub, + grafana_v2, + caas_grafana3, caas_grafana3_sub], + {"added": {caas_grafana3['Name']: [caas_grafana3]}, + "changed": {caas_grafana1['Name']: {'old': [caas_grafana1], + 'new': [caas_grafana1_r2]}, + grafana_v1['Name']: { + 'old': [caas_grafana1_sub, caas_grafana2_sub, grafana_v1], + 'new': [caas_grafana1_r2sub, caas_grafana3_sub, grafana_v2]}}, + "removed": {caas_grafana2['Name']: [caas_grafana2]}, + "counts": { + "added": 1, + "changed": 2, + "removed": 1 + }}) + + +def _assert_json_out(tmpdir, from_build, to_build, expected_output): + input_old, input_new = _gen_input_json(tmpdir, from_build, to_build) + output_json = tmpdir.join('diff.json') + main([str(input_old), str(input_new), '--output-json', str(output_json)]) + assert json.loads(output_json.read()) == expected_output + + +def test_csv(tmpdir): + input_old, input_new = _gen_input_json(tmpdir, + [caas_abc1, caas_abc1_sub, abc1, abc2], + [caas_abc1_r2, caas_abc1_sub_r2, abc1_v2, abc3]) + output_csv = tmpdir.join('diff.csv') + main([str(input_old), str(input_new), '--output-csv', str(output_csv)]) + rows = [['name', + 'old_components', 'old_srpms', 'old_licenses', + 'new_components', 'new_srpms', 'new_licenses'], + ['abc', + 'abc:v1(abc-v1-r1.src.rpm)\nabc:v1(caas-abc-v1-r1.src.rpm)', + 'abc-v1-r1.src.rpm\ncaas-abc-v1-r1.src.rpm', + 'GPL\nUnknown', + 'abc:v1(caas-abc-v1-r2.src.rpm)\nabc:v2(abc-v2-r1.src.rpm)', + 'caas-abc-v1-r2.src.rpm\nabc-v2-r1.src.rpm', + 'Unknown\nGPL'], + ['abc2', + 'abc2:v1(abc2-v1-r1.src.rpm)', 'abc2-v1-r1.src.rpm', 'GPL', + 'None', 'None', 'None'], + ['abc3', + 'None', 'None', 'None', + 'abc3:v1(abc3-v1-r1.src.rpm)', 'abc3-v1-r1.src.rpm', 'GPL'], + ['caas-abc', + 'caas-abc:v1(caas-abc-v1-r1.src.rpm)', 'caas-abc-v1-r1.src.rpm', 'Commercial', + 'caas-abc:v1(caas-abc-v1-r2.src.rpm)', 'caas-abc-v1-r2.src.rpm', 'Commercial']] + expected_csv = '\r\n'.join(['sep=,'] + [_get_csv_row(row) for row in rows]) + assert output_csv.read() == expected_csv + + +def _get_csv_row(items): + return ','.join(['"{}"'.format(i) for i in items]) + + +def _gen_input_json(tmpdir, _from=None, _to=None): + if _from is None: + _from = [component_changed_old, component_removed] + if _to is None: + _to = [component_changed_new, component_added] + input_old = tmpdir.join('input_old.json') + input_new = tmpdir.join('input_new.json') + json_old = json.dumps(_from) + json_new = json.dumps(_to) + input_old.write(json_old) + input_new.write(json_new) + return input_old, input_new diff --git a/tools/script/ci_build_diff_test_data.py b/tools/script/ci_build_diff_test_data.py new file mode 100755 index 0000000..e3df7a3 --- /dev/null +++ b/tools/script/ci_build_diff_test_data.py @@ -0,0 +1,281 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long,invalid-name +import re + +from copy import deepcopy + + +def _copy(d, pattern, repl): + x = deepcopy(d) + for k, v in x.items(): + if isinstance(v, str): + x[k] = re.sub(pattern, repl, v) + return x + + +caas_grafana1 = { + "Architecture": "noarch", + "Build Date": "Wed May 30 11:39:05 2018", + "Build Host": "crf-dev", + "Crypto capable": False, + "Description": "This is grafana, this is SPARTAAAAA!", + "FOSS": "No", + "From repo": "caas-artifactory", + "Group": "Unspecified", + "Install Date": "Sun Nov 18 22:15:41 2018", + "Is sane": True, + "License": "Commercial", + "Name": "caas.grafana", + "Obsoletes": "", + "Release": "1.el7.centos", + "Relocations": "(not relocatable)", + "Repo": "installed", + "Repo data": { + "baseurl": "http://files/20181023", + "name": "caas-artifactory" + }, + "Signature": "(none)", + "Size": "109683450", + "Source RPM": "caas.grafana-4.4.1.9-1.el7.centos.src.rpm", + "Source repo data": { + "baseurl": "http://files/20181023", + "name": "caas-artifactory" + }, + "Source to be delivered": "No", + "Summary": "caasgrafana", + "Vendor": "Something", + "Version": "4.4.1.9" +} + +caas_grafana1_sub = { + "Name": "grafana", + "Version": "4.4.1.9", + "Source RPM": "caas.grafana-4.4.1.9-1.el7.centos.src.rpm", + "Source URL": "https://some/grafana/url", + "FOSS": "yes" +} + +caas_grafana1_sub_new_field = deepcopy(caas_grafana1_sub) +caas_grafana1_sub_new_field['ABC'] = True + +caas_grafana2 = _copy(caas_grafana1, 'caas.grafana', 'caas.grafana2') +caas_grafana2_sub = _copy(caas_grafana1_sub, 'caas.grafana', 'caas.grafana2') + +caas_grafana3 = _copy(caas_grafana1, 'caas.grafana', 'caas.grafana3') +caas_grafana3_sub = _copy(caas_grafana1_sub, 'caas.grafana', 'caas.grafana3') + +caas_grafana1_v2 = _copy(caas_grafana1, '4.4.1.9', '4.4.1.10') +caas_grafana1_v2sub = _copy(caas_grafana1_sub, '4.4.1.9', '4.4.1.10') + +caas_grafana1_r2 = _copy(caas_grafana1, '1.el7.centos', '2.el7.centos') +caas_grafana1_r2sub = _copy(caas_grafana1_sub, '1.el7.centos', '2.el7.centos') + +caas_abc1 = { + "License": "Commercial", + "Name": "caas-abc", + "Version": "v1", + "Release": "r1", + "Source RPM": "caas-abc-v1-r1.src.rpm", +} + +caas_abc1_sub = { + "Name": "abc", + "Version": "v1", + "Source RPM": "caas-abc-v1-r1.src.rpm", +} +caas_abc1_r2 = _copy(caas_abc1, 'r1', 'r2') +caas_abc1_sub_r2 = _copy(caas_abc1_sub, 'r1', 'r2') + +abc1 = { + "License": "GPL", + "Name": "abc", + "Version": "v1", + "Release": "r1", + "Source RPM": "abc-v1-r1.src.rpm", +} +abc1_v2 = _copy(abc1, 'v1', 'v2') +abc2 = _copy(abc1, 'abc', 'abc2') +abc3 = _copy(abc1, 'abc', 'abc3') + +component_added = { + 'Architecture': 'noarch', + 'Build Date': 'Sun Nov 11 12:54:39 2018', + 'Build Host': 'build-7.novalocal', + 'Description': 'This RPM contains configuration management openstack configuration override ' + 'plugin', + 'FOSS': 'No', + 'From repo': 'localrepo', + 'Group': 'Unspecified', + 'Install Date': 'Tue Nov 13 19:14:29 2018', + 'Is sane': True, + 'License': 'Commercial', + 'Name': 'openstack-config-overrides-validator', + 'Obsoletes': '', + 'Packager': 'Something', + 'Release': '1.el7.centos', + 'Relocations': '(not relocatable)', + 'Repo': 'installed', + 'Repo data': { + 'baseurl': 'https://jenkins/ci-build/2490/artifact/results/repo', + 'name': 'localrepo'}, + 'Signature': '(none)', + 'Size': '3097', + 'Source RPM': 'openstack-config-overrides-validator-c2.gd1b7aec-1.el7.centos.src.rpm', + 'Source repo data': { + 'baseurl': 'https://jenkins/ci-build/2490/artifact/results/src_repo', + 'name': 'localrepo'}, + 'Source to be delivered': 'No', + 'Summary': 'Openstack configuration override CM validator plugin.', + 'Vendor': 'Something', + 'Version': 'c2.gd1b7aec' +} + +component_removed = { + 'Architecture': 'x86_64', + 'Build Date': 'Thu Aug 16 14:46:11 2018', + 'Build Host': 'x86-01.bsys.centos.org', + 'Description': 'The fence-agents-ibmblade package contains a fence agent for IBM BladeCenter ' + 'devices that are accessed via the SNMP protocol.', + 'FOSS': 'Undefined', + 'From repo': 'purkki-centos-updates', + 'Group': 'System Environment/Base', + 'Install Date': 'Wed Nov 7 21:20:01 2018', + 'Is sane': False, + 'License': 'GPLv2+ and LGPLv2+', + 'Name': 'fence-agents-ibmblade', + 'Obsoletes': 'fence-agents,', + 'Packager': 'CentOS BuildSystem ', + 'Release': '86.el7_5.3', + 'Relocations': '(not relocatable)', + 'Repo': 'installed', + 'Repo data': { + 'baseurl': 'http://purkki/mirror/centos/snapshot/20181024/7/updates/x86_64/', + 'exclude': 'libgudev1 httpd httpd-devel systemd-libs.i686 resource-agents ' + 'dhcp-libs dhclient dhcp-common php-fpm php-common php-cli php', + 'name': 'purkki-centos-updates'}, + 'Signature': 'RSA/SHA256, Mon Aug 20 14:15:17 2018, Key ID 24c6a8a7f4a80eb5', + 'Size': '3898', + 'Source RPM': 'fence-agents-4.0.11-86.el7_5.3.src.rpm', + 'Source repo data': { + 'baseurl': 'http://purkki/mirror/centos/snapshot/20181024/7/updates/Source/', + 'name': 'purkki-centos-updates'}, + 'Source to be delivered': 'Undefined', + 'Summary': 'Fence agent for IBM BladeCenter', + 'URL': 'https://github.com/ClusterLabs/fence-agents', + 'Vendor': 'CentOS', + 'Version': '4.0.11' +} +component_changed_old = { + 'Architecture': 'noarch', + 'Build Date': 'Fri Oct 19 00:22:00 2018', + 'Build Host': 'f32725a719ce4c82a53b44644dfd2718', + 'Description': 'This RPM contains source code for the Authentication, Authorization and ' + 'Accounting cli', + 'FOSS': 'No', + 'From repo': 'localrepo', + 'Group': 'Unspecified', + 'Install Date': 'Wed Nov 7 21:22:14 2018', + 'Is sane': True, + 'License': 'Commercial', + 'Name': 'aaacli', + 'Obsoletes': '', + 'Packager': 'Something', + 'Release': '2.el7.centos', + 'Relocations': '(not relocatable)', + 'Repo': 'installed', + 'Repo data': { + 'baseurl': 'https://jenkins/ci-build/2432/artifact/results/repo', + 'name': 'localrepo'}, + 'Signature': '(none)', + 'Size': '43968', + 'Source RPM': 'aaacli-c12.gc62d348-2.el7.centos.src.rpm', + 'Source repo data': { + 'baseurl': 'https://jenkins/ci-build/2432/artifact/results/src_repo', + 'name': 'localrepo'}, + 'Source to be delivered': 'No', + 'Summary': 'Authentication, Authorization and Accounting Command Line Interface', + 'Vendor': 'Something', + 'Version': 'c12.gc62d348' +} + +component_changed_new = { + 'Architecture': 'noarch', + 'Build Date': 'Thu Nov 8 05:05:07 2018', + 'Build Host': 'build-3.novalocal', + 'Description': 'This RPM contains source code for the Authentication, ' + 'Authorization and Accounting cli', + 'FOSS': 'No', + 'From repo': 'localrepo', + 'Group': 'Unspecified', + 'Install Date': 'Tue Nov 13 19:09:30 2018', + 'Is sane': True, + 'License': 'Commercial', + 'Name': 'aaacli', + 'Obsoletes': '', + 'Packager': 'Something', + 'Release': '1.el7.centos', + 'Relocations': '(not relocatable)', + 'Repo': 'installed', + 'Repo data': { + 'baseurl': 'https://jenkins/ci-build/2490/artifact/results/repo', + 'name': 'localrepo'}, + 'Signature': '(none)', + 'Size': '44034', + 'Source RPM': 'aaacli-c13.gcb6b490-1.el7.centos.src.rpm', + 'Source repo data': { + 'baseurl': 'https://jenkins/ci-build/2490/artifact/results/src_repo', + 'name': 'localrepo'}, + 'Source to be delivered': 'No', + 'Summary': 'Authentication, Authorization and Accounting Command Line Interface', + 'Vendor': 'Something', + 'Version': 'c13.gcb6b490' +} + +grafana_v1 = { + 'Architecture': 'x86_64', + 'Build Date': 'Mon Sep 17 14:30:25 2018', + 'Build Host': 'd8fb00edf57f4254bb45073a941929ff', + 'Description': 'Grafana is an open source, feature rich metrics dashboard and graph ' + 'editor for\nGraphite, InfluxDB & OpenTSDB.', + 'FOSS': 'Modified', + 'From repo': 'localrepo', + 'Group': 'Unspecified', + 'Install Date': 'Fri Nov 16 02:23:02 2018', + 'Is sane': True, + 'License': 'Commercial and ASL 2.0 and others', + 'Name': 'grafana', + 'Obsoletes': '', + 'Packager': 'Something', + 'Release': '1.el7.centos.1', + 'Relocations': '(not relocatable)', + 'Repo': 'installed', + 'Repo data': { + 'baseurl': 'https://jenkins/ci-build/2506/artifact/results/repo', + 'name': 'localrepo'}, + 'Signature': '(none)', + 'Size': '93957067', + 'Source RPM': 'grafana-5.2.3-1.el7.centos.1.src.rpm', + 'Source repo data': { + 'baseurl': 'https://jenkins/ci-build/2506/artifact/results/src_repo', + 'name': 'localrepo'}, + 'Source to be delivered': 'Upstream', + 'Summary': 'Grafana is an open source, feature rich metrics dashboard and graph editor', + 'URL': 'https://github.com/grafana/grafana', + 'Vendor': 'Something and Grafana modified', + 'Version': '5.2.3' +} + +grafana_v2 = _copy(grafana_v1, '1.el7.centos.1', '1.el7.centos.2') diff --git a/tools/script/create_rpm_data.py b/tools/script/create_rpm_data.py new file mode 100755 index 0000000..9c4b747 --- /dev/null +++ b/tools/script/create_rpm_data.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=too-many-instance-attributes,too-many-arguments + +import argparse +import copy +import sys +import logging +import re +import json +from pprint import pformat + +import os + +from tools.rpm import RpmInfoParser +from tools.utils import apply_jenkins_auth +from tools.yum import Yum, YumInfoParser +from tools.repository import RepositoryConfig +from tools.log import set_logging +from tools.io import read_from, write_to, read_json +from tools.convert import to_json, CsvConverter + + +class RpmDataBuilder(object): + def __init__(self, build_config, yum_info_installed, rpm_info_installed, + crypto_info_installed, boms, remote=False): + self.remote = remote + self.yum_info_installed = yum_info_installed + self.rpm_info_installed = rpm_info_installed + self.crypto_info_installed = json.loads(crypto_info_installed) + self.boms = boms + logging.debug('BOMS: {}'.format(pformat(self.boms))) + self.repoconfig = RepositoryConfig(build_config) + self.installed_rpms = None + self.repos = None + + def run(self): + self.installed_rpms = self.read_installed_rpms() + srpms = set([rpm['Source RPM'] for rpm in self.installed_rpms]) + logging.info('Installed RPMs:{} SRPMs:{}'.format(len(self.installed_rpms), len(srpms))) + self.repos = self._read_configured_repos() + logging.info('Configured repos: {}'.format(len(self.repos))) + available_rpms = self._read_available_rpms(self.repos) + logging.info('Found {} available RPMs in binary repos'.format( + len([rpm for repo_rpms in available_rpms.values() for rpm in repo_rpms]))) + for i_rpm in self.installed_rpms: + i_rpm_repo_name = self._get_rpm_available_in(i_rpm, available_rpms) + i_rpm['Repo data'] = self._get_repo(i_rpm_repo_name) + i_rpm['Obsoletes'] = self._resolve_obsoletes(i_rpm) + i_rpm['Crypto capable'] = self._resolve_ecc(str(i_rpm)) + i_rpm['BOM'] = self._resolve_bom(i_rpm) + self._log_repo_rpm_statistics() + self._log_rpm_statistics() + return self.installed_rpms + + @staticmethod + def _resolve_obsoletes(rpm): + if 'Obsoletes' not in rpm: + return 'N/A' + elif rpm['Obsoletes'] == '(none)': + return 'N/A' + return rpm['Obsoletes'] + + def _resolve_ecc(self, rpm): + for item in self.crypto_info_installed: + if item['name'] == rpm: + return True + return False + + def _resolve_bom(self, rpm): + bom_content = self.boms.get(str(rpm)) + if bom_content is None: + return '' + self._validate_bom(str(rpm), bom_content) + return bom_content['bom'] + + @staticmethod + def _validate_bom(rpm_name, bom_content): + try: + if 'bom' not in bom_content: + raise Exception('BOM base object "bom" missing') + bom = bom_content['bom'] + for material in bom: + for key in ['name', 'version', 'source-url', 'foss']: + if key not in material: + raise Exception('Key "{}" not found in BOM'.format(key)) + if material['foss'].lower() not in ['yes', 'no', 'modified']: + raise Exception('BOM foss value not valid') + missing_crypto_count = len([material for material in bom if + 'crypto-capable' not in material]) + if missing_crypto_count != 0: + logging.warning( + 'crypto-capable missing from %s materials in RPM %s', + missing_crypto_count, rpm_name) + except Exception as e: + correct_format = {'bom': [ + {'name': '', + 'version': '', + 'source-url': '', + 'foss': '', + 'crypto-capable': ''}]} + msg_fmt = 'BOM for {rpm} is not correct format. {error}:\n{correct_format}' + raise Exception(msg_fmt.format(rpm=rpm_name, + error=str(e), + correct_format=pformat(correct_format))) + + def _get_repo(self, name): + for r in self.repos: + if r['name'] == name: + return r + raise Exception('No repository found with name: {}'.format(name)) + + def read_installed_rpms(self): + installed_rpms = [] + yum_rpms = YumInfoParser().parse_installed(self.yum_info_installed) + rpm_rpms = RpmInfoParser().parse_multiple(self.rpm_info_installed) + self._validate_rpm_lists_identical(yum_rpms, rpm_rpms) + yum_rpms_dict = {rpm['Name']: rpm for rpm in yum_rpms} + for rpm_data in rpm_rpms: + yum_data = yum_rpms_dict[rpm_data['Name']] + combined_data = self._combine_rpm_data(rpm_data, yum_data) + installed_rpms.append(combined_data) + logging.debug('One parsed RPM data as example:\n{}'.format(pformat(installed_rpms[0]))) + return installed_rpms + + def _combine_rpm_data(self, rpm_data, yum_data): + combined_data = copy.deepcopy(rpm_data) + fields_known_to_differ = ['Description', # May contain deffering newline and indentation + 'Size'] # Bytes in RPM, humanreadable in yum + yum2rpm_field_name_map = {'Arch': 'Architecture'} + for yum_key in yum_data: + if yum_key in yum2rpm_field_name_map: + rpm_key = yum2rpm_field_name_map[yum_key] + else: + rpm_key = yum_key + if rpm_key in combined_data: + yum_comparable_rpm_string = self._rpm_info_str_to_yum_info_str( + combined_data[rpm_key]) + if yum_comparable_rpm_string != yum_data[yum_key]: + if rpm_key in fields_known_to_differ: + continue + raise Exception( + 'RPM data in "{}" not match in rpm "{}" vs yum "{}" for package {}'.format( + rpm_key, + repr(combined_data[rpm_key]), + repr(yum_data[yum_key]), + combined_data)) + else: + combined_data[rpm_key] = yum_data[yum_key] + return combined_data + + @staticmethod + def _rpm_info_str_to_yum_info_str(string): + try: + string.decode() + except (UnicodeEncodeError, UnicodeDecodeError): + return re.sub(r'[^\x00-\x7F]+', '?', string) + except Exception as e: + logging.error('{}: for string {}'.format(str(e), repr(string))) + raise + return string + + @staticmethod + def _validate_rpm_lists_identical(yum_rpms, rpm_rpms): + yum_rpms_dict = {rpm['Name']: rpm for rpm in yum_rpms} + rpm_rpms_dict = {rpm['Name']: rpm for rpm in rpm_rpms} + if len(yum_rpms) != len(rpm_rpms): + raise Exception( + 'Given RPM lists are unequal: yum RPM count {} != rpm RPM count {}'.format( + len(yum_rpms), len(rpm_rpms))) + assert sorted(yum_rpms_dict.keys()) == sorted(rpm_rpms_dict.keys()) + for name in yum_rpms_dict.keys(): + if not yum_rpms_dict[name].is_same_package_as(rpm_rpms_dict[name]): + raise Exception( + 'Packages are not same: yum {} != rpm {}'.format(yum_rpms_dict[name], + rpm_rpms_dict[name])) + + def _read_configured_repos(self): + repos = self.repoconfig.read_sections( + ['baseimage-repositories', 'repositories']) + repos.append(self.repoconfig.get_localrepo(remote=True)) + logging.debug('Configured repos: {}'.format(pformat(repos))) + return repos + + def _read_available_rpms(self, repos): + Yum.clean_and_remove_cache() + yum = Yum() + for repo in repos: + name = repo['name'] + if name == 'localrepo': + if self.remote: + url = self.repoconfig.get_localrepo(remote=True)['baseurl'] + yum.add_repo(name, apply_jenkins_auth(url)) + else: + url = self.repoconfig.get_localrepo(remote=False)['baseurl'] + yum.add_repo(name, url) + else: + yum.add_repo(name, repo['baseurl']) + yum_available_output = yum.read_all_packages() + available_rpms = YumInfoParser().parse_available(yum_available_output) + rpms_per_repo = {} + for rpm in available_rpms: + repo = rpm.get('Repo') + if repo not in rpms_per_repo: + rpms_per_repo[repo] = [] + rpms_per_repo[repo].append(rpm) + return rpms_per_repo + + def _log_repo_rpm_statistics(self): + logging.info('--- RPM repo statistics ---') + for repo in self.repos: + name = repo['name'] + repo_url = repo['baseurl'] + if name in [r['name'] for r in self._get_nonerepos()]: + expected_from_repo = None + else: + expected_from_repo = name + repo_installed_rpm_count = len([rpm for rpm in self.installed_rpms if + rpm['Repo data']['baseurl'] == repo_url and rpm.get( + 'From repo') == expected_from_repo]) + logging.info( + 'RPMs installed from repo "{}": {}'.format(name, repo_installed_rpm_count)) + if repo_installed_rpm_count is 0: + logging.warning( + 'Repository configured but no RPMs installed: {}={}'.format(name, repo_url)) + + return self.installed_rpms + + def _log_rpm_statistics(self): + def _get_count(func): + return len([rpm for rpm in self.installed_rpms if func(rpm)]) + + logging.info('----- RPMs per type -----') + logging.info(' => Total: %s', len(self.installed_rpms)) + logging.info('----- RPMs per attribute -----') + logging.info(' * Crypto capable: %s', _get_count(lambda rpm: rpm['Crypto capable'])) + logging.info(' * Complex (BOM): %s', _get_count(lambda rpm: rpm['BOM'])) + + def _get_rpm_available_in(self, rpm, available_rpms): + if 'From repo' in rpm.keys(): + if rpm['From repo'] == 'localrepo': + return 'localrepo' + available_repo_rpms = available_rpms[rpm['From repo']] + for a_rpm in available_repo_rpms: + if self._is_same_rpm(a_rpm, rpm): + return rpm['From repo'] + rpms_in_matching_repo = [str(a_rpm) for a_rpm in available_repo_rpms] + rpms_with_matching_name = [str(a_rpm) for a_rpm in available_repo_rpms if + rpm['Name'] == a_rpm['Name']] + if len(rpms_in_matching_repo) <= 1000: + logging.debug( + 'Available RPMs in {}: {}'.format(rpm['From repo'], rpms_in_matching_repo)) + error_str = 'RPM "{}" is not available in configured repo: {}, ' \ + 'RPMs with correct name: {}'.format(str(rpm), rpm['From repo'], + rpms_with_matching_name) + raise Exception(error_str) + else: + none_repos = self._get_nonerepos() + for repo in [r['name'] for r in none_repos]: + for a_rpm in available_rpms[repo]: + if self._is_same_rpm(a_rpm, rpm): + return repo + msg = 'RPM "{}" is not available in any configured "none*" repos: {}'.format( + rpm['Name'], none_repos) + raise Exception(msg) + + def _get_nonerepos(self): + return [repo for repo in self.repos if re.match(r'^none\d+$', repo['name'])] + + @staticmethod + def _is_same_rpm(rpm1, rpm2): + return rpm1['Name'] == rpm2['Name'] and \ + rpm1['Version'] == rpm2['Version'] and \ + rpm1['Release'] == rpm2['Release'] and \ + rpm1['Arch'] == rpm2['Architecture'] + + +def parse(args): + p = argparse.ArgumentParser( + description='Generate package info', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + p.add_argument('--verbose', '-v', action='store_true', + help='More verbose logging') + p.add_argument('--yum-info-path', required=True, + help='"yum info all" output as file') + p.add_argument('--rpm-info-path', required=True, + help='"rpm -qai" output as file') + p.add_argument('--crypto-info-path', + help='Dir from where to find ECC file') + p.add_argument('--boms-path', + help='Dir from where to find RPM bill of material files') + p.add_argument('--output-rpmlist', + help='output as rpm list like "rpm-qa"') + p.add_argument('--output-json', + help='output json file path') + p.add_argument('--output-csv', + help='output csv file path') + p.add_argument('--output-ms-csv', + help='output Microsoft Excel compatible csv file path') + p.add_argument('--build-config-path', required=True, + help='Build configuration ini path') + p.add_argument('--remote', action='store_true', + help='Read localrepo from remote defined by BUILD_URL, ' + 'otherwise use localrepo from WORKSPACE') + args = p.parse_args(args) + return args + + +def read_files(boms_dir): + boms = {} + for f in os.listdir(boms_dir): + boms[f] = read_json(boms_dir + '/' + f) + return boms + + +def main(input_args): + args = parse(input_args) + if args.verbose: + set_logging(debug=True, timestamps=True) + else: + set_logging(debug=False) + rpmdata = RpmDataBuilder(args.build_config_path, + read_from(args.yum_info_path), + read_from(args.rpm_info_path), + read_from(args.crypto_info_path), + read_files(args.boms_path), + remote=args.remote).run() + if args.output_rpmlist: + write_to(args.output_rpmlist, '\n'.join(sorted([str(rpm) for rpm in rpmdata]))) + if args.output_json: + write_to(args.output_json, to_json(rpmdata)) + csv = CsvConverter(rpmdata, preferred_field_order=['Name', 'Version', 'Release', + 'License', 'Vendor', 'From repo', + 'Source RPM']) + if args.output_csv: + write_to(args.output_csv, str(csv)) + if args.output_ms_csv: + write_to(args.output_ms_csv, + csv.convert_to_ms_excel(text_fields=['Version', 'Size', 'Release'])) + if not args.output_json and not args.output_csv: + print(rpmdata) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/tools/script/create_rpm_data_test.py b/tools/script/create_rpm_data_test.py new file mode 100755 index 0000000..1e47e89 --- /dev/null +++ b/tools/script/create_rpm_data_test.py @@ -0,0 +1,84 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=invalid-name +import os +import sys +import logging +import pytest +import mock + +from tools.script.create_rpm_data import RpmDataBuilder +from tools.repository import BuildConfigParser +from tools.executor import Result +import tools.repository +from tools.script.create_rpm_data_test_data import \ + yum_installed_output, \ + rpm_info_output, \ + expected_output, \ + basesystem_combined, \ + cpp_combined, \ + centos_logos_combined, \ + dejavu_fonts_common_combined, yum_available_output, \ + crypto_rpms_json, boms_output + +from tools.test_data_rpm import basesystem_rpm_info, cpp_rpm_info, centos_logos_rpm_info, \ + dejavu_fonts_common_rpm_info +from tools.test_data_yum import basesystem_yum_info, cpp_yum_info, centos_logos_yum_info, \ + dejavu_fonts_common_yum_info +from tools.yum import YumConfig + + +@mock.patch.object(YumConfig, 'write') +@mock.patch.object(tools.yum, 'run') +@mock.patch.object(BuildConfigParser, 'items') +def test_complete_parse(mock_config, mock_reporeader, _): + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + mock_config.side_effect = [ + [('base', 'test-url-for-base'), + ('none1', 'test-url-for-none1'), + ('none2', 'test-url-for-none2')], + [('purkki-3rdparty', 'test-url-for-purkki-3rdparty#test-extra-option=test-value')], + [('base', 'src-test-url-for-base'), + ('none1', 'src-test-url-for-none1'), + ('none2', 'src-test-url-for-none2')], + [('purkki-3rdparty', 'src-test-url-for-purkki-3rdparty#test-extra-option2=test-value2')] + ] + mock_reporeader.side_effect = [Result(0, '', ''), # yum clean all + Result(0, '', ''), # rm -rf /var/yum/cache + Result(0, yum_available_output, '')] + os.environ['BUILD_URL'] = 'test-url/' + os.environ['WORKSPACE'] = '/foo/path' + result = RpmDataBuilder('fake_build_config_path', + yum_installed_output, + rpm_info_output, + crypto_rpms_json, + boms_output).run() + assert mock_reporeader.call_count == 3 + assert result == expected_output + + +@pytest.mark.parametrize('yum_info, rpm_info, expected_combined', [ + (basesystem_yum_info, basesystem_rpm_info, basesystem_combined), + (cpp_yum_info, cpp_rpm_info, cpp_combined), + (centos_logos_yum_info, centos_logos_rpm_info, centos_logos_combined), + (dejavu_fonts_common_yum_info, dejavu_fonts_common_rpm_info, dejavu_fonts_common_combined), +]) +def test_combine_rpm_data(yum_info, rpm_info, expected_combined): + result = RpmDataBuilder('fake_build_config_path', + yum_info, + rpm_info, + '[]', + {}).read_installed_rpms() + assert result[0] == expected_combined diff --git a/tools/script/create_rpm_data_test_data.py b/tools/script/create_rpm_data_test_data.py new file mode 100755 index 0000000..e7d38f3 --- /dev/null +++ b/tools/script/create_rpm_data_test_data.py @@ -0,0 +1,303 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=invalid-name +from tools.rpm_test_data import cpp_expected + +yum_installed_output = """Loaded plugins: fastestmirror, priorities +Loading mirror speeds from cached hostfile +Installed Packages +Name : non-repo-pkg-1 +Arch : x86_64 +Version : 1 +Release : 1 +Repo : installed + +Name : non-repo-pkg-2 +Arch : noarch +Version : 2 +Release : 2 +Repo : installed +Obsoletes : (none) + +Name : base-image-pkg +Arch : x86_64 +Version : 3 +Release : 3 +Repo : installed +From repo : base + +Name : internal-pkg +Arch : noarch +Version : 4 +Release : 4 +Repo : installed +From repo : localrepo + +Name : 3rdparty-pkg +Arch : x86_64 +Version : 5 +Release : 5 +Repo : installed +From repo : purkki-3rdparty +Obsoletes : 2ndparty-pkg + +""" + +rpm_info_output = """Name : non-repo-pkg-1 +Version : 1 +Release : 1 +Architecture: x86_64 +Source RPM : non-repo-pkg-1-1-1.src.rpm +Name : non-repo-pkg-2 +Version : 2 +Release : 2 +Architecture: noarch +Source RPM : non-repo-pkg-2-2-2.src.rpm +Name : base-image-pkg +Version : 3 +Release : 3 +Architecture: x86_64 +Source RPM : base-image-pkg-3-3.src.rpm +Name : internal-pkg +Version : 4 +Release : 4 +Architecture: noarch +Source RPM : internal-pkg-4-4.src.rpm +Name : 3rdparty-pkg +Version : 5 +Release : 5 +Architecture: x86_64 +Source RPM : 3rdparty-pkg-5-5.src.rpm +""" # noqa, PEP-8 disabled because of example output has trailing spaces + +yum_available_output_header = """Added tmprepo repo from http://url1/ +Available Packages +""" + +yum_available_output_base = """ +Name : base-image-pkg +Arch : x86_64 +Epoch : 0 +Version : 3 +Release : 3 +Size : 195 k +Repo : base +""" + +yum_available_output_none2 = """ +Name : non-repo-pkg-1 +Arch : x86_64 +Epoch : 0 +Version : 1 +Release : 1 +Size : 195 k +Repo : none2 +""" + +yum_available_output_none1 = """ +Name : non-repo-pkg-2 +Arch : noarch +Version : 2 +Release : 2 +Size : 195 k +Repo : none1 +""" + +yum_available_output_localrepo = """ +Name : internal-pkg +Arch : x86_64 +Epoch : 0 +Version : 4 +Release : 4 +Size : 195 k +Repo : localrepo +""" + +yum_available_output_purkki_3rdparty = """ +Name : 3rdparty-pkg +Arch : x86_64 +Epoch : 0 +Version : 5 +Release : 5 +Size : 195 k +Repo : purkki-3rdparty +""" + +yum_available_output = yum_available_output_header + yum_available_output_base + \ + yum_available_output_none1 + yum_available_output_none2 + \ + yum_available_output_localrepo + yum_available_output_purkki_3rdparty + +internal_pkg_bom = [{'name': '@types/d3-axis', + 'version': '1.0.10', + 'foss': 'yes', + 'source-url': 'http://some.url/1', + 'crypto-capable': True}, + {'name': '@types/d3-array@*', + 'version': '1.2.1', + 'foss': 'Yes', + 'source-url': 'http://some.url/2'}] +boms_output = {'internal-pkg-4-4.noarch': {"bom": internal_pkg_bom}} + +expected_output = [ + { + 'Name': 'non-repo-pkg-1', + 'Architecture': 'x86_64', + 'Version': '1', + 'Release': '1', + 'Repo': 'installed', + 'Repo data': {'baseurl': 'test-url-for-none2', 'name': 'none2'}, + 'Obsoletes': 'N/A', + 'Source RPM': 'non-repo-pkg-1-1-1.src.rpm', + 'Crypto capable': False, + 'BOM': '', + }, { + 'Name': 'non-repo-pkg-2', + 'Architecture': 'noarch', + 'Version': '2', + 'Release': '2', + 'Repo': 'installed', + 'Repo data': {'baseurl': 'test-url-for-none1', 'name': 'none1'}, + 'Obsoletes': 'N/A', + 'Source RPM': 'non-repo-pkg-2-2-2.src.rpm', + 'Crypto capable': False, + 'BOM': '', + }, { + 'Name': 'base-image-pkg', + 'Architecture': 'x86_64', + 'Version': '3', + 'Release': '3', + 'Repo': 'installed', + 'From repo': 'base', + 'Repo data': {'baseurl': 'test-url-for-base', 'name': 'base'}, + 'Obsoletes': 'N/A', + 'Source RPM': 'base-image-pkg-3-3.src.rpm', + 'Crypto capable': False, + 'BOM': '', + }, { + 'Name': 'internal-pkg', + 'Architecture': 'noarch', + 'Version': '4', + 'Release': '4', + 'Repo': 'installed', + 'From repo': 'localrepo', + 'Repo data': {'baseurl': 'test-url/artifact/results/repo', + 'name': 'localrepo'}, + 'Obsoletes': 'N/A', + 'Source RPM': 'internal-pkg-4-4.src.rpm', + 'Crypto capable': True, + 'BOM': internal_pkg_bom, + }, { + 'Name': '3rdparty-pkg', + 'Architecture': 'x86_64', + 'Version': '5', + 'Release': '5', + 'Repo': 'installed', + 'From repo': 'purkki-3rdparty', + 'Repo data': {'baseurl': 'test-url-for-purkki-3rdparty', 'name': 'purkki-3rdparty', + 'test-extra-option': 'test-value'}, + 'Obsoletes': '2ndparty-pkg', + 'Source RPM': '3rdparty-pkg-5-5.src.rpm', + 'Crypto capable': False, + 'BOM': '', + }] + +basesystem_combined = { + # From RPM info + 'Name': 'basesystem', + 'Version': '10.0', + 'Release': '7.el7.centos', + 'Architecture': 'noarch', + 'Install Date': 'Fri 01 Apr 2016 11:47:25 AM EEST', + 'Group': 'System Environment/Base', + 'Size': '0', + 'License': 'Public Domain', + 'Signature': 'RSA/SHA256, Fri 04 Jul 2014 03:46:57 AM EEST, Key ID 24c6a8a7f4a80eb5', + 'Source RPM': 'basesystem-10.0-7.el7.centos.src.rpm', + 'Build Date': 'Fri 27 Jun 2014 01:37:10 PM EEST', + 'Build Host': 'worker1.bsys.centos.org', + 'Relocations': '(not relocatable)', + 'Packager': 'CentOS BuildSystem ', + 'Vendor': 'CentOS', + 'Summary': 'The skeleton package which defines a simple CentOS Linux system', + 'Description': '\n'.join( + ['Basesystem defines the components of a basic CentOS Linux', + 'system (for example, the package installation order to use during', + 'bootstrapping). Basesystem should be in every installation of a system,', + 'and it should never be removed.']), + # From yum info + 'Repo': 'installed', +} + +centos_logos_combined = { + 'Name': 'centos-logos', + 'Version': '70.0.6', + 'Release': '3.el7.centos', + 'Architecture': 'noarch', + 'License': u'Copyright © 2014 The CentOS Project. All rights reserved.', +} + +cpp_combined = cpp_expected.copy() +cpp_combined.update({'Repo': 'installed', + 'From repo': 'purkki-centos-base'}) + +dejavu_fonts_common_combined = { + 'Name': 'dejavu-fonts-common', + 'Version': '2.33', + 'Release': '6.el7', + 'Architecture': 'noarch', + 'Install Date': 'Wed Feb 7 13:49:27 2018', + 'Group': 'User Interface/X', + 'Size': '130455', + 'License': 'Bitstream Vera and Public Domain', + 'Signature': 'RSA/SHA256, Fri Jul 4 01:06:50 2014, Key ID 24c6a8a7f4a80eb5', + 'Source RPM': 'dejavu-fonts-2.33-6.el7.src.rpm', + 'Build Date': 'Mon Jun 9 21:34:30 2014', + 'Build Host': 'worker1.bsys.centos.org', + 'Relocations': '(not relocatable)', + 'Packager': 'CentOS BuildSystem ', + 'Vendor': 'CentOS', + 'URL': 'http://dejavu-fonts.org/', + 'Summary': 'Common files for the Dejavu font set', + 'Description': '\n'.join( + ['The DejaVu font set is based on the “Bitstream Vera” fonts, release 1.10. Its', + 'purpose is to provide a wider range of characters, while maintaining the', + 'original style, using an open collaborative development process.', + '', + 'This package consists of files used by other DejaVu packages.']), + 'Repo': 'installed', + 'From repo': 'purkki-centos-base' +} + +crypto_rpms_json = """ +[ + { + "name": "internal-pkg-4-4.noarch", + "requires": [ + "libgssapi_krb5.so.2()(64bit)", + "libk5crypto.so.3()(64bit)", + "libkrb5.so.3()(64bit)", + "libcrypto.so.10()(64bit)", + "libcrypto.so.10(OPENSSL_1.0.1_EC)(64bit)", + "libcrypto.so.10(OPENSSL_1.0.2)(64bit)", + "libcrypto.so.10(libcrypto.so.10)(64bit)", + "libssl.so.10()(64bit)", + "libssl.so.10(libssl.so.10)(64bit)", + "openssl-libs(x86-64)", + "rtld(GNU_HASH)" + ] + } +] +""" diff --git a/tools/script/generate_repo_files.py b/tools/script/generate_repo_files.py new file mode 100755 index 0000000..07fdbb3 --- /dev/null +++ b/tools/script/generate_repo_files.py @@ -0,0 +1,74 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import argparse +import logging + +from tools.buildconfig import BuildConfigParser + + +def _parse(args): + parser = argparse.ArgumentParser(description='Generate repo files from config ini sections') + parser.add_argument('config_sections', metavar='config_section', nargs='+', + help='Config ini section') + parser.add_argument('--output-dir', '-d', required=True, + help='Directory to output the repo files') + parser.add_argument('--config-ini', '-c', required=True, + help='Path to the config ini to read') + args = parser.parse_args(args) + return args + + +class RepoGen(object): + + def __init__(self, output_dir, config_ini, config_sections): + self.output_dir = output_dir + self.config_sections = config_sections + self.config = BuildConfigParser(ini_file=config_ini) + + def run(self): + for section in self.config_sections: + repo_file_path = os.path.join(self.output_dir, section + '.repo') + self._write_repo_file(section, repo_file_path) + + def _write_repo_file(self, section, repo_file_path): + with open(repo_file_path, 'w') as f: + for repo in self.config.items(section): + name = repo[0] + parts = repo[1].split('#') + url = parts[0] + f.write('[%s]\n' % name) + f.write('name=%s\n' % name) + f.write('baseurl=%s\n' % url) + f.write('enabled=1\n') + f.write('gpgcheck=0\n') + for part in parts[1:]: + f.write('%s\n' % part) + f.write('\n') + if os.path.getsize(repo_file_path) == 0: + logging.error('Zero size output: {}'.format(repo_file_path)) + sys.exit(1) + logging.info('Wrote repo: {}'.format(repo_file_path)) + + +def main(input_args): + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + args = _parse(input_args) + RepoGen(args.output_dir, args.config_ini, args.config_sections).run() + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/tools/script/process_rpmdata.py b/tools/script/process_rpmdata.py new file mode 100755 index 0000000..bfab70b --- /dev/null +++ b/tools/script/process_rpmdata.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import sys + +from tools.convert import to_json, CsvConverter +from tools.log import set_logging +from tools.io import write_to, read_json + + +class RpmDataProcesser(object): + def __init__(self, rpmdata): + self.components = self._get_components(rpmdata) + + @staticmethod + def _get_components(rpmdata): + components = [] + bom_field_map = {'name': 'Name', + 'version': 'Version', + 'foss': 'FOSS', + 'source-url': 'Source URL', + 'crypto-capable': 'Crypto capable'} + for rpm in rpmdata: + components.append(unicode_recursively(rpm)) + bom = rpm['BOM'] + if bom: + for material in bom: + component = {'Source RPM': rpm['Source RPM']} + for field in material: + component[bom_field_map[field]] = material[field] + components.append(unicode_recursively(component)) + return components + + def gen_components(self, path): + write_to(path, to_json(self.components)) + + def gen_components_csv(self, path): + csv = CsvConverter(self.components, + preferred_field_order=['Name', 'Version', 'Release', 'Source RPM']) + write_to(path, csv.convert_to_ms_excel(text_fields=['Version'])) + + +def unicode_recursively(something): + if isinstance(something, dict): + return {unicode_recursively(key): unicode_recursively(value) for key, value in + something.iteritems()} + elif isinstance(something, list): + return [unicode_recursively(element) for element in something] + elif isinstance(something, unicode): + return something.encode('utf-8') + return something + + +def parse(args): + p = argparse.ArgumentParser( + description='Process rpmdata for multitude of tools', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + p.add_argument('--verbose', '-v', action='store_true', + help='More verbose logging') + p.add_argument('--rpmdata-path', required=True, + help='RPM data json file path') + p.add_argument('--output-components', + help='Component list that includes also RPM sub-components') + p.add_argument('--output-components-csv', + help='Component list that includes also RPM sub-components as CSV') + args = p.parse_args(args) + return args + + +def main(input_args): + args = parse(input_args) + if args.verbose: + set_logging(debug=True) + else: + set_logging(debug=False) + p = RpmDataProcesser(read_json(args.rpmdata_path)) + if args.output_components: + p.gen_components(args.output_components) + if args.output_components_csv: + p.gen_components_csv(args.output_components_csv) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/tools/script/process_rpmdata_test.py b/tools/script/process_rpmdata_test.py new file mode 100755 index 0000000..d704bf7 --- /dev/null +++ b/tools/script/process_rpmdata_test.py @@ -0,0 +1,58 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +from tools.convert import CsvConverter +from tools.script.create_rpm_data_test_data import expected_output +from tools.script.process_rpmdata import main + + +def gen_input(tmpdir, rpmdata): + p = tmpdir.join('rpmdata.json') + p.write(rpmdata) + return str(p) + + +def test_no_output(tmpdir): + main(['--rpmdata-path', gen_input(tmpdir, {})]) + + +def test_components(tmpdir): + input_content = [expected_output[0], + expected_output[3]] + input_ = json.dumps(input_content) + output_json = tmpdir.join('components.json') + output_csv = tmpdir.join('components.csv') + main(['--rpmdata-path', gen_input(tmpdir, input_), + '--output-components', str(output_json), + '--output-components-csv', str(output_csv)]) + additions = [{'FOSS': u'yes', + 'Source RPM': 'internal-pkg-4-4.src.rpm', + 'Name': '@types/d3-axis', + 'Source URL': 'http://some.url/1', + 'Version': '1.0.10', + 'Crypto capable': True}, + {'FOSS': u'Yes', + 'Source RPM': 'internal-pkg-4-4.src.rpm', + 'Name': '@types/d3-array@*', + 'Source URL': 'http://some.url/2', + 'Version': '1.2.1'}] + assert json.loads(output_json.read()) == input_content + additions + assert output_csv.read() == _gen_components_csv(input_content + additions) + + +def _gen_components_csv(_list): + csv = CsvConverter(_list, preferred_field_order=['Name', 'Version', 'Release', 'Source RPM']) + return csv.convert_to_ms_excel(text_fields=['Version']) diff --git a/tools/script/read_build_config.py b/tools/script/read_build_config.py new file mode 100755 index 0000000..1e6400a --- /dev/null +++ b/tools/script/read_build_config.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +from tools.buildconfig import BuildConfigParser + + +def main(): + config = BuildConfigParser(sys.argv[1]) + if len(sys.argv) == 3: + print(config.items(sys.argv[2])) + elif len(sys.argv) == 4: + print(config.get(sys.argv[2], sys.argv[3])) + else: + raise Exception('Invalid parameter count: {}'.format(len(sys.argv))) + + +if __name__ == "__main__": + main() diff --git a/tools/script/read_package_config.py b/tools/script/read_package_config.py new file mode 100755 index 0000000..fc0b8b2 --- /dev/null +++ b/tools/script/read_package_config.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import sys + +from tools.package import PackageConfigReader + + +def parse(args): + p = argparse.ArgumentParser( + description='Read package configuration', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + p.add_argument('--config', required=False, help='Package YAML config file path') + p.add_argument('--separator', required=False, default=' ', + help='Separator for the resulting stdout list') + p.add_argument('package_operation_type', choices=['install', 'uninstall'], + help='list packages by operation type') + args = p.parse_args(args) + return args + + +def main(input_args): + args = parse(input_args) + kwargs = {} + if args.config is not None: + kwargs['config_file'] = args.config + print(args.separator.join(PackageConfigReader(**kwargs).get(args.package_operation_type))) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/tools/statics.py b/tools/statics.py new file mode 100755 index 0000000..f1ebd9c --- /dev/null +++ b/tools/statics.py @@ -0,0 +1,26 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +WORK_ROOT = os.path.dirname( + os.path.dirname( + os.path.dirname( + os.path.dirname( + os.path.realpath(__file__))))) + +MAINFEST_PATH = os.path.join(WORK_ROOT, '.repo/manifests') + +BUILD_CONFIG_PATH = os.path.join(MAINFEST_PATH, 'build_config.ini') +PACKAGES_CONFIG_PATH = os.path.join(MAINFEST_PATH, 'packages.yaml') diff --git a/tools/test_data_rpm.py b/tools/test_data_rpm.py new file mode 100755 index 0000000..440a2bf --- /dev/null +++ b/tools/test_data_rpm.py @@ -0,0 +1,230 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=invalid-name,line-too-long + +bash_rpm_info = """Name : bash +Version : 4.2.46 +Release : 21.el7_3 +Architecture: x86_64 +Install Date: Thu 11 Jan 2018 12:32:51 PM EET +Group : System Environment/Shells +Size : 3663714 +License : GPLv3+ +Signature : RSA/SHA256, Wed 07 Dec 2016 02:11:28 AM EET, Key ID 24c6a8a7f4a80eb5 +Source RPM : bash-4.2.46-21.el7_3.src.rpm +Build Date : Wed 07 Dec 2016 01:21:54 AM EET +Build Host : c1bm.rdu2.centos.org +Relocations : (not relocatable) +Packager : CentOS BuildSystem +Vendor : CentOS +URL : http://www.gnu.org/software/bash +Summary : The GNU Bourne Again shell +Description : +The GNU Bourne Again shell (Bash) is a shell or command language +interpreter that is compatible with the Bourne shell (sh). Bash +incorporates useful features from the Korn shell (ksh) and the C shell +(csh). Most sh scripts can be run by bash without modification. +""" + +basesystem_rpm_info = """Name : basesystem +Version : 10.0 +Release : 7.el7.centos +Architecture: noarch +Install Date: Fri 01 Apr 2016 11:47:25 AM EEST +Group : System Environment/Base +Size : 0 +License : Public Domain +Signature : RSA/SHA256, Fri 04 Jul 2014 03:46:57 AM EEST, Key ID 24c6a8a7f4a80eb5 +Source RPM : basesystem-10.0-7.el7.centos.src.rpm +Build Date : Fri 27 Jun 2014 01:37:10 PM EEST +Build Host : worker1.bsys.centos.org +Relocations : (not relocatable) +Packager : CentOS BuildSystem +Vendor : CentOS +Summary : The skeleton package which defines a simple CentOS Linux system +Description : +Basesystem defines the components of a basic CentOS Linux +system (for example, the package installation order to use during +bootstrapping). Basesystem should be in every installation of a system, +and it should never be removed. +""" + +centos_logos_rpm_info = u"""Name : centos-logos +Version : 70.0.6 +Release : 3.el7.centos +Architecture: noarch +License : Copyright © 2014 The CentOS Project. All rights reserved. + +""" + +conntrack_tools_rpm_info = """Name : conntrack-tools +Version : 1.4.4 +Release : 3.el7_3 +Architecture: x86_64 +Install Date: Thu 11 Jan 2018 12:39:20 PM EET +Group : System Environment/Base +Size : 562826 +License : GPLv2 +Signature : RSA/SHA256, Thu 29 Jun 2017 03:36:05 PM EEST, Key ID 24c6a8a7f4a80eb5 +Source RPM : conntrack-tools-1.4.4-3.el7_3.src.rpm +Build Date : Thu 29 Jun 2017 03:18:42 AM EEST +Build Host : c1bm.rdu2.centos.org +Relocations : (not relocatable) +Packager : CentOS BuildSystem +Vendor : CentOS +URL : http://netfilter.org +Summary : Manipulate netfilter connection tracking table and run High Availability +Description : +With conntrack-tools you can setup a High Availability cluster and +synchronize conntrack state between multiple firewalls. + +The conntrack-tools package contains two programs: +- conntrack: the command line interface to interact with the connection + tracking system. +- conntrackd: the connection tracking userspace daemon that can be used to + deploy highly available GNU/Linux firewalls and collect + statistics of the firewall use. + +conntrack is used to search, list, inspect and maintain the netfilter +connection tracking subsystem of the Linux kernel. +Using conntrack, you can dump a list of all (or a filtered selection of) +currently tracked connections, delete connections from the state table, +and even add new ones. +In addition, you can also monitor connection tracking events, e.g. +show an event message (one line) per newly established connection. +""" + +cpp_rpm_info = """Name : cpp +Version : 4.8.5 +Release : 11.el7 +Architecture: x86_64 +Install Date: Thu 11 Jan 2018 12:37:55 PM EET +Group : Development/Languages +Size : 15632501 +License : GPLv3+ and GPLv3+ with exceptions and GPLv2+ with exceptions and LGPLv2+ and BSD +Signature : RSA/SHA256, Sun 20 Nov 2016 07:27:00 PM EET, Key ID 24c6a8a7f4a80eb5 +Source RPM : gcc-4.8.5-11.el7.src.rpm +Build Date : Fri 04 Nov 2016 06:01:22 PM EET +Build Host : worker1.bsys.centos.org +Relocations : (not relocatable) +Packager : CentOS BuildSystem +Vendor : CentOS +URL : http://gcc.gnu.org +Summary : The C Preprocessor +Description : +Cpp is the GNU C-Compatible Compiler Preprocessor. +Cpp is a macro processor which is used automatically +by the C compiler to transform your program before actual +compilation. It is called a macro processor because it allows +you to define macros, abbreviations for longer +constructs. + +The C preprocessor provides four separate functionalities: the +inclusion of header files (files of declarations that can be +substituted into your program); macro expansion (you can define macros, +and the C preprocessor will replace the macros with their definitions +throughout the program); conditional compilation (using special +preprocessing directives, you can include or exclude parts of the +program according to various conditions); and line control (if you use +a program to combine or rearrange source files into an intermediate +file which is then compiled, you can use line control to inform the +compiler about where each source line originated). + +You should install this package if you are a C programmer and you use +macros. +""" # noqa, PEP-8 disabled because of example output has trailing spaces + +dejavu_fonts_common_rpm_info = """Name : dejavu-fonts-common +Version : 2.33 +Release : 6.el7 +Architecture: noarch +Install Date: Wed Feb 7 13:49:27 2018 +Group : User Interface/X +Size : 130455 +License : Bitstream Vera and Public Domain +Signature : RSA/SHA256, Fri Jul 4 01:06:50 2014, Key ID 24c6a8a7f4a80eb5 +Source RPM : dejavu-fonts-2.33-6.el7.src.rpm +Build Date : Mon Jun 9 21:34:30 2014 +Build Host : worker1.bsys.centos.org +Relocations : (not relocatable) +Packager : CentOS BuildSystem +Vendor : CentOS +URL : http://dejavu-fonts.org/ +Summary : Common files for the Dejavu font set +Description : + +The DejaVu font set is based on the “Bitstream Vera” fonts, release 1.10. Its +purpose is to provide a wider range of characters, while maintaining the +original style, using an open collaborative development process. + +This package consists of files used by other DejaVu packages. +""" + +usbredir_rpm_info = """Name : usbredir +Version : 0.7.1 +Release : 1.el7 +Architecture: x86_64 +Install Date: Wed Feb 7 13:49:24 2018 +Group : System Environment/Libraries +Size : 108319 +License : LGPLv2+ +Signature : RSA/SHA256, Sun Nov 20 20:56:49 2016, Key ID 24c6a8a7f4a80eb5 +Source RPM : usbredir-0.7.1-1.el7.src.rpm +Build Date : Sat Nov 5 18:33:15 2016 +Build Host : worker1.bsys.centos.org +Relocations : (not relocatable) +Packager : CentOS BuildSystem +Vendor : CentOS +URL : http://spice-space.org/page/UsbRedir +Summary : USB network redirection protocol libraries +Description : +The usbredir libraries allow USB devices to be used on remote and/or virtual +hosts over TCP. The following libraries are provided: + +usbredirparser: +A library containing the parser for the usbredir protocol + +usbredirhost: +A library implementing the USB host side of a usbredir connection. +All that an application wishing to implement a USB host needs to do is: +* Provide a libusb device handle for the device +* Provide write and read callbacks for the actual transport of usbredir data +* Monitor for usbredir and libusb read/write events and call their handlers +""" + +perl_compress_rpm_info = """Name : perl-Compress-Raw-Zlib +Epoch : 1 +Version : 2.061 +Release : 4.el7 +Architecture: x86_64 +Install Date: Sat Jan 26 20:05:50 2019 +Group : Development/Libraries +Size : 139803 +License : GPL+ or Artistic +Signature : RSA/SHA256, Fri Jul 4 04:15:33 2014, Key ID 24c6a8a7f4a80eb5 +Source RPM : perl-Compress-Raw-Zlib-2.061-4.el7.src.rpm +Build Date : Tue Jun 10 01:12:08 2014 +Build Host : worker1.bsys.centos.org +Relocations : (not relocatable) +Packager : CentOS BuildSystem +Vendor : CentOS +URL : http://search.cpan.org/dist/Compress-Raw-Zlib/ +Summary : Low-level interface to the zlib compression library +Description : +The Compress::Raw::Zlib module provides a Perl interface to the zlib +compression library, which is used by IO::Compress::Zlib. +Obsoletes : +""" # noqa, PEP-8 disabled because of example output has trailing spaces diff --git a/tools/test_data_yum.py b/tools/test_data_yum.py new file mode 100755 index 0000000..6655be4 --- /dev/null +++ b/tools/test_data_yum.py @@ -0,0 +1,176 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=invalid-name + + +basesystem_yum_info = """Loaded plugins: fastestmirror, priorities +Loading mirror speeds from cached hostfile +Installed Packages +Name : basesystem +Arch : noarch +Version : 10.0 +Release : 7.el7.centos +Size : 0.0 +Repo : installed +Summary : The skeleton package which defines a simple CentOS Linux system +License : Public Domain +Description : Basesystem defines the components of a basic CentOS Linux + : system (for example, the package installation order to use during + : bootstrapping). Basesystem should be in every installation of a + : system, and it should never be removed. + +""" # noqa, PEP-8 disabled because of example output has trailing spaces + +bash_yum_info = """Name : bash +Arch : x86_64 +Version : 4.2.46 +Release : 21.el7_3 +Size : 3.5 M +Repo : installed +From repo : updates +Summary : The GNU Bourne Again shell +URL : http://www.gnu.org/software/bash +License : GPLv3+ +Description : The GNU Bourne Again shell (Bash) is a shell or command language + : interpreter that is compatible with the Bourne shell (sh). Bash + : incorporates useful features from the Korn shell (ksh) and the C + : shell (csh). Most sh scripts can be run by bash without + : modification. +""" + +centos_logos_yum_info = """ +Installed Packages +Name : centos-logos +Arch : noarch +Version : 70.0.6 +Release : 3.el7.centos +License : Copyright ? 2014 The CentOS Project. All rights reserved. + +""" + +conntrack_tools_yum_info = """Name : conntrack-tools +Arch : x86_64 +Version : 1.4.4 +Release : 3.el7_3 +Size : 550 k +Repo : installed +From repo : centos-updates +Summary : Manipulate netfilter connection tracking table and run High + : Availability +URL : http://netfilter.org +License : GPLv2 +Description : With conntrack-tools you can setup a High Availability cluster and + : synchronize conntrack state between multiple firewalls. + : + : The conntrack-tools package contains two programs: + : - conntrack: the command line interface to interact with the + : connection tracking system. + : - conntrackd: the connection tracking userspace daemon that can be + : used to deploy highly available GNU/Linux firewalls and collect + : statistics of the firewall use. + : + : conntrack is used to search, list, inspect and maintain the + : netfilter connection tracking subsystem of the Linux kernel. + : Using conntrack, you can dump a list of all (or a filtered + : selection of) currently tracked connections, delete connections + : from the state table, and even add new ones. + : In addition, you can also monitor connection tracking events, e.g. + : show an event message (one line) per newly established connection. +""" # noqa, PEP-8 disabled because of example output has trailing spaces + +cpp_yum_info = """ +Installed Packages +Name : cpp +Arch : x86_64 +Version : 4.8.5 +Release : 11.el7 +Size : 15 M +Repo : installed +From repo : purkki-centos-base +Summary : The C Preprocessor +URL : http://gcc.gnu.org +License : GPLv3+ and GPLv3+ with exceptions and GPLv2+ with exceptions and + : LGPLv2+ and BSD +Description : Cpp is the GNU C-Compatible Compiler Preprocessor. + : Cpp is a macro processor which is used automatically + : by the C compiler to transform your program before actual + : compilation. It is called a macro processor because it allows + : you to define macros, abbreviations for longer + : constructs. + : + : The C preprocessor provides four separate functionalities: the + : inclusion of header files (files of declarations that can be + : substituted into your program); macro expansion (you can define + : macros, and the C preprocessor will replace the macros with their + : definitions throughout the program); conditional compilation + : (using special preprocessing directives, you can include or + : exclude parts of the program according to various conditions); and + : line control (if you use a program to combine or rearrange source + : files into an intermediate file which is then compiled, you can + : use line control to inform the compiler about where each source + : line originated). + : + : You should install this package if you are a C programmer and you + : use macros. + +""" # noqa, PEP-8 disabled because of example output has trailing spaces + +dejavu_fonts_common_yum_info = """ +Installed Packages +Name : dejavu-fonts-common +Arch : noarch +Version : 2.33 +Release : 6.el7 +Size : 127 k +Repo : installed +From repo : purkki-centos-base +Summary : Common files for the Dejavu font set +URL : http://dejavu-fonts.org/ +License : Bitstream Vera and Public Domain +Description : + : The DejaVu font set is based on the ?Bitstream Vera? fonts, + : release 1.10. Its purpose is to provide a wider range of + : characters, while maintaining the original style, using an open + : collaborative development process. + : + : This package consists of files used by other DejaVu packages. + +""" # noqa, PEP-8 disabled because of example output has trailing spaces + +pacemaker_yum_info = """ +Name : pacemaker +Arch : x86_64 +Version : 1.1.15 +Release : 11.el7_3.5 +Size : 1.1 M +Repo : installed +From repo : purkki-centos-updates +Summary : Scalable High-Availability cluster resource manager +URL : http://www.clusterlabs.org +License : GPLv2+ and LGPLv2+ +Description : Pacemaker is an advanced, scalable High-Availability cluster + : resource manager for Corosync, CMAN and/or Linux-HA. + : + : It supports more than 16 node clusters with significant + : capabilities for managing resources and dependencies. + : + : It will run scripts at initialization, when machines go up or + : down, when related resources fail and can be configured to + : periodically check resource health. + : + : Available rpmbuild rebuild options: + : --with(out) : cman stonithd doc coverage profiling pre_release + : hardening +""" # noqa, PEP-8 disabled because of example output has trailing spaces diff --git a/tools/utils.py b/tools/utils.py new file mode 100755 index 0000000..caea09f --- /dev/null +++ b/tools/utils.py @@ -0,0 +1,31 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +def apply_jenkins_auth(url): + validate_environ(['JENKINS_USERNAME', 'JENKINS_TOKEN']) + protocol, address = url.split('://') + url = '{protocol}://{user}:{token}@{address}'.format(**dict(protocol=protocol, + user=os.environ['JENKINS_USERNAME'], + token=os.environ['JENKINS_TOKEN'], + address=address)) + return url + + +def validate_environ(expected_vars): + for env in expected_vars: + if env not in os.environ: + raise Exception('{} must be defined in environment'.format(env)) diff --git a/tools/yum.py b/tools/yum.py new file mode 100755 index 0000000..573da4c --- /dev/null +++ b/tools/yum.py @@ -0,0 +1,215 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import logging + +from tools.executor import run +from tools.rpm import RpmData + + +class YumRpm(RpmData): + @property + def arch(self): + return self['Arch'] + + +class YumInfoParserException(BaseException): + pass + + +class YumDownloaderException(BaseException): + def __init__(self, msg, failed_packages): + super(YumDownloaderException, self).__init__() + self.msg = msg + self.failed_packages = failed_packages + + def __str__(self): + return self.msg + + +class YumInfoParser(object): + """ + Parse 'yum info' output + """ + + def parse_file(self, yum_info_installed_file_path): + with open(yum_info_installed_file_path, 'r') as f: + return self.parse_installed(f.read()) + + def parse_installed(self, yum_info_installed): + return self._parse_rpms_with_regexp(yum_info_installed, r'\nInstalled Packages\n') + + def parse_available(self, yum_info_installed): + return self._parse_rpms_with_regexp(yum_info_installed, r'Available Packages\n') + + def _parse_rpms_with_regexp(self, yum_output, regexp): + parsed_output = self._split_yum_output_with(yum_output, regexp) + return [self.parse_package(pkg) for pkg in parsed_output[1].split('\n\n') if pkg] + + @staticmethod + def _split_yum_output_with(output, regexp): + parsed_output = re.split(regexp, output) + if len(parsed_output) != 2: + raise YumInfoParserException( + '{} not found from output: {}'.format(repr(regexp), output[:1000])) + return parsed_output + + @staticmethod + def parse_package(yum_info_output): + result = YumRpm() + current_key = None + for line in re.findall(r'^(.+?) : (.*)$', yum_info_output, re.MULTILINE): + parsed_key = line[0].strip() + parsed_value = line[1].rstrip(' ') + if parsed_key: + result[parsed_key] = parsed_value + current_key = parsed_key + elif current_key in ['License', 'Summary']: + result[current_key] = result[current_key] + ' ' + parsed_value + else: + result[current_key] = result[current_key] + '\n' + parsed_value + return result + + +class Yum(object): + def __init__(self): + self.config = YumConfig(filename='tmp_yum.conf') + + @classmethod + def clean_and_remove_cache(cls): + cls.clean_all() + cls.remove_cache_dir() + + @classmethod + def clean_all(cls): + run(['yum', 'clean', 'all'], raise_on_stderr=False) + + @classmethod + def remove_cache_dir(cls): + run(['rm', '-rf', '/var/cache/yum'], raise_on_stderr=True) + + @classmethod + def read_available_pkgs(cls, name, url): + filename = 'tmp_yum.conf' + yum_tmp_conf = YumConfig() + yum_tmp_conf.add_repository(name, url) + with open(filename, 'w') as f: + f.write(yum_tmp_conf.render()) + cmd = ['yum', + '--config={}'.format(filename), + '--showduplicates', + '--setopt=keepcache=0', + '--disablerepo=*', + '--enablerepo={}'.format(name), + 'info', + 'available'] + return run(cmd, raise_on_stderr=False).stdout + + def add_repo(self, name, url): + self.config.add_repository(name, url) + + def read_all_packages(self): + self.config.write() + logging.debug('Yum config:\n{}'.format(self.config)) + cmd = ['yum', + '--config={}'.format(self.config.filename), + '--showduplicates', + '--setopt=keepcache=0', + '--enablerepo=*', + 'info', + 'available'] + return run(cmd, raise_on_stderr=False).stdout + + +class YumDownloader(object): + def download(self, rpms, repositories, to_dir=None, source=False): + logging.debug('Downloading {} RPMs from repositories: {}'.format(len(rpms), + [r['name'] for r in + repositories])) + result = self._download(rpms, repositories, to_dir=to_dir, source=source) + downloaded_rpms = [rpm.rstrip('.rpm') for rpm in os.listdir(to_dir)] + not_downloaded_rpms = [rpm for rpm in rpms if rpm not in downloaded_rpms] + if len(rpms) != len(downloaded_rpms): + logging.debug('Downloaded {}/{} RPMs: {}'.format(len(downloaded_rpms), len(rpms), + downloaded_rpms)) + # Not precise way to list not downloaded RPMs - RPM name may not match the metadata + # We should read "rpm -qip" for each rpm and parse it + raise YumDownloaderException( + 'Failed to download {}/{} RPMs: {}.\nYumdownloader result: {}'.format( + len(not_downloaded_rpms), len(rpms), not_downloaded_rpms, str(result)), + not_downloaded_rpms + ) + + @staticmethod + def _download(rpms, repositories, to_dir=None, source=False): + filename = 'tmp_yum.conf' + yum_tmp_conf = YumConfig() + for r in repositories: + yum_tmp_conf.add_repository(r['name'], r['baseurl']) + with open(filename, 'w') as f: + f.write(yum_tmp_conf.render()) + logging.debug('Downloading RPMs: {}'.format(rpms)) + cmd = ['yumdownloader'] + if to_dir is not None: + cmd += ['--destdir={}'.format(to_dir)] + cmd += ['--config={}'.format(filename), + '--disablerepo=*', + '--enablerepo={}'.format(','.join([r['name'] for r in repositories])), + '--showduplicates'] + if source: + cmd.append('--source') + cmd += rpms + return run(cmd, raise_on_stderr=False) + + +class YumConfig(object): + def __init__(self, filename=None): + self.filename = filename + self.config = ['[main]', + '#cachedir=/var/cache/yum/$basearch/$releasever', + 'reposdir=/foo/bar/xyz', + 'keepcache=0', + 'debuglevel=2', + '#logfile=/var/log/yum.log', + 'exactarch=1', + 'obsoletes=1', + 'gpgcheck=0', + 'plugins=1', + 'installonly_limit=5', + 'override_install_langs=en_US.UTF-8', + 'tsflags=nodocs'] + self.repositories = [] + + def add_repository(self, name, url, exclude=None): + repo = ['[{}]'.format(name), + 'name = ' + name, + 'baseurl = ' + url, + 'enabled = 1', + 'gpgcheck = 0'] + if exclude is not None: + repo.append('exclude = ' + exclude) + self.repositories.append(repo) + + def __str__(self): + return self.render() + + def render(self): + blocks = ['\n'.join(self.config)] + ['\n'.join(repo) for repo in self.repositories] + return '\n\n'.join(blocks) + + def write(self): + with open(self.filename, 'w') as f: + f.write(self.render()) diff --git a/tools/yum_info_installed.sample b/tools/yum_info_installed.sample new file mode 100644 index 0000000..fb6f64f --- /dev/null +++ b/tools/yum_info_installed.sample @@ -0,0 +1,223 @@ +Loaded plugins: fastestmirror, priorities +Installed Packages +Name : GeoIP +Arch : x86_64 +Version : 1.5.0 +Release : 11.el7 +Size : 2.8 M +Repo : installed +From repo : purkki-centos-base +Summary : Library for country/city/organization to IP address or hostname + : mapping +URL : http://www.maxmind.com/app/c +License : LGPLv2+ and GPLv2+ and CC-BY-SA +Description : GeoIP is a C library that enables the user to find the country + : that any IP address or hostname originates from. It uses a file + : based database that is accurate as of June 2007 and can optionally + : be updated on a weekly basis by installing the GeoIP-update + : package. This database simply contains IP blocks as keys, and + : countries as values. This database should be more complete and + : accurate than using reverse DNS lookups. + : + : This package includes GeoLite data created by MaxMind, available + : from http://www.maxmind.com/ + +Name : MySQL-python +Arch : x86_64 +Version : 1.2.5 +Release : 1.el7 +Size : 284 k +Repo : installed +From repo : purkki-centos-base +Summary : An interface to MySQL +URL : https://github.com/farcepest/MySQLdb1 +License : GPLv2+ +Description : Python interface to MySQL + : + : MySQLdb is an interface to the popular MySQL database server for + : Python. The design goals are: + : + : - Compliance with Python database API version 2.0 + : - Thread-safety + : - Thread-friendliness (threads will not block each other) + : - Compatibility with MySQL 3.23 and up + : + : This module should be mostly compatible with an older interface + : written by Joe Skinner and others. However, the older version is + : a) not thread-friendly, b) written for MySQL 3.21, c) apparently + : not actively maintained. No code from that version is used in + : MySQLdb. + +Name : OpenIPMI-libs +Arch : x86_64 +Version : 2.0.19 +Release : 15.el7 +Size : 1.7 M +Repo : installed +From repo : purkki-centos-base +Summary : The OpenIPMI runtime libraries +URL : http://sourceforge.net/projects/openipmi/ +License : LGPLv2+ and GPLv2+ or BSD +Description : The OpenIPMI-libs package contains the runtime libraries for + : shared binaries and applications. + +Name : OpenIPMI-modalias +Arch : x86_64 +Version : 2.0.19 +Release : 15.el7 +Size : 210 +Repo : installed +From repo : purkki-centos-base +Summary : Module aliases for IPMI subsystem +URL : http://sourceforge.net/projects/openipmi/ +License : LGPLv2+ and GPLv2+ or BSD +Description : The OpenIPMI-modalias provides configuration file with module + : aliases of ACPI and PNP wildcards. + +Name : PyPAM +Arch : x86_64 +Version : 0.5.0 +Release : 19.el7 +Size : 51 k +Repo : installed +From repo : purkki-centos-base +Summary : PAM bindings for Python +URL : http://www.pangalactic.org/PyPAM +License : LGPLv2 +Description : PAM (Pluggable Authentication Module) bindings for Python. + +Name : PyYAML +Arch : x86_64 +Version : 3.10 +Release : 11.el7 +Size : 630 k +Repo : installed +Summary : YAML parser and emitter for Python +URL : http://pyyaml.org/ +License : MIT +Description : YAML is a data serialization format designed for human readability + : and interaction with scripting languages. PyYAML is a YAML parser + : and emitter for Python. + : + : PyYAML features a complete YAML 1.1 parser, Unicode support, + : pickle support, capable extension API, and sensible error + : messages. PyYAML supports standard YAML tags and provides + : Python-specific tags that allow to represent an arbitrary Python + : object. + : + : PyYAML is applicable for a broad range of tasks from complex + : configuration files to object serialization and persistance. + +Name : SDL +Arch : x86_64 +Version : 1.2.15 +Release : 14.el7 +Size : 467 k +Repo : installed +From repo : purkki-centos-base +Summary : A cross-platform multimedia library +URL : http://www.libsdl.org/ +License : LGPLv2+ +Description : Simple DirectMedia Layer (SDL) is a cross-platform multimedia + : library designed to provide fast access to the graphics frame + : buffer and audio device. + +Name : XStatic-Angular-common +Arch : noarch +Epoch : 1 +Version : 1.5.8.0 +Release : 1.el7 +Size : 3.1 M +Repo : installed +From repo : purkki-delorean +Summary : Common files for XStatic-Angular (XStatic packaging standard) +URL : http://angularjs.org +License : MIT +Description : Common files for XStatic-Angular (XStatic packaging standard) + +Name : acl +Arch : x86_64 +Version : 2.2.51 +Release : 12.el7 +Size : 196 k +Repo : installed +Summary : Access control list utilities +URL : http://acl.bestbits.at/ +License : GPLv2+ +Description : This package contains the getfacl and setfacl utilities needed for + : manipulating access control lists. + +Name : adwaita-cursor-theme +Arch : noarch +Version : 3.14.1 +Release : 1.el7 +Size : 1.9 M +Repo : installed +From repo : purkki-centos-base +Summary : Adwaita cursor theme +URL : http://www.gnome.org +License : LGPLv3+ or CC-BY-SA +Description : The adwaita-cursor-theme package contains a modern set of cursors + : originally designed for the GNOME desktop. + +Name : adwaita-icon-theme +Arch : noarch +Version : 3.14.1 +Release : 1.el7 +Size : 13 M +Repo : installed +From repo : purkki-centos-base +Summary : Adwaita icon theme +URL : http://www.gnome.org +License : LGPLv3+ or CC-BY-SA +Description : This package contains the Adwaita icon theme used by the GNOME + : desktop. + +Name : agg +Arch : x86_64 +Version : 2.5 +Release : 18.el7 +Size : 410 k +Repo : installed +From repo : purkki-centos-base +Summary : Anti-Grain Geometry graphical rendering engine +URL : http://www.antigrain.com +License : GPLv2+ +Description : A High Quality Rendering Engine for C++. + +Name : alsa-lib +Arch : x86_64 +Version : 1.1.1 +Release : 1.el7 +Size : 1.2 M +Repo : installed +From repo : purkki-centos-base +Summary : The Advanced Linux Sound Architecture (ALSA) library +URL : http://www.alsa-project.org/ +License : LGPLv2+ +Description : The Advanced Linux Sound Architecture (ALSA) provides audio and + : MIDI functionality to the Linux operating system. + : + : This package includes the ALSA runtime libraries to simplify + : application programming and provide higher level functionality as + : well as support for the older OSS API, providing binary + : compatibility for most OSS programs. + +Name : ansible +Arch : noarch +Version : 2.3.0.0 +Release : 3.el7 +Size : 27 M +Repo : installed +From repo : purkki-delorean +Summary : SSH-based configuration management, deployment, and task execution + : system +URL : http://ansible.com +License : GPLv3+ +Description : + : Ansible is a radically simple model-driven configuration + : management, multi-node deployment, and remote task execution + : system. Ansible works over SSH and does not require any software + : or daemons to be installed on remote nodes. Extension modules can + : be written in any language and are transferred to managed machines + : automatically. diff --git a/tools/yum_test.py b/tools/yum_test.py new file mode 100755 index 0000000..1a3dca6 --- /dev/null +++ b/tools/yum_test.py @@ -0,0 +1,67 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=wildcard-import +import os +import pytest + +from tools.test_data_yum import bash_yum_info, \ + conntrack_tools_yum_info, \ + pacemaker_yum_info +from tools.yum import YumInfoParser +from tools.yum_test_data import bash_expected, pacemaker_expected, \ + yum_info_installed_header, \ + yum_info_available_header, \ + yum_info_available_header2 +from tools.yum_test_data import conntrack_tools_expected + + +@pytest.mark.parametrize('yum_info, expected_output', [ + (bash_yum_info, bash_expected), + (conntrack_tools_yum_info, conntrack_tools_expected), + (pacemaker_yum_info, pacemaker_expected) +]) +def test_parse_package(yum_info, expected_output): + parsed = YumInfoParser().parse_package(yum_info) + expected = expected_output + assert parsed == expected + + +def test_parse_installed(): + fake_out = '\n'.join([yum_info_installed_header, + bash_yum_info, + conntrack_tools_yum_info]) + parsed = YumInfoParser().parse_installed(fake_out) + expected = [bash_expected, conntrack_tools_expected] + assert parsed == expected + + +@pytest.mark.parametrize('available_header', [ + yum_info_available_header, + yum_info_available_header2 +]) +def test_parse_available(available_header): + fake_out = '\n'.join([available_header, + bash_yum_info, + conntrack_tools_yum_info]) + parsed = YumInfoParser().parse_available(fake_out) + expected = [bash_expected, conntrack_tools_expected] + assert parsed == expected + + +def test_parse_file(): + test_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'yum_info_installed.sample') + parsed = YumInfoParser().parse_file(test_file) + assert len(parsed) == 14 diff --git a/tools/yum_test_data.py b/tools/yum_test_data.py new file mode 100755 index 0000000..d5d6a8b --- /dev/null +++ b/tools/yum_test_data.py @@ -0,0 +1,106 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=invalid-name,line-too-long + +yum_info_installed_header = """Loaded plugins: fastestmirror, priorities +Loading mirror speeds from cached hostfile +Installed Packages +""" + +yum_info_available_header = """Added tmprepo repo from http://purkki/mirror/centos/snapshot/20170705-2/7/os/x86_64/ +Available Packages +""" # noqa + +yum_info_available_header2 = """Available Packages +""" + +bash_expected = { + 'Name': 'bash', + 'Arch': 'x86_64', + 'Version': '4.2.46', + 'Release': '21.el7_3', + 'Size': '3.5 M', + 'Repo': 'installed', + 'From repo': 'updates', + 'Summary': 'The GNU Bourne Again shell', + 'URL': 'http://www.gnu.org/software/bash', + 'License': 'GPLv3+', + 'Description': '\n'.join( + ['The GNU Bourne Again shell (Bash) is a shell or command language', + 'interpreter that is compatible with the Bourne shell (sh). Bash', + 'incorporates useful features from the Korn shell (ksh) and the C', + 'shell (csh). Most sh scripts can be run by bash without', + 'modification.']) +} + +conntrack_tools_expected = { + 'Name': 'conntrack-tools', + 'Arch': 'x86_64', + 'Version': '1.4.4', + 'Release': '3.el7_3', + 'Size': '550 k', + 'Repo': 'installed', + 'From repo': 'centos-updates', + 'Summary': ' '.join( + ['Manipulate netfilter connection tracking table and run High', + 'Availability']), + 'URL': 'http://netfilter.org', + 'License': 'GPLv2', + 'Description': '\n'.join( + ['With conntrack-tools you can setup a High Availability cluster and', + 'synchronize conntrack state between multiple firewalls.', + '', + 'The conntrack-tools package contains two programs:', + '- conntrack: the command line interface to interact with the', + ' connection tracking system.', + '- conntrackd: the connection tracking userspace daemon that can be', + ' used to deploy highly available GNU/Linux firewalls and collect', + ' statistics of the firewall use.', + '', + 'conntrack is used to search, list, inspect and maintain the', + 'netfilter connection tracking subsystem of the Linux kernel.', + 'Using conntrack, you can dump a list of all (or a filtered', + 'selection of) currently tracked connections, delete connections', + 'from the state table, and even add new ones.', + 'In addition, you can also monitor connection tracking events, e.g.', + 'show an event message (one line) per newly established connection.']) +} + +pacemaker_expected = { + 'Name': 'pacemaker', + 'Arch': 'x86_64', + 'Version': '1.1.15', + 'Release': '11.el7_3.5', + 'Size': '1.1 M', + 'Repo': 'installed', + 'From repo': 'purkki-centos-updates', + 'Summary': 'Scalable High-Availability cluster resource manager', + 'URL': 'http://www.clusterlabs.org', + 'License': 'GPLv2+ and LGPLv2+', + 'Description': '\n'.join( + ['Pacemaker is an advanced, scalable High-Availability cluster', + 'resource manager for Corosync, CMAN and/or Linux-HA.', + '', + 'It supports more than 16 node clusters with significant', + 'capabilities for managing resources and dependencies.', + '', + 'It will run scripts at initialization, when machines go up or', + 'down, when related resources fail and can be configured to', + 'periodically check resource health.', + '', + 'Available rpmbuild rebuild options:', + ' --with(out) : cman stonithd doc coverage profiling pre_release', + 'hardening']) +} diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..80cf235 --- /dev/null +++ b/tox.ini @@ -0,0 +1,35 @@ +[tox] +envlist = py27, pylint +skipsdist=true + +[testenv] +commands = py.test -v \ + --basetemp={toxinidir}/.pytest-tmpdir \ + --junitxml=junit.xml \ + --pep8 \ + --cov-config {toxinidir}/.coveragerc \ + --cov-branch \ + --cov-report term-missing \ + --cov-report html:htmlcov \ + --cov=. \ + {posargs:.} + +setenv = + PYTHONPATH = {toxinidir}/tests/mocked_dependencies:{toxinidir}/src +deps=-rrequirements.txt + +[pytest] +cache_dir = .pytest-cache +pep8maxlinelength = 100 + +[testenv:pylint] +basepython = python2.7 +commands = pylint --rcfile={toxinidir}/pylintrc {posargs:tools/} + +deps=pylint < 2.0 + -rrequirements.txt + +[testenv:clean] +deps= +whitelist_externals = rm +commands = rm -rf .coverage .pytest-cache .pytest-tmpdir junit.xml htmlcov -- 2.16.6