From: Saku Chydenius Date: Wed, 13 Mar 2019 14:28:54 +0000 (+0200) Subject: Initial commit X-Git-Url: https://gerrit.akraino.org/r/gitweb?p=ta%2Frpmbuilder.git;a=commitdiff_plain;h=876631a959303430aafc0be7897b086ee9b921fe Initial commit Change-Id: I1348bac40021fb73ecc586c03ab813b80211220d Signed-off-by: Saku Chydenius --- diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..f3a5cd4 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + .tox/* diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9fa5b33 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.idea +*.pyc +.tox/ +.pytest-cache/ +.coverage-* +htmlcov diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..66727d8 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,378 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Use multiple processes to speed up Pylint. +jobs=1 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Allow optimization of some AST trees. This will activate a peephole AST +# optimizer, which will apply various small optimizations. For instance, it can +# be used to obtain the result of joining multiple strings with the addition +# operator. Joining a lot of strings can lead to a maximum recursion error in +# Pylint and this flag can prevent that. It has one side effect, the resulting +# AST will be different than the one from reality. +optimize-ast=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time. See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,max-line-length,missing-docstring,logging-too-many-args,import-error, line-too-long,no-name-in-module + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[BASIC] + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter,input + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + + +[ELIF] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=100 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=_$|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). This supports can work +# with qualified names. +ignored-classes= + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..c7e0c63 --- /dev/null +++ b/README.md @@ -0,0 +1,188 @@ +# RPMBuilder + +This tool allows you to build RPM files in mock environment. + +## What is Mock? + +Mock is a tool for building packages. It can build packages for different architectures and +different Fedora or RHEL versions than the build host has. Mock creates chroots and builds packages +in them. Its only task is to reliably populate a chroot and attempt to build a package in that +chroot. + +Source: https://fedoraproject.org/wiki/Mock?rd=Subprojects/Mock + + +## How does rpmbuilder work? + +Tool reads user provided configuration and creates checkouts of Mock settings and Git hosted +projects. Dependency mapping between projects is created from spec files found in every project. +When a project is build the spec file is patched so that "Version:" field is set to "git describe" +value. The "Release:" field is incremented based on existing rpm files. + +The idea for this tool has been taken from openSuse Build Service (OBS). Own tool was created since +configuring OBS takes a lot of work and it does not have great support for password protected Git +repositories. + + +## Prerequisite + +### Installing mock and createrepo + +As a requirement for the tool to work, you need to install rpm building tool "mock" and repository +tool "createrepo" to your host. + +In Redhat/Fedora: +``` +$ yum install mock createrepo rpmdevtools +``` + +Ubuntu: +``` +$ apt-get install mock createrepo +``` + +### Assign users to mock group + +Users of mock also need to belong to group "mock". This allows them to run mock which uses chroot. + +Create mock group if it does not exists in your host +``` +$ getent group mock || groupadd mock +``` + +Add yourself to mock group +``` +$ usermod -a -G mock +``` + + +## Running script + +Create a workspace directory which builder can use to do checkouts and compilation. This directory +should have sufficient space to store your checkouts and rpm files. + +Example: +``` +$ mkdir /home//rpmworkspace +``` + +### Building project to a rpm file + +You can build local projects as rpm files. This is useful if you are developing a project and want +to create rpm files without commiting to version control system. + +Example of building helloworld to rpm: +``` +$ ./makebuild.py -w /home//rpmworkspace /home//helloworld +``` + +If you want to reconfigure Mock environment (e.g. extra Yum repositories) to be available during +building, create a copy of default Mock configuration and provide it with -m option. + +Example: +``` +$ cp defaults/epel-7-x86_64.cfg ~/mymock.cfg +$ vim ~/mymock.cfg +$ ./makebuild.py -w /home//rpmworkspace -m ~/mymock.cfg /home//helloworld +``` + +Note: + - RPM package version is created from "git describe". If git describe cannot be used, package + version is hard coded to a.b + +### Access built RPM packages + +If there are no errors during building, rpm files can be copied from build repository under your +workspace directory. +Example: /home//rpmworkspace/buildrepository/epel-7-x86_64/ + + +## RPM file name convention + +Rpm packages are named and versioned so that it can be found from version control system. Rpmbuilder +uses command "git describe --dirty --tags" to produce a rpm package file name. If git is unable to +describe the checkout a another "git describe --dirty --all --always" command is used. + +**Example 1:** Clone with no tags +File name mymodule-master.c110.gce32b26-1.el7.x86_64.rpm states that package mymodule has been made +from master branch. Package was made from 110th commit and this commit has git hash ce32b26. + +**Example 2:** Clone with tag 1.0 +File name mymodule-1.0-1.el7.x86_64.rpm shows that mymodule package was made from tag 1.0. + +**Example 3:** Clone with two changes on top of tag 1.0 git clone +File mymodule-1.0.c2.gad96bc2-1.el7.x86_64.rpm shows two changes have been made on top of 1.0 and +also the identifying hash. + +**Example 4:** Clone from Example 3 and local changes +File mymodule-master.dirty.c112.g1.0.2.g8193b3a-1.el7.x86_64.rpm shows that the clone directory +contains local modifications which make it dirty. + + +## More usage examples + +### Storing build products to remote server + +If projects are built in Jenkins, there is always danger that somebody might wipe the workspace. To +protect against workspace wiping you can use stashmakebuild.py script. In your build configuration +file define remote server and directory. + +When building starts script checks from your workspace that you have directories for projects and +builder. If these are missing, your remote server/directory is used to pull previous build state. +After each successful build your project configuration and rpm files are stored to remote server. + +With safemakebuild.py you need to use two additional properties: --remotehost and --remotedir. + +### Create package with version taken from Git + +To read package verion from Git ("git describe") set the package Version directive in spec file as +"%{_version}". + +### Build multiple Git projects with a build configuration file + +Create a configuration file which contains information about the projects and mock environment. +Syntax for the configuration file can be copied from provided configuration-example.ini file. + +Example: +``` +$ cp configuration-example.ini /home//buildconfig.ini +$ vim /home//buildconfig.ini +$ ./makebuild.py -w /home//rpmworkspace -b /home//buildconfig.ini +``` + +Note: + - Builder keeps track of Git commit hash. Rpm building if done if project hash has changed. + - If commit hash has not changed since previous build, building is skipped. + + +## Known issues + +1. If you are not using RedHat, CentOS or Fedora building hosts you might have problems running + mock. Build requirements that have been installed by rpmbuilder to mock chroot, might not be +recognized by rpm tools during building. Because of this rpm building will fail complaining that +necessary build requirements are missing. + +If your components do not have build requirements to each other, then there are no problems. This +problem has been seen with Debian based distributions and it is a known bug: +https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=794495 + +Also spectool seems to be missing from Debian based packages. Without this tool rpmbuilder is +cripled to work only with local source files. As long as you do not have any spec lines such as +"SourceX http://example.com/package.tgz" you are safe. +See **Debian custom spectool intallation** chapter how to install spectool to your home directory. + + +2. If you change git repository url this will force rebuilding of this project even if the hash of + the code remains unchanged. + +### Debian custom spectool intallation + +Clone spectool git somewhere in your home directory. For example to ~/gitrepos -directory: +``` +cd gitrepos +git clone https://pagure.io/spectool.git +``` +Create a symbolic link to spectool where your search PATH can find it. +``` +ln -s ~/gitrepos/spectool/spectool ~/bin/ +``` diff --git a/configuration-example.ini b/configuration-example.ini new file mode 100644 index 0000000..e37e60d --- /dev/null +++ b/configuration-example.ini @@ -0,0 +1,43 @@ +## List all projects separately. Mandatory fields for every project +## are type, url and ref. If your project contains more than one spec +## file you need to define also which spec file should be used. + +## Example of rpm package/software called Project-name + +## Define project name. This is used to name git clones +## and does not affect the rpm content +#[helloworld] + +## Define git url. Make sure you have all read access to the repository. +## You can use url formats which are compatible with normal git command. +#url = git@gitlab:project/helloworld.git + +## Define type as project. +#type = project + +## ref should be a git tag, branch or hash from where you would like to +## get your source and spec files +#ref = master + +## IF you have more than one spec file, specify which spec file should +## be used. This is optional +#spec = helloworld.spec + +## IF you want to enable/disable entry from building, define enabled +## variable as 0/false/off or 1/true/on. This is optional +#enabled = 1 + +## Mock settings define the compilation environment. Easiest way to get a working mock settings +## is to fork/branch the below repository if the default settings do not work for you. +## Multiple roots can be separated with comma. + +[mock] +url = git@gitlab:project/mocksettings.git +ref = master +roots = epel-7-x86_64 + +## sshbackup defines a remote host where build is stored when done. +## This backup also restores workspace if disk is cleaned +[sshbackup] +remotehost = localhost.example.com +remotedir = /var/www/html/mybuild diff --git a/defaults/epel-7-x86_64.cfg b/defaults/epel-7-x86_64.cfg new file mode 100644 index 0000000..b275bdd --- /dev/null +++ b/defaults/epel-7-x86_64.cfg @@ -0,0 +1,60 @@ +# Root name to be used for chroot and caching, must differ between products +config_opts['root'] = 'epel-7-x86_64' + +config_opts['target_arch'] = 'x86_64' +config_opts['legal_host_arches'] = ('x86_64',) +config_opts['dist'] = 'el7' # only useful for --resultdir variable subst +config_opts['chroot_setup_cmd'] = 'install createrepo yum-utils bison byacc cscope ctags cvs diffstat doxygen flex gcc gcc-c++ gcc-gfortran gettext git indent intltool libtool patch patchutils rcs redhat-rpm-config rpm-build subversion swig systemtap sudo' +config_opts['plugin_conf']['yum_cache_enable'] = False +config_opts['plugin_conf']['ccache_enable'] = False +config_opts['plugin_conf']['ccache_opts']['max_cache_size'] = '1G' +config_opts['plugin_conf']['ccache_opts']['dir'] = "/dev/shm/ccache.epel-7/" +config_opts['rpmbuild_networking'] = True +config_opts['cleanup_on_success'] = True +config_opts['cleanup_on_failure'] = False +config_opts['exclude_from_homedir_cleanup'] = ('build/SOURCES', '.bash_history', '.bashrc', 'build/RPMS', ) + +# Common RPM directive values +config_opts['macros']['%_platform_vendor']="My Product" +config_opts['macros']['%_platform_licence']="ASL 2.0" + +# Product specific macros +config_opts['macros']['%_platform_root_path'] = "/opt/product" + +# Compilation +#config_opts['macros']['%_smp_mflags'] = "-j6" +#config_opts['macros']['%_smp_ncpus_max'] = 0 + +# Yum configuration +config_opts['yum.conf'] = """ +[main] +cachedir=/var/cache/yum +keepcache=1 +debuglevel=2 +reposdir=/dev/null +logfile=/var/log/yum.log +retries=20 +obsoletes=1 +gpgcheck=0 +assumeyes=1 +syslog_ident=mock +syslog_device= + +# RPM repositories for yum +[internal-epel-mirror] +name=internal-epel-mirror +baseurl=http://intranet.mycompany.org/mirrors/EPEL/7/$basearch/ +enabled=0 +metadata_expire=7d +gpgcheck=0 + +[local] +name=local repository +baseurl=file:///usr/localrepo/ +enabled=1 +gpgcheck=0 +protect=1 +priority=10 +skip_if_unavailable=True + +""" diff --git a/defaults/logging.ini b/defaults/logging.ini new file mode 100644 index 0000000..8186ead --- /dev/null +++ b/defaults/logging.ini @@ -0,0 +1,84 @@ +[formatters] +keys: detailed,simple,unadorned,state + +[handlers] +keys: simple_console,detailed_console,unadorned_console,simple_console_warnings_only + +[loggers] +keys: root,build,state,mockbuild + +[formatter_state] +format: %(asctime)s - %(message)s + +[formatter_unadorned] +format: %(message)s + +[formatter_simple] +format: %(levelname)s: %(message)s + +;useful for debugging: +[formatter_detailed] +format: %(levelname)s %(filename)s:%(lineno)d: %(message)s + +[handler_unadorned_console] +class: StreamHandler +args: [] +formatter: unadorned +level: INFO + +[handler_simple_console] +class: StreamHandler +args: [] +formatter: simple +level: INFO + +[handler_simple_console_warnings_only] +class: StreamHandler +args: [] +formatter: simple +level: WARNING + +[handler_detailed_console] +class: StreamHandler +args: [] +formatter: detailed +level: WARNING + +; usually dont want to set a level for loggers +; this way all handlers get all messages, and messages can be filtered +; at the handler level +; +; all these loggers default to a console output handler +; +[logger_root] +level: NOTSET +handlers: simple_console + +; mockbuild logger normally has no output +; catches stuff like mockbuild.trace_decorator and mockbuild.util +; dont normally want to propagate to root logger, either +[logger_mockbuild] +level: NOTSET +handlers: +qualname: mockbuild +propagate: 1 + +[logger_state] +level: NOTSET +; unadorned_console only outputs INFO or above +handlers: unadorned_console +qualname: mockbuild.Root.state +propagate: 0 + +[logger_build] +level: NOTSET +handlers: simple_console_warnings_only +qualname: mockbuild.Root.build +propagate: 0 + +; the following is a list mock logger qualnames used within the code: +; +; qualname: mockbuild.util +; qualname: mockbuild.uid +; qualname: mockbuild.trace_decorator + diff --git a/defaults/site-defaults.cfg b/defaults/site-defaults.cfg new file mode 100644 index 0000000..9fff7af --- /dev/null +++ b/defaults/site-defaults.cfg @@ -0,0 +1,160 @@ +# mock defaults +# vim:tw=0:ts=4:sw=4:et: +# +# This config file is for site-specific default values that apply across all +# configurations. Options specified in this config file can be overridden in +# the individual mock config files. +# +# The site-defaults.cfg delivered by default has NO options set. Only set +# options here if you want to override the defaults. +# +# Entries in this file follow the same format as other mock config files. +# config_opts['foo'] = bar + +############################################################################# +# +# Things that we recommend you set in site-defaults.cfg: +# +# config_opts['basedir'] = '/var/lib/mock/' +# config_opts['cache_topdir'] = '/var/cache/mock' +# Note: the path pointed to by basedir and cache_topdir must be owned +# by group 'mock' and must have mode: g+rws +# config_opts['rpmbuild_timeout'] = 0 +# config_opts['use_host_resolv'] = True + +# You can configure log format to pull from logging.ini formats of these names: +# config_opts['build_log_fmt_name'] = "unadorned" +# config_opts['root_log_fmt_name'] = "detailed" +# config_opts['state_log_fmt_name'] = "state" +# +# mock will normally set up a minimal chroot /dev. +# If you want to use a pre-configured /dev, disable this and use the bind-mount +# plugin to mount your special /dev +# config_opts['internal_dev_setup'] = True +# +# internal_setarch defaults to 'True' if the python 'ctypes' package is +# available. It is in the python std lib on >= python 2.5. On older versions, +# it is available as an addon. On systems w/o ctypes, it will default to 'False' +# config_opts['internal_setarch'] = False +# +# the cleanup_on_* options allow you to automatically clean and remove the +# mock build directory, but only take effect if --resultdir is used. +# config_opts provides fine-grained control. cmdline only has big hammer +# +# config_opts['cleanup_on_success'] = 1 +# config_opts['cleanup_on_failure'] = 1 + +# if you want mock to automatically run createrepo on the rpms in your +# resultdir. +# config_opts['createrepo_on_rpms'] = False +# config_opts['createrepo_command'] = '/usr/bin/createrepo -d -q -x *.src.rpm' + +# if you want mock to backup the contents of a result dir before clean +# config_opts['backup_on_clean'] = False +# config_opts('backup_base_dir'] = config_opts['basedir'] + "backup" + + +############################################################################# +# +# plugin related. Below are the defaults. Change to suit your site +# policy. site-defaults.cfg is a good place to do this. +# +# NOTE: Some of the caching options can theoretically affect build +# reproducability. Change with care. +# +# config_opts['plugin_conf']['package_state_enable'] = True +# config_opts['plugin_conf']['ccache_enable'] = True +# config_opts['plugin_conf']['ccache_opts']['max_cache_size'] = '4G' +# config_opts['plugin_conf']['ccache_opts']['compress'] = None +# config_opts['plugin_conf']['ccache_opts']['dir'] = "%(cache_topdir)s/%(root)s/ccache/" +# config_opts['plugin_conf']['yum_cache_enable'] = True +# config_opts['plugin_conf']['yum_cache_opts']['max_age_days'] = 30 +# config_opts['plugin_conf']['yum_cache_opts']['dir'] = "%(cache_topdir)s/%(root)s/yum_cache/" +# config_opts['plugin_conf']['root_cache_enable'] = True +# config_opts['plugin_conf']['root_cache_opts']['max_age_days'] = 15 +# config_opts['plugin_conf']['root_cache_opts']['dir'] = "%(cache_topdir)s/%(root)s/root_cache/" +# config_opts['plugin_conf']['root_cache_opts']['compress_program'] = "pigz" +# config_opts['plugin_conf']['root_cache_opts']['extension'] = ".gz" +# config_opts['plugin_conf']['root_cache_opts']['exclude_dirs'] = ["./proc", "./sys", "./dev", +# "./tmp/ccache", "./var/cache/yum" ] +# +# bind mount plugin is enabled by default but has no configured directories to +# mount +# config_opts['plugin_conf']['bind_mount_enable'] = True +# config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/host/path', '/bind/mount/path/in/chroot/' )) +# +# config_opts['plugin_conf']['tmpfs_enable'] = False +# config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 1024 +# config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = '512m' +# config_opts['plugin_conf']['tmpfs_opts']['mode'] = '0755' +# config_opts['plugin_conf']['chroot_scan_enable'] = False +# config_opts['plugin_conf']['chroot_scan_opts'] = [ "core(\.\d+)?", "\.log$",] + +############################################################################# +# +# environment for chroot +# +# config_opts['environment']['TERM'] = 'vt100' +# config_opts['environment']['SHELL'] = '/bin/bash' +# config_opts['environment']['HOME'] = '/builddir' +# config_opts['environment']['HOSTNAME'] = 'mock' +# config_opts['environment']['PATH'] = '/usr/bin:/bin:/usr/sbin:/sbin' +# config_opts['environment']['PROMPT_COMMAND'] = 'echo -n ""' +# config_opts['environment']['LANG'] = os.environ.setdefault('LANG', 'en_US.UTF-8') +# config_opts['environment']['TZ'] = os.environ.setdefault('TZ', 'EST5EDT') + +############################################################################# +# +# Things that you can change, but we dont recommend it: +# config_opts['chroothome'] = '/builddir' +# config_opts['clean'] = True + +############################################################################# +# +# Things that must be adjusted if SCM integration is used: +# +# config_opts['scm'] = True +# config_opts['scm_opts']['method'] = 'git' +# config_opts['scm_opts']['cvs_get'] = 'cvs -d /srv/cvs co SCM_BRN SCM_PKG' +# config_opts['scm_opts']['git_get'] = 'git clone SCM_BRN git://localhost/SCM_PKG.git SCM_PKG' +# config_opts['scm_opts']['svn_get'] = 'svn co file:///srv/svn/SCM_PKG/SCM_BRN SCM_PKG' +# config_opts['scm_opts']['spec'] = 'SCM_PKG.spec' +# config_opts['scm_opts']['ext_src_dir'] = '/dev/null' +# config_opts['scm_opts']['write_tar'] = True +# config_opts['scm_opts']['git_timestamps'] = True + +# These options are also recognized but usually defined in cmd line +# with --scm-option package= --scm-option branch= +# config_opts['scm_opts']['package'] = 'mypkg' +# config_opts['scm_opts']['branch'] = 'master' + +############################################################################# +# +# Things that are best suited for individual chroot config files: +# +# MUST SET (in individual chroot cfg file): +# config_opts['root'] = 'name-of-yum-build-dir' +# config_opts['target_arch'] = 'i386' +# config_opts['yum.conf'] = '' +# config_opts['yum_common_opts'] = [] +# +# CAN SET, defaults usually work ok: +# config_opts['chroot_setup_cmd'] = 'install buildsys-build' +# config_opts['log_config_file'] = 'logging.ini' +# config_opts['more_buildreqs']['srpm_name-version-release'] = 'dependencies' +# config_opts['macros']['%Add_your_macro_name_here'] = "add macro value here" +# config_opts['files']['path/name/no/leading/slash'] = "put file contents here." +# config_opts['chrootuid'] = os.getuid() + +# If you change chrootgid, you must also change "mock" to the correct group +# name in this line of the mock PAM config: +# auth sufficient pam_succeed_if.so user ingroup mock use_uid quiet +# config_opts['chrootgid'] = grp.getgrnam("mock")[2] + +# config_opts['useradd'] = '/usr/sbin/useradd -m -u %(uid)s -g %(gid)s -d %(home)s -n %(user)s' # Fedora/RedHat +# +# Security related +# config_opts['no_root_shells'] = False +# +# Proxy settings (https_proxy, ftp_proxy, and no_proxy can also be set) +# config_opts['http_proxy'] = 'http://localhost:3128' diff --git a/makebuild.py b/makebuild.py new file mode 100755 index 0000000..108f4c4 --- /dev/null +++ b/makebuild.py @@ -0,0 +1,396 @@ +#! /usr/bin/python -tt +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module loops through user given configuration and creates +projects based on that information. Projects are then build +""" +import argparse +import logging +import os +import platform +import random +import re +import shutil +import sys + +from rpmbuilder.baseerror import RpmbuilderError +from rpmbuilder.buildhistory import Buildhistory +from rpmbuilder.configfile import Configfilereader +from rpmbuilder.log import configure_logging +from rpmbuilder.mockbuilder import GitMockbuilder, LocalMockbuilder +from rpmbuilder.packagebuilding import Packagebuilding +from rpmbuilder.project import GitProject, LocalMountProject +from rpmbuilder.prettyprinter import Prettyprint +from rpmbuilder.rpmtools import Repotool +from rpmbuilder.utils import find_files + + +class Build(object): + + """ + Build configuration module which creates projects and does building + """ + + def __init__(self, args): + self.logger = logging.getLogger(__name__) + self.workspace = os.path.abspath(args.workspace) + if hasattr(args, 'buildconfig') and args.buildconfig: + self.configuration = Configfilereader(os.path.abspath(args.buildconfig)) + self.builder = None + self.projects = {} + self.args = args + self.packagebuilder = Packagebuilding(args) + + def update_building_blocks(self): + """ Update version control system components and project configuration """ + # Mock building tools + Prettyprint().print_heading("Initialize builders", 80) + default_conf_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'defaults/lcc-epel-7-x86_64.cfg') + if hasattr(self.args, 'mockconf') and self.args.mockconf: + self.logger.debug("Loading Mock builder from local disk") + self.builder = LocalMockbuilder(self.args.mockconf) + elif hasattr(self, 'configuration') and self.configuration: + self.logger.debug("Loading Mock builder from Git") + self.builder = GitMockbuilder(self.workspace, self.configuration) + if self.builder.check_builder_changed(): + self.args.forcerebuild = True + elif os.path.isfile(default_conf_file): + self.logger.debug("Loading default Mock configuration from %s file", default_conf_file) + self.builder = LocalMockbuilder(default_conf_file) + else: + self.logger.critical("No Mock builder configured. Define one in build config file or provide it with -m option.") + raise BuildingError("No Mock builder configured.") + + # Projects outside of project configuration + if hasattr(self.args, 'localproj') and self.args.localproj: + self.update_local_mount_projects() + + # Projects from build configuration file + if hasattr(self, 'configuration') and self.configuration: + self.update_configini_projects() + + if not self.projects: + raise BuildingError("No projects defined. Nothing to build.") + + def update_local_mount_projects(self): + """ Create project objects and initialize project configuration. + Project has been defined as argument """ + + Prettyprint().print_heading("Initialize local projects", 80) + for projectdir in self.args.localproj: + if not os.path.isdir(projectdir): + raise BuildingError("Given \"%s\" is not a directory" % projectdir) + project_specs = list(find_files(os.path.abspath(projectdir), r'.*\.spec$')) + for spec in project_specs: + projectname = os.path.basename(projectdir.rstrip('/')) + if len(list(project_specs)) > 1: + projectname = projectname + '_' + os.path.splitext(os.path.basename(spec))[0] + self.projects[projectname] = LocalMountProject(projectname, + os.path.abspath(projectdir), + self.workspace, + self.projects, + self.builder, + self.packagebuilder, + self.args, + spec_path=spec) + + def update_configini_projects(self): + """ Create project objects and initialize project configuration. + Project has been defined in configuration file """ + Prettyprint().print_heading("Initialize projects", 80) + for section in self.configuration.get_sections(): + if self.configuration.get_string(section, "type") == "project" \ + and self.configuration.get_bool(section, "enabled", defaultvalue=True): + if section in self.projects: + self.logger.warning("Local %s project already configured. Skipping build config entry", section) + else: + self.projects[section] = GitProject(section, + self.workspace, + self.configuration, + self.projects, + self.builder, + self.packagebuilder, + self.args) + + def start_building(self): + """ search for changes and start building """ + Prettyprint().print_heading("Summary of changes", 80) + projects_to_build = self.get_projects_to_build() + self.logger.debug("Final list of projects to build: %s", + str(projects_to_build)) + + Prettyprint().print_heading("Projects to build", 80) + if projects_to_build: + self.logger.info("%-30s %10s %10s", "Name", "Changed", "Rebuild") + for project in projects_to_build: + req_by = "" + if self.projects[project].buildrequires_upstream: + req_by = "(build requires: {})".format( + ', '.join(self.projects[project].buildrequires_upstream)) + self.logger.info("%-30s %10s %10s %s", + self.projects[project].name, + self.projects[project].project_changed, + self.projects[project].project_rebuild_needed, + req_by) + + Prettyprint().print_heading("Building projects", 80) + + if self.mock_projects(projects_to_build): + self.logger.info("All built succesfully..") + Prettyprint().print_heading("Running final steps", 80) + self.finalize(projects_to_build) + + # Clean mock chroot + for mockroot in self.builder.roots: + if self.args.scrub: + self.packagebuilder.scrub_mock_chroot(self.builder.get_configdir(), + mockroot) + return True + else: + self.logger.critical("Problems while building") + raise BuildingError("Error during rpm mock") + else: + self.logger.info("No projects to build.. no changes") + return None + + def get_projects_to_build(self): + """ Find which project are not built yet """ + buildlist = [] + # Find projects that need to be build because of change + for project in self.projects: + if self.projects[project].project_changed \ + or self.projects[project].project_rebuild_needed: + self.logger.info("Project \"%s\": Need to build", project) + buildlist.append(project) + else: + self.logger.info("Project \"%s\": OK. Already built", project) + + # Find projects that have list changed projects in buildrequires + if buildlist: + self.logger.debug("Projects %s need building.", str(buildlist)) + self.logger.debug("Looking for projects that need rebuild") + projects_to_rebuild = [] + for project in buildlist: + self.logger.debug("Project \"%s\" need building.", project) + self.logger.debug("Checking if downstream requires rebuilding") + need_rebuild = \ + self.projects[ + project].mark_downstream_for_rebuild(set(buildlist)) + self.logger.debug("Rebuild needed for: %s", str(need_rebuild)) + projects_to_rebuild.extend(need_rebuild) + buildlist.extend(projects_to_rebuild) + buildlist = list(set(buildlist)) + random.shuffle(buildlist) + return buildlist + + def mock_projects(self, build_list): + """ Loop through all mock chroots to build projects """ + for mockroot in self.builder.roots: + Prettyprint().print_heading("Processing chroot " + mockroot, 70) + if self.args.init: + # Create mock chroot for project building + self.packagebuilder.init_mock_chroot(os.path.join(self.workspace, "mocksettings", "logs"), + self.builder.get_configdir(), + mockroot) + # Restore local yum repository to Mock environment + hostyumrepository = os.path.join(self.workspace, "buildrepository", mockroot, "rpm") + if os.path.isdir(os.path.join(hostyumrepository, "repodata")): + logfile = os.path.join(self.workspace, 'restore-mock-env-yum-repository.log') + self.packagebuilder.restore_local_repository(hostyumrepository, + "/usr/localrepo", + self.builder.get_configdir(), + mockroot, + logfile=logfile) + + # Mock projects + if not self.build_projects(build_list, mockroot): + return False + return True + + def upstream_packages_in_buildlist(self, project, buildlist): + for proj in self.projects[project].buildrequires_upstream: + if proj in buildlist: + return True + return False + + def build_projects(self, build_list, mockroot): + """ Build listed projects """ + self.logger.debug("%s: Projects to build=%s", + mockroot, + str(build_list)) + self.packagebuilder.update_local_repository(self.builder.get_configdir(), mockroot) + something_was_built = True + while something_was_built: + something_was_built = False + not_built = [] + for project in build_list: + self.logger.debug("Trying to build: {}".format(project)) + self.logger.debug("Build list: {}".format(build_list)) + if not self.upstream_packages_in_buildlist(project, build_list): + self.projects[project].resolve_dependencies(mockroot) + self.logger.debug("OK to build {}".format(project)) + self.projects[project].build_project(mockroot) + something_was_built = True + self.packagebuilder.update_local_repository(self.builder.get_configdir(), mockroot) + else: + self.logger.debug("Skipping {} because upstream is not built yet".format(project)) + not_built.append(project) + build_list = not_built + + if build_list: + self.logger.warning("Requirements not available for \"%s\"", + ", ".join(build_list)) + return False + return True + + def finalize(self, projectlist): + """ Do final work such as create yum repositories """ + commonrepo = os.path.join(self.workspace, 'buildrepository') + self.logger.info("Hard linking rpm packages to %s", commonrepo) + for project in projectlist: + self.projects[project].store_build_products(commonrepo) + + for mockroot in self.builder.roots: + Repotool().createrepo(os.path.join(self.workspace, + 'buildrepository', + mockroot, + 'rpm')) + Repotool().createrepo(os.path.join(self.workspace, + 'buildrepository', + mockroot, + 'srpm')) + # Store information of used builder + # Next run then knows what was used in previous build + self.builder.store_builder_status() + + buildhistory = Buildhistory() + historyfile = os.path.join(commonrepo, "buildhistory") + buildhistory.update_history(historyfile, + projectlist, + self.projects) + return True + + def rm_obsolete_projectdirs(self): + """ Clean projects which are not listed in configuration """ + self.logger.debug("Cleaning unused project directories") + projects_directory = os.path.join(self.workspace, 'projects') + if not os.path.isdir(projects_directory): + return True + for subdir in os.listdir(projects_directory): + fulldir = os.path.join(projects_directory, subdir) + if subdir in self.projects: + self.logger.debug("Project directory %s is active", + fulldir) + else: + self.logger.debug("Removing directory %s. No match in projects", + fulldir) + shutil.rmtree(fulldir) + return True + + +class BuildingError(RpmbuilderError): + """ Exceptions originating from builder """ + pass + + +def warn_if_incompatible_distro(): + if platform.linux_distribution()[0].lower() not in ['fedora', 'redhat', 'rhel', 'centos']: + logger = logging.getLogger() + logger.warning("Distribution compatibility check failed.\n" + "If you use other than Fedora, RedHat or CentOS based Linux distribution, you might experience problems\n" + "in case there are BuildRequirements between your own packages. For more information, read README.md") + + +class ArgumentMakebuild(object): + """ Default arguments which are always needed """ + + def __init__(self): + """ init """ + self.parser = argparse.ArgumentParser(description=''' + RPM building tool for continuous integration and development usage. + ''') + self.set_arguments(self.parser) + + def set_arguments(self, parser): + """ Add relevant arguments """ + parser.add_argument("localproj", + metavar="dir", + help="Local project directory outside of buildconfig. This option can be used multiple times.", + nargs="*") + parser.add_argument("-w", + "--workspace", + help="Sandbox directory for builder. Used to store repository clones and built rpm files. Required option.", + required=True) +# parser.add_argument("-b", +# "--buildconfig", +# help="Build configuration file lists projects and mock configuration. Required option.") + parser.add_argument("-m", + "--mockconf", + help="Local Mock configuration file. Overrides mock settings from build configuration.") + parser.add_argument("--mockarguments", + help="Arguments to be passed to mock. Check possible arguments from mock man pages") + parser.add_argument("-v", + "--verbose", + help="Verbosed printing.", + action="store_true") + parser.add_argument("-f", + "--forcerebuild", + help="Force rebuilding of all projects.", + action="store_true") + parser.add_argument("--nowipe", + help="Skip cleaning of Mock chroot if build fails. " + "Old chroot can be used for debugging but if you use this option, then you need to clean unused chroot manually.", + action="store_false", + dest="scrub") + parser.add_argument("--nosrpm", + help="Skip source rpm creation.", + action="store_true") + parser.add_argument("--noinit", + help="Skip initialization (cleaning) of mock chroot.", + default=True, + action="store_false", + dest="init") + parser.add_argument("--uniqueext", + help="Unique extension used for cache.", + default=str(os.getpid()), + dest="uniqueext") + + +def main(): + """ Read arguments and start processing build configuration """ + args = ArgumentMakebuild().parser.parse_args() + + debugfiletarget = os.path.join(args.workspace, 'debug.log') + configure_logging(args.verbose, debugfiletarget) + + warn_if_incompatible_distro() + + # Start the build system + try: + build = Build(args) + build.update_building_blocks() + build.start_building() + except RpmbuilderError as err: + logger = logging.getLogger() + logger.error("Could not produce a build. %s", err) + warn_if_incompatible_distro() + raise + +if __name__ == "__main__": + try: + main() + except RpmbuilderError: + sys.exit(1) diff --git a/rpmbuilder/__init__.py b/rpmbuilder/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rpmbuilder/baseerror.py b/rpmbuilder/baseerror.py new file mode 100644 index 0000000..a897904 --- /dev/null +++ b/rpmbuilder/baseerror.py @@ -0,0 +1,17 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class RpmbuilderError(Exception): + """ Base error for rpmbuilder errors """ + pass diff --git a/rpmbuilder/buildhistory.py b/rpmbuilder/buildhistory.py new file mode 100644 index 0000000..01c2e63 --- /dev/null +++ b/rpmbuilder/buildhistory.py @@ -0,0 +1,103 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Writing of history for the build. History explain why different +projects were built at some time. """ + +import logging +import datetime +import json +import os + +class Buildhistory(object): + + """ Build history checks what has been built and + creates a file using this information """ + + def __init__(self): + self.logger = logging.getLogger(__name__) + + def update_history(self, outfile, built_projects, projects): + """ Request history and push it to be written into file """ + history = self.__gather_history(built_projects, projects) + self.__write_history_txt(outfile + '.log', history) + self.__write_history_json(outfile + '.json', history) + + def __write_history_txt(self, outfile, history): + """ Write history to a file """ + self.logger.info("Writing build history to %s", outfile) + with open(outfile, 'a') as fpoint: + for change in history: + fpoint.write(change + '\n') + for project in history[change]: + fpoint.write(' ' + project) + if 'commit' in history[change][project]: + fpoint.write(' ' + history[change][project]['commit'] + '\n') + else: + fpoint.write('\n') + for rpmfile in history[change][project]['rpmfiles']: + fpoint.write(' ' + rpmfile + '\n') + for rpmfile in history[change][project]['srpmfiles']: + fpoint.write(' ' + rpmfile + '\n') + + def __write_history_json(self, outfile, history): + """ Write dict history to a file as json """ + self.logger.info("Writing build history to %s", outfile) + jsondata = {} + if os.path.isfile(outfile): + with open(outfile, 'r') as fpoint: + jsondata = json.load(fpoint)[0] + jsondata.update(history) + with open(outfile, 'w') as fpoint: + fpoint.write(json.dumps([jsondata], indent=2, sort_keys=True) + '\n') + + """ Example of output content +{ + "2018-10-11 08:39:16.918914": { + "ansible-fm": { + "rpmfiles": [ + "ansible-fm-c46.gde71b7e-1.el7.centos.noarch.rpm" + ], + "commit": "de71b7e7fc0410df3d74cf209f5216b24157988a", + "srpmfiles": [ + "ansible-fm-c46.gde71b7e-1.el7.centos.src.rpm" + ] + } + } +} + """ + + @staticmethod + def __gather_history(built_projects, projects): + """ Loop projects and check what are the versions. This is then history """ + builddate = str(datetime.datetime.now()) + historydict = {builddate: {}} # dict for all projects + for project in built_projects: + # Store commit hash version + commitsha = None + if projects[project].project_changed and hasattr(projects[project], 'vcs') and projects[project].vcs.commitsha: + commitsha = projects[project].vcs.commitsha + + # List new rpm files from a project + rpmfiles = [] + srpmfiles = [] + for buildroot in projects[project].builders.roots: + (rpmlist, srpmlist) = projects[project].list_buildproducts_for_mockroot(buildroot) + rpmfiles.extend(rpmlist) + srpmfiles.extend(srpmlist) + projectchange = {project: {'rpmfiles': rpmfiles, 'srpmfiles': srpmfiles}} + if commitsha: + projectchange[project].update({'commit': commitsha}) + historydict[builddate].update(projectchange) + return historydict diff --git a/rpmbuilder/configfile.py b/rpmbuilder/configfile.py new file mode 100644 index 0000000..7ef8ca7 --- /dev/null +++ b/rpmbuilder/configfile.py @@ -0,0 +1,102 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Read sections from a build configuration file and check that +all required values have been given. +""" +import ConfigParser +import logging +import re + +from rpmbuilder.baseerror import RpmbuilderError + + +class Configfilereader(object): + + """ Reading and processing of user given configuration file """ + + def __init__(self, configfile): + self.logger = logging.getLogger(__name__) + self.configfile = configfile + self.configuration = self.readconfig(configfile) + + def readconfig(self, configfile): + """ Configuration file reading """ + conf = ConfigParser.ConfigParser() + try: + with open(configfile) as filep: + conf.readfp(filep) + except IOError: + raise ConfigError("Failed to open configuration file %s" % configfile) + + self.__validate_section_names(conf) + return conf + + def get_bool(self, section, option, mandatory=False, defaultvalue=False): + """ Get boolean values from configuration. In case of problems do raise + or just return default value """ + try: + return self.configuration.getboolean(section, option) + except ConfigParser.NoSectionError: + raise ConfigError("Could not find required [%s] section in configuration" % section) + except ConfigParser.NoOptionError: + if mandatory: + raise ConfigError("Could not find option %s from [%s] section" % option, section) + else: + return defaultvalue + except: + raise + + def get_string(self, section, option, mandatory=False, defaultvalue=None): + """ Return the requested value from the given section. In case of problems + do raise or just return default value""" + try: + return self.configuration.get(section, option) + except ConfigParser.NoSectionError: + raise ConfigError("Could not find required [%s] section in configuration" % section) + except ConfigParser.NoOptionError: + if mandatory: + raise ConfigError("Could not find option %s from [%s] section" % option, section) + else: + return defaultvalue + except: + raise + + def get_sections(self): + """ List all sections from the configuration """ + try: + return self.configuration.sections() + except: + raise + + def __validate_section_names(self, configuration): + """ Loop through all section names and do validation """ + for section in configuration.sections(): + self.__validate_section_name(section) + + def __validate_section_name(self, name): + """ Check that section contains characters that + do not cause problems for directory names """ + if not re.match('^[A-Za-z0-9-]+$', name): + self.logger.critical("Configuration of [%s] has problems.", name, + "Section name can has illegal characters" + "Use only alphanumeric and dashes") + raise ConfigError("Section %s name contains illegal characters" % name) + + +class ConfigError(RpmbuilderError): + + """ Exception for configuration file content problems """ + pass diff --git a/rpmbuilder/executor.py b/rpmbuilder/executor.py new file mode 100644 index 0000000..27c6d30 --- /dev/null +++ b/rpmbuilder/executor.py @@ -0,0 +1,33 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import subprocess + +logger = logging.getLogger(__name__) + + +class Executor(object): + def run(self, cmd): + logger.debug('Executing: {}'.format(cmd)) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + if stderr and p.returncode == 0: + logger.debug( + 'Command {} exit status {} but stderr not empty: "{}"'.format(cmd, p.returncode, + stderr)) + if p.returncode != 0: + raise Exception('Command {} returned non-zero exit status {}: ' + 'stdout="{}", stderr="{}"'.format(cmd, p.returncode, stdout, stderr)) + return stdout diff --git a/rpmbuilder/executor_test.py b/rpmbuilder/executor_test.py new file mode 100644 index 0000000..d659667 --- /dev/null +++ b/rpmbuilder/executor_test.py @@ -0,0 +1,48 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import mock +import pytest + +from rpmbuilder.executor import Executor + + +@pytest.mark.parametrize('input_cmd, expected_output', [ + (['true'], ''), + (['echo', 'foo'], 'foo\n'), +]) +def test_run_cmd(input_cmd, expected_output): + assert Executor().run(input_cmd) == expected_output + + +@mock.patch('logging.Logger.debug') +@mock.patch('subprocess.Popen') +def test_stderr_is_logged(mock_popen, mock_log): + process_mock = mock.Mock() + process_mock.configure_mock(**{ + 'communicate.return_value': ('some ouput', 'some errput'), + 'returncode': 0, + }) + mock_popen.return_value = process_mock + Executor().run(['ls']) + assert re.match('.*exit status 0 but stderr not empty.*', mock_log.call_args[0][0]) + + +def test_run_cmd_fail(): + err_regexp = 'Command .* returned non-zero exit status 2: stdout="", ' \ + 'stderr="ls: .* No such file or directory' + with pytest.raises(Exception, + match=err_regexp): + Executor().run(['ls', 'bar']) diff --git a/rpmbuilder/get_sources.py b/rpmbuilder/get_sources.py new file mode 100644 index 0000000..3fc986f --- /dev/null +++ b/rpmbuilder/get_sources.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import shutil +import hashlib +import lxml.html +import urllib3 + + +HTTP = urllib3.PoolManager() + +def debug(log): + print(log) + +def verbose(log): + print(log) + +def filter_dot(lst): + return filter(lambda path: path[0] != '.', lst) + +def get_url(url, file_hash): + debug("http get {}".format(url)) + request = HTTP.request('GET', url) + dom = lxml.html.fromstring(request.data) + for link in filter_dot(dom.xpath('//a/@href')): + path = '{}/{}'.format(url, link) + debug("http get {}".format(path)) + request = HTTP.request('GET', path) + dom = lxml.html.fromstring(request.data) + if file_hash in dom.xpath('//a/@href'): + return '{}{}'.format(path, file_hash) + +def get_repo_name(path): + regex = re.compile(r'^\.([^.]*).metadata$') + meta = list(filter(regex.match, os.listdir(path))) + if len(meta) == 0: + return None + if len(meta) != 1: + raise Exception('Multiple metadata files: {}'.format(", ".join(meta))) + repo_name = regex.search(meta[0]).group(1) + debug("repo name is {}".format(repo_name)) + return repo_name + +def parse_metadatafile(path, repo_name): + result = {} + filename = "{}/.{}.metadata".format(path, repo_name) + debug("metadata file: {}".format(filename)) + with open(filename) as metadata: + for line in metadata: + items = line.split() + result[items[1]] = items[0] + debug('found {}: {}'.format(items[1], items[0])) + return result + +def get_hash(filename, hashfunc): + with open(filename, 'rb', buffering=0) as contents: + for buffer in iter(lambda: contents.read(128*1024), b''): + hashfunc.update(buffer) + digest = hashfunc.hexdigest() + debug("digest is {}".format(digest)) + return digest + +def check_file(filename, checksum): + debug("checking {} {}".format(filename, checksum)) + hashmap = { + 32 : hashlib.md5(), + 40 : hashlib.sha1(), + 64 : hashlib.sha256(), + 128 : hashlib.sha512() + } + if len(checksum) not in hashmap: + raise Exception('Checksum lenght unsupported: {}'.format(checksum)) + if get_hash(filename, hashmap[len(checksum)]) != checksum: + raise Exception("Checksum doesn't match: {} {}".format(filename, checksum)) + debug("checksum ok") + +def download(url, destination, checksum): + tmpfile = "{}.tmp".format(destination) + try: + debug("downloading {} to {}".format(url, tmpfile)) + with HTTP.request('GET', url, preload_content=False) as resp, open(tmpfile, 'wb') as out_file: + shutil.copyfileobj(resp, out_file) + check_file(tmpfile, checksum) + debug("renaming {} to {}".format(tmpfile, destination)) + os.rename(tmpfile, destination) + finally: + try: + os.remove(tmpfile) + debug("removed {}".format(tmpfile)) + except OSError: + pass + +def get_sources(path, sources_list, logger): + if logger: + global debug + global verbose + debug = logger.debug + verbose = logger.info + + repo = get_repo_name(path) + if not repo: + verbose('no metadata file in "{}".'.format(path)) + return + + for k, v in parse_metadatafile(path, repo).items(): + filename = os.path.join(path, k) + try: + check_file(filename, v) + except: + found = False + for sources in sources_list: + repo_root = "{}/{}".format(sources, repo) + url = get_url(repo_root, v) + if url: + debug("retrieving {} to {}".format(url, filename)) + download(url, filename, v) + verbose('retrieved "{}"'.format(k)) + found = True + break + if not found: + raise Exception('File "{}" not found'.format(v)) diff --git a/rpmbuilder/log.py b/rpmbuilder/log.py new file mode 100644 index 0000000..73e4003 --- /dev/null +++ b/rpmbuilder/log.py @@ -0,0 +1,44 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Logging configuration for rpm builder +""" +import logging +import os +import sys + +def configure_logging(verbose=False, debugfile="debug.log"): + """ Logging to screen(console) and file """ + + debugfile_dirname = os.path.dirname(debugfile) + if not os.path.isdir(debugfile_dirname): + os.mkdir(debugfile_dirname) + + logging.basicConfig(level=logging.DEBUG, + format='%(asctime)s %(levelname)s: %(message)s', + filename=debugfile, + filemode='w') + # define a Handler which writes INFO messages or higher to the sys.stderr + console = logging.StreamHandler(stream=sys.stdout) + if verbose: + console.setLevel(logging.DEBUG) + else: + console.setLevel(logging.INFO) + # set a format which is simpler for console use + formatter = logging.Formatter('%(levelname)s: %(message)s') + # tell the handler to use this format + console.setFormatter(formatter) + # add the handler to the root log + logging.getLogger('').addHandler(console) diff --git a/rpmbuilder/mockbuilder.py b/rpmbuilder/mockbuilder.py new file mode 100644 index 0000000..2365762 --- /dev/null +++ b/rpmbuilder/mockbuilder.py @@ -0,0 +1,135 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Handling of mock building environment """ +import json +import logging +import os + +from rpmbuilder.baseerror import RpmbuilderError +from rpmbuilder.version_control import VersionControlSystem + + +class Mockbuilder(object): + + """ Mockbuilder handled mock building configuration """ + + def __init__(self): + self.logger = logging.getLogger(__name__) + self.roots = [] + + +class LocalMockbuilder(Mockbuilder): + + """ Mock configuration contains information of chroot used for building. + Configuration is taken from local file""" + + def __init__(self, configfile): + super(LocalMockbuilder, self).__init__() + + self.roots.append(os.path.basename(configfile.rstrip('/').rstrip('.cfg'))) + self.configdir = os.path.dirname(os.path.abspath(configfile)) + + def get_configdir(self): + return self.configdir + + def store_builder_status(self): + pass + + +class GitMockbuilder(Mockbuilder): + + """ Mock configuration contains information of chroot used for building. + Configuration is taken from git""" + + def __init__(self, workspace, conf): + super(GitMockbuilder, self).__init__() + + self.mock_settings_dir = os.path.join(workspace, "mocksettings") + self.mock_settings_checkout_dir = os.path.join(self.mock_settings_dir, + "checkout") + + confsection = "mock" + self.roots = self.__list_from_csv(conf.get_string(confsection, + "roots", + mandatory=True)) + + self.vcs = VersionControlSystem(self.mock_settings_checkout_dir) + + try: + self.vcs.update_git_project(conf.get_string(confsection, + "url", + mandatory=True), + conf.get_string(confsection, + "ref", + mandatory=True)) + except: + self.logger.critical("Problems updating git clone") + raise + + def get_configdir(self): + return os.path.join(self.mock_settings_checkout_dir, 'etc', 'mock') + + def store_builder_status(self): + """ Save information of the builder checkout. This way we can + check if mock configuration has changed and all projects can be + rebuild """ + statusfile = os.path.join(self.mock_settings_dir, 'status.txt') + self.logger.debug("Updating %s", statusfile) + projectstatus = {"sha": self.vcs.commitsha} + try: + with open(statusfile, 'w') as outfile: + json.dump(projectstatus, outfile) + except: + self.logger.error("Could not create a status file") + raise + + def check_builder_changed(self): + """ + Check if there has been changes in the project + if project has not been compiled -> return = True + if project has GIT/VCS changes -> return = True + if project has not changed -> return = False + """ + statusfile = os.path.join(self.mock_settings_dir, 'status.txt') + + if os.path.isfile(statusfile): + with open(statusfile, 'r') as filep: + previousprojectstatus = json.load(filep) + # Compare old values against new values + if previousprojectstatus['sha'] != self.vcs.commitsha: + self.logger.debug("Mock configuration has changed") + return True + else: + self.logger.debug("Mock configuration has NO changes") + return False + else: + # No configuration means that project has not been compiled + pass + return True + + @staticmethod + def __list_from_csv(csv): + """ Create a list of comma separated value list + For example foo,bar would be converted to ["foo","bar"] """ + outlist = [] + for entry in set(csv.split(',')): + outlist.append(entry.strip()) + return outlist + + +class MockbuilderError(RpmbuilderError): + + """ Exceptions originating from Builder and main level """ + pass diff --git a/rpmbuilder/packagebuilding.py b/rpmbuilder/packagebuilding.py new file mode 100644 index 0000000..121eeea --- /dev/null +++ b/rpmbuilder/packagebuilding.py @@ -0,0 +1,448 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Module in charge of building a project """ +import glob +import logging +import os +import pwd +import shutil +import subprocess +from distutils.spawn import find_executable + +import datetime +from rpmbuilder.baseerror import RpmbuilderError +from rpmbuilder.prettyprinter import Prettyprint + +PIGZ_INSTALLED = False +PBZIP2_INSTALLED = False +PXZ_INSTALLED = False + +class Packagebuilding(object): + + """ Object for building rpm files with mock """ + + def __init__(self, masterargs): + # Chroothousekeeping cleans chroot in case of mock errors. This should + # keep /var/lib/mock from growing too much + self.masterargs = masterargs + self.logger = logging.getLogger(__name__) + self.__check_tool_availability() + self.chroot_installed_rpms = [] + + if find_executable("pigz"): + global PIGZ_INSTALLED + PIGZ_INSTALLED = True + self.logger.debug("pigz is available") + if find_executable("pbzip2"): + global PBZIP2_INSTALLED + PBZIP2_INSTALLED = True + self.logger.debug("pbzip2 is available") + if find_executable("pxz"): + global PXZ_INSTALLED + PXZ_INSTALLED = True + self.logger.debug("pxz is available") + + @staticmethod + def __check_tool_availability(): + """ Verify that user belongs to mock group for things to work """ + username = pwd.getpwuid(os.getuid())[0] + cmd = "id " + username + "| grep \\(mock\\) > /dev/null" + if os.system(cmd) != 0: + raise PackagebuildingError("Mock tool requires user to " + "belong to group called mock") + return True + + def patch_specfile(self, origspecfile, outputdir, newversion, newrelease): + """ Spec file is patched with version information from git describe """ + Prettyprint().print_heading("Patch spec", 50) + self.logger.info("Patching new spec from %s", origspecfile) + self.logger.debug(" - Version: %s", newversion) + self.logger.debug(" - Release: %s", newrelease) + + specfilebasename = os.path.basename(origspecfile) + patchedspecfile = os.path.join(outputdir, specfilebasename) + self.logger.debug("Writing new spec file to %s", patchedspecfile) + + with open(origspecfile, 'r') as filepin: + filepin_lines = filepin.readlines() + + with open(patchedspecfile, 'w') as filepout: + for line in filepin_lines: + linestripped = line.strip() + if not linestripped.startswith("#"): + # Check if version could be patched + if linestripped.lower().startswith("version:"): + filepout.write("Version: " + newversion + '\n') + elif linestripped.lower().startswith("release:"): + filepout.write("Release: " + newrelease + '\n') + else: + filepout.write(line) + return patchedspecfile + + def init_mock_chroot(self, resultdir, configdir, root): + """ + Start a mock chroot where build requirements + can be installed before building + """ + Prettyprint().print_heading("Mock init in " + root, 50) + + self.clean_directory(resultdir) + + mock_arg_resultdir = "--resultdir=" + resultdir + + mocklogfile = resultdir + '/mock-init-' + root + '.log' + + arguments = [mock_arg_resultdir, + "--scrub=all"] + self.run_mock_command(arguments, mocklogfile, configdir, root) + + #Allow the builder to run sudo without terminal and without password + #This makes it possible to run disk image builder needed by ipa-builder + allow_sudo_str = "mockbuild ALL=(ALL) NOPASSWD: ALL" + notty_str = "Defaults:mockbuild !requiretty" + sudoers_file = "/etc/sudoers" + command = "grep \'%s\' %s || echo -e \'%s\n%s\' >> %s" %(allow_sudo_str, sudoers_file, allow_sudo_str, notty_str, sudoers_file) + arguments=["--chroot", + command ] + self.run_mock_command(arguments, mocklogfile, configdir, root) + + return True + + def restore_local_repository(self, localdir, destdir, configdir, root, logfile): + """ + Mock copying local yum repository to mock environment so that it can + be used during building of other RPM packages. + """ + Prettyprint().print_heading("Restoring local repository", 50) + arguments = ["--copyin", + localdir, + destdir] + self.run_mock_command(arguments, logfile, configdir, root) + + def mock_source_rpm(self, hostsourcedir, specfile, resultdir, configdir, root): + """ Mock SRPM file which can be used to build rpm """ + Prettyprint().print_heading("Mock source rpm in " + root, 50) + self.logger.info("Build from:") + self.logger.info(" - source directory %s", hostsourcedir) + self.logger.info(" - spec %s", specfile) + + self.clean_directory(resultdir) + + mock_arg_resultdir = "--resultdir=" + resultdir + mock_arg_spec = "--spec=" + specfile + mock_arg_sources = "--sources=" + hostsourcedir + arguments = [mock_arg_resultdir, + "--no-clean", + "--no-cleanup-after", + "--buildsrpm", + mock_arg_sources, + mock_arg_spec] + + mocklogfile = resultdir + '/mock.log' + self.run_mock_command(arguments, mocklogfile, configdir, root) + + # Find source rpm and return the path + globstring = resultdir + '/*.src.rpm' + globmatches = glob.glob(globstring) + assert len(globmatches) == 1, "Too many source rpm files" + + return globmatches[0] + + def mock_rpm(self, sourcerpm, resultdir, configdir, root): + """ Mock RPM binary file from SRPM """ + Prettyprint().print_heading("Mock rpm in " + root, 50) + self.logger.info("Building from:") + self.logger.info(" - source rpm %s", sourcerpm) + + self.clean_directory(resultdir) + + mock_arg_resultdir = "--resultdir=" + resultdir + arguments = [mock_arg_resultdir, + "--no-clean", + "--no-cleanup-after", + "--rebuild", + sourcerpm] + + mocklogfile = resultdir + '/mock.log' + self.run_mock_command(arguments, mocklogfile, configdir, root) + + self.logger.debug("RPM files build to: %s", resultdir) + return True + + def mock_rpm_from_archive(self, source_tar_packages, resultdir, configdir, root): + """ Mock rpm binary file straight from archive file """ + self.clean_directory(resultdir) + + # Copy source archive to chroot + chroot_sourcedir = "/builddir/build/SOURCES/" + self.copy_to_chroot(configdir, root, resultdir, source_tar_packages, chroot_sourcedir) + + # Create rpm from source archive + sourcebasename = os.path.basename(source_tar_packages[0]) + chrootsourcefile = os.path.join(chroot_sourcedir, sourcebasename) + + Prettyprint().print_heading("Mock rpm in " + root, 50) + self.logger.info("Building from:") + self.logger.info(" - source archive %s", chrootsourcefile) + + mock_arg_resultdir = "--resultdir=" + resultdir + rpmbuildcommand = "/usr/bin/rpmbuild --noclean -tb -v " + rpmbuildcommand += os.path.join(chroot_sourcedir, chrootsourcefile) + arguments = [mock_arg_resultdir, + "--chroot", + rpmbuildcommand] + mocklogfile = resultdir + '/mock-rpmbuild.log' + self.run_mock_command(arguments, mocklogfile, configdir, root) + + def mock_rpm_from_filesystem(self, path, spec, resultdir, configdir, root, srpm_resultdir): + """ Mock rpm binary file straight from archive file """ + self.clean_directory(resultdir) + # Copy source archive to chroot + chroot_sourcedir = "/builddir/build/" + self.copy_to_chroot(configdir, root, resultdir, [os.path.join(path, 'SPECS', spec)], os.path.join(chroot_sourcedir, 'SPECS')) + self.copy_to_chroot(configdir, root, resultdir, [os.path.join(path, 'SOURCES', f) for f in os.listdir(os.path.join(path, 'SOURCES'))], os.path.join(chroot_sourcedir, 'SOURCES')) + + Prettyprint().print_heading("Mock rpm in " + root, 50) + mocklogfile = resultdir + '/mock-rpmbuild.log' + mock_arg_resultdir = "--resultdir=" + resultdir + arguments = [mock_arg_resultdir, + "--chroot", + "chown -R root:root "+chroot_sourcedir] + self.run_mock_command(arguments, mocklogfile, configdir, root) + rpmbuildcommand = "/usr/bin/rpmbuild --noclean -ba -v " + rpmbuildcommand += os.path.join(chroot_sourcedir, 'SPECS', spec) + arguments = [mock_arg_resultdir, + "--chroot", + rpmbuildcommand] + mocklogfile = resultdir + '/mock-rpmbuild.log' + self.run_mock_command(arguments, mocklogfile, configdir, root) + + arguments = ["--copyout", + "/builddir/build", resultdir+"/tmp/packages"] + mocklogfile = resultdir + '/mock-copyout.log' + self.run_mock_command(arguments, mocklogfile, configdir, root) + + for filename in glob.glob(resultdir+"/tmp/packages/RPMS/*"): + shutil.move(filename, resultdir) + + for filename in glob.glob(resultdir+"/tmp/packages/SRPMS/*"): + shutil.move(filename, srpm_resultdir) + + def mock_wipe_buildroot(self, resultdir, configdir, root): + """ Wipe buildroot clean """ + Prettyprint().print_heading("Wiping buildroot", 50) + arguments = ["--chroot", + "mkdir -pv /usr/localrepo && " \ + "cp -v /builddir/build/RPMS/*.rpm /usr/localrepo/. ;" \ + "rm -rf /builddir/build/{BUILD,RPMS,SOURCES,SPECS,SRPMS}/*"] + mocklogfile = resultdir + '/mock-wipe-buildroot.log' + self.run_mock_command(arguments, mocklogfile, configdir, root) + + def update_local_repository(self, configdir, root): + Prettyprint().print_heading("Update repository " + root, 50) + + arguments = ["--chroot", + "mkdir -pv /usr/localrepo && " \ + "createrepo --update /usr/localrepo && yum clean expire-cache"] + self.run_mock_command(arguments, configdir+"/log", configdir, root) + + def copy_to_chroot(self, configdir, root, resultdir, source_files, destination): + # Copy source archive to chroot + Prettyprint().print_heading("Copy source archive to " + root, 50) + self.logger.info(" - Copy from %s", source_files) + self.logger.info(" - Copy to %s", destination) + + mock_arg_resultdir = "--resultdir=" + resultdir + arguments = [mock_arg_resultdir, + "--copyin"] + arguments.extend(source_files) + arguments.append(destination) + + mocklogfile = resultdir + '/mock-copyin.log' + self.run_mock_command(arguments, mocklogfile, configdir, root) + + def scrub_mock_chroot(self, configdir, root): + time_start = datetime.datetime.now() + Prettyprint().print_heading("Scrub mock chroot " + root, 50) + mock_clean_command = ["/usr/bin/mock", + "--configdir=" + configdir, + "--root=" + root, + "--uniqueext=" + self.masterargs.uniqueext, + "--orphanskill", + "--scrub=chroot"] + self.logger.info("Removing mock chroot.") + self.logger.debug(" ".join(mock_clean_command)) + try: + subprocess.check_call(mock_clean_command, + shell=False, + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as err: + raise PackagebuildingError("Mock chroot removal failed. Error code %s" % (err.returncode)) + time_delta = datetime.datetime.now() - time_start + self.logger.debug('[mock-end] cmd="%s" took=%s (%s sec)', mock_clean_command, time_delta, time_delta.seconds) + + def run_builddep(self, specfile, resultdir, configdir, root): + arguments = ["--copyin"] + arguments.append(specfile) + arguments.append("/builddir/"+os.path.basename(specfile)) + + mocklogfile = resultdir + '/mock-builddep.log' + self.run_mock_command(arguments, mocklogfile, configdir, root) + + builddepcommand = "/usr/bin/yum-builddep -y "+"/builddir/"+os.path.basename(specfile) + arguments = ["--chroot", + builddepcommand] + mocklogfile = resultdir + '/mock-builddep.log' + return self.run_mock_command(arguments, mocklogfile, configdir, root, True) == 0 + + def run_mock_command(self, arguments, outputfile, configdir, root, return_error=False): + """ Mock binary rpm package """ + mock_command = ["/usr/bin/mock", + "--configdir=" + configdir, + "--root=" + root, + "--uniqueext=" + self.masterargs.uniqueext, + "--verbose", + "--old-chroot", + "--enable-network"] + mock_command.extend(arguments) + if self.masterargs.mockarguments: + mock_command.extend([self.masterargs.mockarguments]) + self.logger.info("Running mock. Log goes to %s", outputfile) + self.logger.debug('[mock-start] cmd="%s"', mock_command) + time_start = datetime.datetime.now() + self.logger.debug(" ".join(mock_command)) + with open(outputfile, 'a') as filep: + try: + mockproc = subprocess.Popen(mock_command, + shell=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + for line in iter(mockproc.stdout.readline, b''): + if self.masterargs.verbose: + self.logger.debug("mock-%s", line.rstrip('\n')) + filep.write(line) + _, stderr = mockproc.communicate() # wait for the subprocess to exit + if return_error: + return mockproc.returncode + if mockproc.returncode != 0: + raise Mockcommanderror(returncode=mockproc.returncode) + except Mockcommanderror as err: + self.logger.error("There was a failure during mocking") + if self.masterargs.scrub: + self.scrub_mock_chroot(configdir, root) + guidance_message = "" + else: + mock_shell_command = ["/usr/bin/mock", + "--configdir=" + configdir, + "--root=" + root, + "--uniqueext=" + self.masterargs.uniqueext, + "--shell"] + guidance_message = ". To open mock shell, run the following: " + " ".join(mock_shell_command) + raise PackagebuildingError("Mock exited with value \"%s\". " + "Log for debuging: %s %s" % (err.returncode, outputfile, guidance_message)) + except OSError: + raise PackagebuildingError("Mock executable not found. " + "Have you installed mock?") + except: + raise + time_delta = datetime.datetime.now() - time_start + self.logger.debug('[mock-end] cmd="%s" took=%s (%s sec)', mock_command, time_delta, time_delta.seconds) + + def clean_directory(self, directory): + """ Make sure given directory exists and is clean """ + if os.path.isdir(directory): + shutil.rmtree(directory) + os.makedirs(directory) + + def tar_filter(self, tarinfo): + """ Filter git related and spec files away """ + if tarinfo.name.endswith('.spec') or tarinfo.name.endswith('.git'): + self.logger.debug("Ignore %s", tarinfo.name) + return None + self.logger.debug("Archiving %s", tarinfo.name) + return tarinfo + + def create_source_archive(self, + package_name, + sourcedir, + outputdir, + project_changed, + archive_file_extension): + """ + Create tar file. Example helloworld-2.4.tar.gz + Tar file has naming -.tar.gz + """ + Prettyprint().print_heading("Tar package creation", 50) + + tar_file = package_name + '.' + 'tar' + # Directory where tar should be stored. + # Example /var/mybuild/workspace/sources + + tarfilefullpath = os.path.join(outputdir, tar_file) + if os.path.isfile(tarfilefullpath) and not project_changed: + self.logger.info("Using cached %s", tarfilefullpath) + return tarfilefullpath + + self.logger.info("Creating tar file %s", tarfilefullpath) + # sourcedir = /var/mybuild/helloworld/checkout + # sourcedir_dirname = /var/mybuild/helloworld + # sourcedir_basename = checkout + sourcedir_dirname = os.path.dirname(sourcedir) + + os.chdir(sourcedir_dirname) + + tar_params = ["tar", "cf", tarfilefullpath, "--directory="+os.path.dirname(sourcedir)] + tar_params = tar_params+["--exclude-vcs"] + tar_params = tar_params+["--transform=s/" + os.path.basename(sourcedir) + "/" + os.path.join(package_name) + "/"] + tar_params = tar_params+[os.path.basename(sourcedir)] + self.logger.debug("Running: %s", " ".join(tar_params)) + ret = subprocess.call(tar_params) + if ret > 0: + raise PackagebuildingError("Tar error: %s", ret) + + git_dir = os.path.join(os.path.basename(sourcedir), '.git') + if os.path.exists(git_dir): + tar_params = ["tar", "rf", tarfilefullpath, "--directory="+os.path.dirname(sourcedir)] + tar_params += ["--transform=s/" + os.path.basename(sourcedir) + "/" + os.path.join(package_name) + "/"] + tar_params += ['--dereference', git_dir] + self.logger.debug("Running: %s", " ".join(tar_params)) + ret = subprocess.call(tar_params) + if ret > 1: + self.logger.warning("Git dir tar failed") + + if archive_file_extension == "tar.gz": + if PIGZ_INSTALLED: + cmd = ['pigz', '-f'] + else: + cmd = ['gzip', '-f'] + resultfile = tarfilefullpath + '.gz' + else: + raise PackagebuildingError("Unknown source archive format: %s" % archive_file_extension) + cmd += [tarfilefullpath] + self.logger.debug("Running: %s", " ".join(cmd)) + ret = subprocess.call(cmd) + if ret > 0: + raise PackagebuildingError("Cmd error: %s", ret) + + return resultfile + +class Mockcommanderror(RpmbuilderError): + def __init__(self, returncode): + self.returncode = returncode + +class PackagebuildingError(RpmbuilderError): + + """ Exceptions originating from Builder and main level """ + pass diff --git a/rpmbuilder/prettyprinter.py b/rpmbuilder/prettyprinter.py new file mode 100644 index 0000000..ed75b77 --- /dev/null +++ b/rpmbuilder/prettyprinter.py @@ -0,0 +1,25 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Pretty printer provides pretty printing """ +import logging + +class Prettyprint(object): + """ Pretty and common printing for all modules """ + + @staticmethod + def print_heading(string, weight): + """ Print pretty heading title """ + logger = logging.getLogger(__name__) + logger.info(str(" " + string + " ").center(weight, "*")) diff --git a/rpmbuilder/project.py b/rpmbuilder/project.py new file mode 100644 index 0000000..e100d94 --- /dev/null +++ b/rpmbuilder/project.py @@ -0,0 +1,561 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Project is a subsystem which contains one spec file which +defines how it is build. Every project has one git +repository from where it is cloned from. +""" +import glob +import json +import logging +import os +import shutil +import subprocess + +import re + +import datetime +from rpmbuilder.baseerror import RpmbuilderError +from rpmbuilder.prettyprinter import Prettyprint +from rpmbuilder.rpmtools import Repotool, Specworker, RepotoolError, SpecError +from rpmbuilder.utils import find_files +from rpmbuilder.version_control import VersionControlSystem, VcsError +from rpmbuilder.get_sources import get_sources + + +class Project(object): + + """ Instance of a project """ + + def __init__(self, name, workspace, projects, builders, packagebuilder, chrootscrub=True, nosrpm=False): + self.name = name + + self.logger = logging.getLogger(__name__ + "." + self.name) + + self.project_rebuild_needed = False + + self.project_workspace = os.path.join(workspace, + 'projects', + self.name) + + self.projects = projects + self.builders = builders + self.directory_of_specpatch = os.path.join(self.project_workspace, + 'rpmbuild', + 'spec') + self.directory_of_sourcepackage = os.path.join(self.project_workspace, + 'rpmbuild', + 'sources') + self.directory_of_srpms = os.path.join(self.project_workspace, + 'rpmbuild', + 'srpm') + self.directory_of_rpm = os.path.join(self.project_workspace, + 'rpmbuild', + 'rpm') + self.directory_of_commonrepo = os.path.join(workspace, + 'buildrepository') + + self.directory_of_builder = self.builders.get_configdir() + + self.__create_directories([self.directory_of_specpatch, + self.directory_of_srpms], + verify_empty=True) + self.__create_directories([self.directory_of_sourcepackage], + verify_empty=False) + + self.packagebuilder = packagebuilder + + self.chrootscrub = chrootscrub + self.built = {} + for mockroot in builders.roots: + self.built[mockroot] = False + + self.project_changed = False + self.projconf = None + self.spec = None + self.useversion = None + self.directory_of_checkout = None + self.nosrpm = nosrpm + self.centos_style = False + self.buildrequires_downstream = set() + self.buildrequires_upstream = set() + + def mark_for_rebuild(self): + """ Marking project for rebuild only if project has not changed """ + if not self.project_changed: + self.logger.debug("Marking project %s for rebuild.", self.name) + self.project_rebuild_needed = True + + def mark_downstream_for_rebuild(self, marked_for_build=None): + """ + Recursively mark downstream projects for rebuilding. + Return set of projects marked for rebuild + """ + if marked_for_build is None: + marked_for_build = set() + self.logger.debug("Marking downstream for rebuild in \"%s\"", + self.name) + for project in self.who_buildrequires_me(): + self.logger.debug("BuildRequires to \"%s\" found in \"%s\"", + self.name, project) + if project in marked_for_build: + self.logger.debug("\"%s\" already marked for build", project) + elif self.projects[project].project_rebuild_needed: + self.logger.debug("\"%s\" already marked for rebuild", project) + else: + self.projects[project].mark_for_rebuild() + marked_for_build.add(project) + # Check if downstream has downstream projects + tmpset = self.projects[project].mark_downstream_for_rebuild( + marked_for_build) + marked_for_build.update(tmpset) + return marked_for_build + + def build_project(self, mockroot): + """ Do building of SRPM and RPM files """ + time_start = datetime.datetime.now() + Prettyprint().print_heading("Build " + self.name, 60) + assert not self.built[mockroot], "Project already built" + + # Produce spec file + if self.spec.version == '%{_version}': + self.logger.debug("patching spec file") + self.logger.debug("Version in spec is going to be %s", self.useversion) + + rpm = Repotool() + userelease = rpm.next_release_of_package( + os.path.join(self.directory_of_commonrepo, + self.builders.roots[0], + "rpm"), + self.spec.name, + self.useversion, + self.spec.release) + self.logger.debug("Release in spec is going to be %s", userelease) + + specfile = self.packagebuilder.patch_specfile(self.spec.specfilefullpath, + self.directory_of_specpatch, + self.useversion, + userelease) + else: + self.logger.debug("Skipping spec patching") + specfile = self.spec.specfilefullpath + + # Start mocking + self.logger.debug("Starting building in root \"%s\"", mockroot) + if self.centos_style: + shutil.rmtree(self.directory_of_sourcepackage) + ignore_git = shutil.ignore_patterns('.git') + shutil.copytree(self.directory_of_checkout, self.directory_of_sourcepackage, ignore=ignore_git) + sources_key = 'CENTOS_SOURCES' + if sources_key not in os.environ: + raise RpmbuilderError('Cannot build CentOS style RPM, %s not defined in the environment' % sources_key) + get_sources(self.directory_of_sourcepackage, os.environ[sources_key].split(','), self.logger) + self.create_rpm_from_filesystem(self.directory_of_sourcepackage, mockroot) + elif self.nosrpm: + list_of_source_packages = self.get_source_package() + self.create_rpm_from_archive(list_of_source_packages, mockroot) + else: + self.get_source_package() + # Create source RPM file + sourcerpm = self.get_source_rpm(self.directory_of_sourcepackage, specfile, mockroot) + + # Create final RPM file(s) + self.create_rpm_from_srpm(sourcerpm, mockroot) + + # Mark build completed + self.built[mockroot] = True + time_delta = datetime.datetime.now() - time_start + self.logger.info('Building success: %s (took %s [%s sec])', self.name, time_delta, time_delta.seconds) + + # We wipe buildroot of previously built rpm, source etc. packages + # This is custom cleaning which does not remove chroot + self.packagebuilder.mock_wipe_buildroot(self.project_workspace, self.directory_of_builder, mockroot) + + def pull_source_packages(self, target_dir): + cmd = ['/usr/bin/spectool', '-d', 'KVERSION a.b', '-g', '--directory', target_dir, self.spec.specfilefullpath] + self.logger.info('Pulling source packages: %s', cmd) + try: + subprocess.check_call(cmd, shell=False) + self.logger.info('Pulling source packages ok') + except OSError as err: + self.logger.info('Pulling source packages nok %s', err.strerror) + raise RepotoolError("Calling of command spectool caused: \"%s\"" % err.strerror) + except: + self.logger.info('Pulling source packages nok ??', err.strerror) + raise RepotoolError("There was error pulling source content") + + def get_source_package(self): + # Produce source package + source_package_list = [] + for source_file_hit in self.spec.source_files: + self.logger.info("Acquiring source file \"%s\"", source_file_hit) + if re.match(r'^(http[s]*|ftp)://', source_file_hit): + self.logger.info("PULL %s", self.directory_of_sourcepackage) + self.pull_source_packages(self.directory_of_sourcepackage) + source_package_list.append(self.directory_of_sourcepackage + '/' + source_file_hit.split('/')[-1]) + continue + for subdir in ["", "SOURCES"]: + if os.path.isfile(os.path.join(self.directory_of_checkout, subdir, source_file_hit)): + shutil.copy(os.path.join(self.directory_of_checkout, subdir, source_file_hit), self.directory_of_sourcepackage) + source_package_list.append(os.path.join(self.directory_of_sourcepackage, source_file_hit)) + break + else: + tarname = self.spec.name + '-' + self.useversion + source_package_list.append(self.packagebuilder.create_source_archive(tarname, + self.directory_of_checkout, + self.directory_of_sourcepackage, + self.project_changed, + self.spec.source_file_extension)) + + for patch_file_hit in self.spec.patch_files: + self.logger.info("Copying %s to directory %s", patch_file_hit, self.directory_of_sourcepackage) + for subdir in ["", "SOURCES"]: + if os.path.isfile(os.path.join(self.directory_of_checkout, subdir, patch_file_hit)): + shutil.copy(os.path.join(self.directory_of_checkout, subdir, patch_file_hit), self.directory_of_sourcepackage) + break + else: + raise ProjectError("Spec file lists patch \"%s\" but no file found" % patch_file_hit) + return source_package_list + + + def get_source_rpm(self, hostsourcedir, specfile, mockroot): + return self.packagebuilder.mock_source_rpm(hostsourcedir, + specfile, + self.directory_of_srpms, + self.directory_of_builder, + mockroot) + + def create_rpm_from_srpm(self, sourcerpm, mockroot): + directory_of_rpm = os.path.join(self.directory_of_rpm, mockroot) + self.packagebuilder.mock_rpm(sourcerpm, + directory_of_rpm, + self.directory_of_builder, + mockroot) + # Delete duplicated src.rpm which is returned by rpm creation + os.remove(os.path.join(directory_of_rpm, os.path.basename(sourcerpm))) + + def create_rpm_from_archive(self, source_tar_packages, mockroot): + directory_of_rpm = os.path.join(self.directory_of_rpm, mockroot) + self.packagebuilder.mock_rpm_from_archive(source_tar_packages, directory_of_rpm, self.directory_of_builder, mockroot) + + def create_rpm_from_filesystem(self, path, mockroot): + directory_of_rpm = os.path.join(self.directory_of_rpm, mockroot) + self.packagebuilder.mock_rpm_from_filesystem(path, + self.spec.specfilename, + directory_of_rpm, + self.directory_of_builder, + mockroot, + self.directory_of_srpms) + + def list_buildproducts_for_mockroot(self, mockroot): + """ List both source and final rpm packages """ + srpmlist = [] + rpmlist = [] + for occurence in os.listdir(os.path.join(self.directory_of_rpm, mockroot)): + if occurence.endswith(".rpm"): + rpmlist.append(occurence) + for occurence in os.listdir(self.directory_of_srpms): + if occurence.endswith(".src.rpm"): + srpmlist.append(occurence) + return rpmlist, srpmlist + + def resolve_dependencies(self, mockroot): + return self.packagebuilder.run_builddep(self.spec.specfilefullpath, + self.directory_of_srpms, + self.directory_of_builder, + mockroot) + + def store_build_products(self, commonrepo): + """ Save build products under common yum repository """ + self.__create_directories([commonrepo]) + for mockroot in self.builders.roots: + srpmtargetdir = os.path.join(commonrepo, mockroot, 'srpm') + rpmtargetdir = os.path.join(commonrepo, mockroot, 'rpm') + self.__create_directories([srpmtargetdir, rpmtargetdir]) + (rpmlist, srpmlist) = self.list_buildproducts_for_mockroot(mockroot) + build_product_dir = os.path.join(self.directory_of_rpm, mockroot) + self.logger.debug("Hard linking %s rpm packages to %s", self.name, rpmtargetdir) + for rpm_file in rpmlist: + self.logger.info("Hard linking %s", rpm_file) + try: + os.link(os.path.join(build_product_dir, rpm_file), + os.path.join(rpmtargetdir, os.path.basename(rpm_file))) + except OSError: + pass + self.logger.debug("Hard linking %s srpm packages to %s", self.name, srpmtargetdir) + for srpm_file in srpmlist: + self.logger.info("Hard linking %s", srpm_file) + try: + os.link(os.path.join(self.directory_of_srpms, srpm_file), + os.path.join(srpmtargetdir, srpm_file)) + except OSError: + pass + + # Store info of latest build + self.store_project_status() + + + def who_buildrequires_me(self): + """ + Return a list of projects which directly buildrequires this project (non-recursive) + """ + downstream_projects = set() + # Loop through my packages + for package in self.spec.packages: + # Loop other projects and check if they need me + # To need me, they have my package in buildrequires + for project in self.projects: + if package in self.projects[project].spec.buildrequires: + self.logger.debug("Found dependency in {}: my package {} is required by project {}".format(self.name, package, project)) + self.projects[project].buildrequires_upstream.add(self.name) + self.projects[self.name].buildrequires_downstream.add(project) + downstream_projects.add(project) + return downstream_projects + + + def who_requires_me(self, recursive=False, depth=0): + """ + Return a list of projects which have requirement to this project + """ + if depth > 10: + self.logger.warn("Hit infinite recursion limiter in {}".format(self.name)) + recursive = False + # Loop through my packages + downstream_projects = set() + for package in self.spec.packages: + # Loop other projects and check if they need me + # To need me, they have my package in buildrequires or requires + for project in self.projects: + if package in self.projects[project].spec.buildrequires \ + or package in self.projects[project].spec.requires: + downstream_projects.add(project) + if recursive: + downstream_projects.update( + self.projects[project].who_requires_me(True, depth+1)) + self.logger.debug("Returning who_requires_me for %s: %s", + self.name, ', '.join(downstream_projects)) + return downstream_projects + + def get_project_changed(self): + raise NotImplementedError + + def store_project_status(self): + raise NotImplementedError + + def __create_directories(self, directories, verify_empty=False): + """ Directory creation """ + for directory in directories: + if os.path.isdir(directory): + if verify_empty and os.listdir(directory) != []: + self.logger.debug("Cleaning directory %s", directory) + globstring = directory + "/*" + files = glob.glob(globstring) + for foundfile in files: + self.logger.debug("Removing file %s", foundfile) + os.remove(foundfile) + else: + self.logger.debug("Creating directory %s", directory) + try: + os.makedirs(directory) + except OSError: + raise + return True + +class LocalMountProject(Project): + """ Projects coming from local disk mount """ + def __init__(self, name, directory, workspace, projects, builders, packagebuilder, masterargs, spec_path): + chrootscrub = masterargs.scrub + nosrpm = masterargs.nosrpm + forcebuild = masterargs.forcerebuild + + Prettyprint().print_heading("Initializing %s from disk" % name, 60) + super(LocalMountProject, self).__init__(name, workspace, projects, builders, packagebuilder) + + if not os.path.isdir(directory): + raise ProjectError("No directory %s found", directory) + + self.vcs = VersionControlSystem(directory) + self.directory_of_checkout = directory + + # Values from build configuration file + self.projconf = {} + # Read spec + if len(list(find_files(directory, r'\..+\.metadata$'))) > 0 and \ + os.path.isdir(os.path.join(directory, 'SOURCES')) and \ + os.path.isdir(os.path.join(directory, 'SPECS')): + self.centos_style = True + self.logger.debug('CentOS stype RPM detected') + self.spec = Specworker(os.path.dirname(spec_path), os.path.basename(spec_path)) + + self.gitversioned = False + try: + citag = self.vcs.get_citag() + self.gitversioned = True + except VcsError: + self.logger.debug("Project does not come from Git") + except: + raise + + if self.spec.version == '%{_version}': + if self.gitversioned: + self.logger.debug("Using Git describe for package version") + self.useversion = citag + else: + self.logger.debug("Project not from Git. Using a.b package version") + self.useversion = 'a.b' + else: + self.logger.debug("Using spec definition for package version") + self.useversion = self.spec.version + + self.packageversion = self.useversion + self.project_changed = self.get_project_changed() + self.nosrpm = nosrpm + + if forcebuild: + self.mark_for_rebuild() + + self.chrootscrub = chrootscrub + + def get_project_changed(self): + """ + Project status is read from status.txt file. Dirty git clones always require rebuild. + """ + statusfile = os.path.join(self.project_workspace, 'status.txt') + + if os.path.isfile(statusfile): + with open(statusfile, 'r') as filep: + previousprojectstatus = json.load(filep) + # Compare old values against new values + if not self.gitversioned: + self.logger.warning("Project %s is not git versioned. Forcing rebuild.", self.name) + return True + elif self.vcs.is_dirty(): + self.logger.warning("Project %s contains unversioned changes and is \"dirty\". Forcing rebuild.", self.name) + return True + elif previousprojectstatus['sha'] != self.vcs.commitsha: + self.logger.info("Project %s log has new hash. Rebuild needed.", self.name) + return True + else: + self.logger.info("Project %s has NO new changes.", self.name) + return False + else: + # No configuration means that project has not been compiled + self.logger.warning("No previous build found for %s. Building initial version.", self.name) + return True + + def store_project_status(self): + """ Write information of project version to status.txt + This can only be done for git versioned projects """ + if self.gitversioned: + # Save information of the last compilation + statusfile = os.path.join(self.project_workspace, 'status.txt') + self.logger.debug("Updating status file %s", statusfile) + + projectstatus = {"packageversion": self.packageversion, + "sha": self.vcs.commitsha, + "project": self.name} + + with open(statusfile, 'w') as outfile: + json.dump(projectstatus, outfile) + +class GitProject(Project): + """ Projects cloned from Git version control system """ + def __init__(self, name, workspace, conf, projects, builders, packagebuilder, masterargs): + forcebuild = masterargs.forcerebuild + chrootscrub = masterargs.scrub + + Prettyprint().print_heading("Initializing %s from Git" % name, 60) + super(GitProject, self).__init__(name, workspace, projects, builders, packagebuilder) + + # Values from build configuration file + self.projconf = {'url': conf.get_string(name, "url", mandatory=True), + 'ref': conf.get_string(name, "ref", mandatory=True), + 'spec': conf.get_string(name, "spec", mandatory=False, defaultvalue=None)} + + # Do version control updates + self.directory_of_checkout = os.path.join(self.project_workspace, + 'checkout') + self.vcs = VersionControlSystem(self.directory_of_checkout) + self.vcs.update_git_project(self.projconf["url"], self.projconf["ref"]) + self.useversion = self.vcs.get_citag() + + # Read spec + try: + self.spec = Specworker(self.directory_of_checkout, + self.projconf["spec"]) + except SpecError: + self.spec = Specworker(os.path.join(self.directory_of_checkout, "SPEC"), None) + self.centos_style = True + + # Define what version shall be used in spec file + if self.spec.version == '%{_version}': + self.packageversion = self.vcs.get_citag() + self.logger.debug("Taking package version from VCS") + else: + self.packageversion = self.spec.version + self.logger.debug("Taking package version from spec") + self.logger.debug("Package version: %s", self.packageversion) + + self.project_changed = self.get_project_changed() + if forcebuild: + self.mark_for_rebuild() + + self.chrootscrub = chrootscrub + + def get_project_changed(self): + """ + Check if there has been changes in the project + if project has not been compiled -> return = True + if project has GIT/VCS changes -> return = True + if project has not changed -> return = False + """ + statusfile = os.path.join(self.project_workspace, 'status.txt') + + if os.path.isfile(statusfile): + with open(statusfile, 'r') as filep: + previousprojectstatus = json.load(filep) + # Compare old values against new values + if previousprojectstatus['url'] != self.projconf["url"] \ + or previousprojectstatus['ref'] != self.projconf["ref"] \ + or previousprojectstatus['sha'] != self.vcs.commitsha: + self.logger.debug("Returning info that changes found") + return True + else: + self.logger.debug("Returning info of NO changes") + return False + else: + # No configuration means that project has not been compiled + self.logger.debug("Doing first build of this project") + return True + + def store_project_status(self): + """ Save information of the last compilation """ + statusfile = os.path.join(self.project_workspace, 'status.txt') + self.logger.debug("Updating status file %s", statusfile) + + projectstatus = {"url": self.projconf["url"], + "ref": self.projconf["ref"], + "spec": self.projconf["spec"], + "packageversion": self.packageversion, + "sha": self.vcs.commitsha, + "project": self.name} + + with open(statusfile, 'w') as outfile: + json.dump(projectstatus, outfile) + +class ProjectError(RpmbuilderError): + + """ Exceptions originating from Project """ + pass diff --git a/rpmbuilder/rpmtools.py b/rpmbuilder/rpmtools.py new file mode 100644 index 0000000..a9ab625 --- /dev/null +++ b/rpmbuilder/rpmtools.py @@ -0,0 +1,307 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""For handling rpm related work""" +import logging +import os +import re +import subprocess +from rpmUtils.miscutils import splitFilename + +from rpmbuilder.baseerror import RpmbuilderError +from rpmbuilder.executor import Executor + + +class Specworker(object): + """ Working with spec files """ + + def __init__(self, directory, specfile=None): + self.logger = logging.getLogger(__name__) + if specfile: + if self.__verify_specfile_exists(os.path.join(directory, specfile)): + self.specfilename = specfile + else: + self.logger.critical("Specfile %s not found", specfile) + raise SpecError("Spec file not found") + else: + self.specfilename = self.__locate_spec_file(directory) + + self.specfilefullpath = os.path.join(directory, self.specfilename) + + self.name = "" + self.version = "" + self.release = "" + self.source_files = [] + self.source_file_extension = None + self.patch_files = [] + self.buildrequires = [] + self.requires = [] + self.packages = [] + self.files = [] + self.spec_globals = {} + self.read_spec() + + def __str__(self): + return 'name:%s version:%s' % (self.name, self.version) + + def __getattr__(self, item): + return self.spec_globals.get(item) + + @staticmethod + def __locate_spec_file(directory): + """ Finding spec files from directory """ + logger = logging.getLogger(__name__) + logger.debug("Searching for spec files under: %s", directory) + specfile = '' + + for occurence in os.listdir(directory): + filefullpath = os.path.join(directory, occurence) + if os.path.isfile(filefullpath) and filefullpath.endswith(".spec"): + logger.info("Found spec file: %s", occurence) + if specfile: + logger.critical("Project has more than one spec files." + "I don't know which one to use.") + raise SpecError("Multiple spec files") + else: + specfile = occurence + if specfile: + return specfile + else: + raise SpecError("No spec file available") + + def _read_spec_sources(self): + cmd = ['spectool', '-n', '-S', self.specfilefullpath] + sources = self._parse_spectool_output(Executor().run(cmd)) + self.source_file_extension = self.__get_source_file_extension(sources[0]) + return sources + + def _read_spec_patches(self): + cmd = ['spectool', '-n', '-P', self.specfilefullpath] + return self._parse_spectool_output(Executor().run(cmd)) + + def _parse_spectool_output(self, output): + return [line.split(':', 1)[1].strip() for line in output.splitlines()] + + def _get_package_names(self): + cmd = ['rpm', '-q', '--qf', '%{NAME}\n', '--specfile', self.specfilefullpath] + return Executor().run(cmd).splitlines() + + def _get_version(self): + cmd = ['rpmspec', '-q', '--queryformat', '%{VERSION}\n', self.specfilefullpath] + return Executor().run(cmd).splitlines()[0] + + def read_spec(self): + """ Reading spec file values to variables """ + self.logger.debug("Reading spec file %s", self.specfilefullpath) + self.source_files = self._read_spec_sources() + self.patch_files = self._read_spec_patches() + self.packages = self._get_package_names() + self.name = self.packages[0] + self.version = self._get_version() + + with open(self.specfilefullpath, 'r') as filep: + name_found = False + for line in filep: + linestripped = line.strip() + + if linestripped.startswith("#") or not linestripped: + continue + + if linestripped.lower().startswith("%global"): + try: + var, val = re.match(r'^%global (\w+) (.+)$', linestripped).groups() + self.spec_globals[var] = val + except Exception as err: + logger = logging.getLogger(__name__) + logger.warning( + 'Failed to parse %global macro "{}" (error: {})'.format(linestripped, + str(err))) + + elif linestripped.lower().startswith("buildrequires:"): + self.buildrequires.extend(self.__get_value_from_line(linestripped)) + + elif linestripped.lower().startswith("requires:"): + self.requires.extend(self.__get_value_from_line(linestripped)) + + elif linestripped.lower().startswith("release:"): + templist = self.__get_value_from_line(linestripped) + self.release = templist[0] + + elif linestripped.lower().startswith("name:"): + name_found = True + + elif linestripped.lower().startswith("%package"): + if not name_found: + self.logger.error( + "SPEC file is faulty. Name of the package should be defined before defining subpackages") + raise SpecError( + "Problem in spec file. Subpackages defined before %packages") + + elif linestripped.lower().startswith("%files"): + if name_found: + templist = self.__get_package_names_from_line(self.name, linestripped) + self.files.extend(templist) + else: + self.logger.critical( + "SPEC file is faulty. Name of the package should be defined before defining subpackages") + raise SpecError("Problem in spec file. No %files defined") + + if not self.verify_spec_ok(): + raise SpecError("Inspect file %s" % self.specfilefullpath) + self.logger.info("Reading spec file done: %s", str(self)) + + def verify_spec_ok(self): + """ Check that spec file contains the necessary building blocks """ + spec_status = True + if not self.name: + self.logger.critical("Spec does not have name defined") + spec_status = False + if not self.version: + self.logger.critical("Spec does not contain version") + spec_status = False + if not self.release: + self.logger.critical("Spec does not contain release") + spec_status = False + if not self.source_file_extension: + self.logger.critical( + "Spec does not define source information with understandable archive method") + spec_status = False + return spec_status + + @staticmethod + def __get_source_file_extension(line): + """ Read source file archive file end """ + + if line.endswith('.tar.gz'): + return "tar.gz" + elif line.endswith('.tgz'): + return "tgz" + elif line.endswith('.tar'): + return "tar" + elif line.endswith('.tar.bz2'): + return "tar.bz2" + elif line.endswith('.tar.xz'): + return "tar.xz" + elif line.endswith('.zip'): + return "zip" + else: + raise SpecError( + "Unknown source archive format. Supported are: tar.gz, tgz, tar, tar.bz2, tar.xz, zip") + + @staticmethod + def __get_value_from_line(line): + """ Read spec line where values come after double-colon """ + valuelist = [] + linewithgroups = re.search('(.*):(.*)$', line) + linevalues = linewithgroups.group(2).strip().replace(' ', ',').split(',') + for linevalue in linevalues: + valuelist.append(linevalue.strip(' \t\n\r')) + return valuelist + + @staticmethod + def __get_package_names_from_line(name, line): + """ Read spec line where package names are defined """ + linewithgroups = re.search('%(.*) (.*)$', line) + if linewithgroups: + value = linewithgroups.group(2).strip(' \t\n\r') + return [name + '-' + value] + return [name] + + def __verify_specfile_exists(self, specfile): + """ Check that the given spec file exists """ + if not specfile.endswith(".spec"): + self.logger.error("Given specfile %s does not end with .spec prefix", specfile) + return False + + if os.path.isfile(specfile): + return True + self.logger.error("Could not locate specfile %s", specfile) + return False + + +class Repotool(object): + """ Module for handling rpm related functions """ + + def __init__(self): + self.logger = logging.getLogger(__name__) + + def createrepo(self, directory): + """ Create a yum repository of the given directory """ + createrepo_executable = "/usr/bin/createrepo" + createrepocommand = [createrepo_executable, '--update', directory] + outputfile = os.path.join(directory, 'log.txt') + with open(outputfile, 'w') as filep: + try: + subprocess.check_call(createrepocommand, shell=False, stdout=filep, + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + self.logger.critical("There was error running createrepo") + raise RepotoolError("There was error running createrepo") + except OSError: + self.logger.error(createrepo_executable + "command not available") + raise RepotoolError("No createrepo tool available") + + def latest_release_of_package(self, directory, package, version): + """ Return latest release of the given package """ + self.logger.debug("Looking for latest %s - %s under %s", + package, version, directory) + latest_found_release = 0 + if os.path.isdir(directory): + for occurence in os.listdir(directory): + filefullpath = os.path.join(directory, occurence) + if os.path.isfile(filefullpath) \ + and filefullpath.endswith(".rpm") \ + and not filefullpath.endswith(".src.rpm"): + (rpmname, rpmversion, rpmrelease, _, _) = splitFilename(occurence) + if rpmname == package and rpmversion == version: + self.logger.debug("Found rpm " + filefullpath) + if latest_found_release < rpmrelease: + self.logger.debug("Found rpm to match and to be the latest") + latest_found_release = rpmrelease + if latest_found_release == 0: + self.logger.debug("Did not find any previous releases of %s", package) + return str(latest_found_release) + + def next_release_of_package(self, directory, package, version, oldrelease): + """ Return next release of the given package """ + self.logger.debug("Looking for next release number for %s - %s under %s ", package, version, + directory) + + specreleasematch = re.search('^([0-9]+)(.*)$', oldrelease) + if specreleasematch and specreleasematch.group(2): + releasesuffix = specreleasematch.group(2) + else: + releasesuffix = '' + + latest_release = self.latest_release_of_package(directory, package, version) + self.logger.debug("Latest release of the package: " + latest_release) + rematches = re.search('^([0-9]+)(.*)$', latest_release) + if rematches.group(1).isdigit(): + nextrelease = str(int(rematches.group(1)) + 1) + releasesuffix + self.logger.debug("Next release of the package: " + nextrelease) + return nextrelease + else: + self.logger.critical("Could not parse release \"%s\" from package \"%s\"", + latest_release, package) + raise RepotoolError("Could not process release in rpm") + + +class RepotoolError(RpmbuilderError): + """ Exceptions originating from repotool """ + pass + + +class SpecError(RpmbuilderError): + """ Exceptions originating from spec content """ + pass diff --git a/rpmbuilder/utils.py b/rpmbuilder/utils.py new file mode 100755 index 0000000..09e0c53 --- /dev/null +++ b/rpmbuilder/utils.py @@ -0,0 +1,26 @@ +#! /usr/bin/python -tt +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import re + + +def find_files(path, pattern=None): + for root, folders, files in os.walk(path): + for filename in folders + files: + if pattern is not None: + if re.search(pattern, filename): + yield os.path.join(root, filename) + else: + yield os.path.join(root, filename) diff --git a/rpmbuilder/version_control.py b/rpmbuilder/version_control.py new file mode 100644 index 0000000..5928fc7 --- /dev/null +++ b/rpmbuilder/version_control.py @@ -0,0 +1,210 @@ +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Module for handling git repository clones +""" +import logging +import os +import re +import subprocess +from time import strftime, localtime + +from rpmbuilder.baseerror import RpmbuilderError + + +class VersionControlSystem(object): + """ Handling of project's repositories """ + + def __init__(self, clone_target_dir): + self.logger = logging.getLogger(__name__) + self.clone_target_dir = clone_target_dir + self.citag = None + self.commitsha = None + self.commitauth = None + self.commitepocdate = None + self.commitmessage = None + self.describe = None + try: + self.__store_head_state() + except VcsError: + pass + + def update_git_project(self, url, usergivenref): + """ Update of a single repository based on given reference """ + self.logger.info("%-18s: %s", "Git cloning from", url) + self.logger.info("%-18s: %s", "Git cloning to", self.clone_target_dir) + self.logger.info("%-18s: %s", "Git reference", usergivenref) + + # Check if we already have local clone of the repository + self.__clone_repo(url) + + # Change to user given ref value. + self.__update_head(url, usergivenref) + + self.__store_head_state() + self.citag = self.get_citag() + + def __clone_repo(self, url): + """ Create a clone from URL. If already exists, update it """ + if not os.path.isdir(self.clone_target_dir): + self.logger.debug("Creating a fresh clone") + cmd = ['git', 'clone', url, self.clone_target_dir] + self.logger.debug(self.__run_git(cmd)) + else: + self.logger.debug("We already have a clone. Using old clone.") + # Remove any possible garbage from clone directory + self.logger.debug("Running cleaning of existing repository") + cmd = ['git', 'reset', '--hard'] + self.logger.debug(self.__run_git(cmd, self.clone_target_dir)) + # Verify that correct remote is being used + self.__set_remoteurl(url) + # Run fetch twice. From Git 1.9 onwards this is not necessary, + # but to make sure of all server compatibility we do it twice + self.logger.debug("Fetching latest from remote") + cmd = ['git', 'fetch', 'origin'] + self.logger.debug(self.__run_git(cmd, self.clone_target_dir)) + cmd = ['git', 'fetch', 'origin', '--tags'] + self.logger.debug(self.__run_git(cmd, self.clone_target_dir)) + + def __update_head(self, url, usergivenref): + """ Change head to point to given ref. Ref can also be tag/commit """ + self.logger.debug("Reseting git head to %s", usergivenref) + try: + self.logger.debug("Checking out %s as reference", usergivenref) + cmd = ['git', 'checkout', '--force', '--detach', 'origin/' + usergivenref] + self.logger.debug(self.__run_git(cmd, self.clone_target_dir)) + except: + self.logger.debug("Unable to checkout %s as reference", usergivenref) + try: + self.logger.debug("Checking out %s as tag/commit", usergivenref) + cmd = ['git', 'checkout', '--force', '--detach', usergivenref] + self.logger.debug(self.__run_git(cmd, self.clone_target_dir)) + except GitError: + raise VcsError( + "Could not checkout branch/ref/commit \"%s\" from %s." % (usergivenref, url)) + + def __run_git(self, gitcmd, gitcwd=None): + """ Run given git command """ + assert gitcmd + self.logger.debug("Running \'%s\' under directory %s", " ".join(gitcmd), gitcwd) + try: + return subprocess.check_output(gitcmd, + shell=False, + cwd=gitcwd) + except subprocess.CalledProcessError as err: + raise GitError("Could not execute %s command. Return code was %d" % (err.cmd, + err.returncode)) + except: + raise + + def __set_remoteurl(self, url): + """ + Verify that repository is using the correct remote URL. If not + then it should be changed to the desired one. + """ + self.logger.info("Verifying we have correct remote repository configured") + cmd = ["git", "config", "--get", "remote.origin.url"] + existing_clone_url = self.__run_git(cmd, self.clone_target_dir).strip() + if existing_clone_url != url: + self.logger.info("Existing repo has url: %s", existing_clone_url) + self.logger.info("Changing repo url to: %s", url) + cmd = ["git", "remote", "set-url", "origin", url] + self.logger.debug(self.__run_git(cmd, self.clone_target_dir)) + + def __store_head_state(self): + """ Read checkout values to be used elsewhere """ + self.logger.info("State of the checkout:") + + try: + cmd = ["git", "log", "-1", "--pretty=%H"] + self.commitsha = self.__run_git(cmd, self.clone_target_dir).strip() + self.logger.info(" %-10s: %s", "SHA", self.commitsha) + + cmd = ["git", "log", "-1", "--pretty=%ae"] + self.commitauth = self.__run_git(cmd, self.clone_target_dir).strip() + self.logger.info(" %-10s: %s", "Author", self.commitauth) + + cmd = ["git", "log", "-1", "--pretty=%ct"] + self.commitepocdate = float(self.__run_git(cmd, self.clone_target_dir).strip()) + self.logger.info(" %-10s: %s", "Date:", + strftime("%a, %d %b %Y %H:%M:%S", + localtime(self.commitepocdate))) + + cmd = ["git", "log", "-1", "--pretty=%B"] + self.commitmessage = self.__run_git(cmd, self.clone_target_dir).strip() + self.logger.info(" %-10s: %s", "Message:", self.commitmessage.split('\n', 1)[0]) + except GitError: + raise VcsError("Directory \"%s\" does not come from vcs" % self.clone_target_dir) + + def is_dirty(self): + """ Check the status of directory. Return true if version control is dirty. + Git clone is dirty if status shows anything """ + cmd = ["git", "status", "--porcelain"] + return len(self.__run_git(cmd, self.clone_target_dir).strip()) > 0 + + def get_citag(self): + """ This is for creating the tag for the rpm. """ + + if self.citag: + return self.citag + + setup_py = os.path.join(self.clone_target_dir, 'setup.py') + if os.path.exists(setup_py): + with open(setup_py, 'r') as fpoint: + if re.search(r'^.*setup_requires=.*pbr.*$', fpoint.read(), re.MULTILINE): + cmd = ['python', 'setup.py', '--version'] + citag = self.__run_git(cmd, self.clone_target_dir).strip() + if ' ' in citag or '\n' in citag: + # 1st execution output may contains extra stuff such as locally installed eggs + citag = self.__run_git(cmd, self.clone_target_dir).strip() + return citag + + try: + cmd = ["git", "describe", "--dirty", "--tags"] + describe = self.__run_git(cmd, self.clone_target_dir).strip() + self.logger.debug("Git describe from tags: %s", describe) + if re.search("-", describe): + # if describe format is 2.3-3-g4324323, we need to modify it + dmatch = re.match('^(.*)-([0-9]+)-(g[a-f0-9]{7,}).*$', describe) + if dmatch: + citag = describe.replace('-', '-c', 1) + else: + raise Exception('no match, falling back to non-tagged describe') + else: + # if describe format is 2.3 + citag = describe + except: + try: + count = self.__run_git(["git", "rev-list", "HEAD", "--count"], + self.clone_target_dir).strip() + sha = self.__run_git(["git", "describe", "--long", "--always"], + self.clone_target_dir).strip() + citag = 'c{}.g{}'.format(count, sha) + except: + raise VcsError("Could not create a name for the package with git describe") + # Replace all remaining '-' characters with '.' from version number + if re.search("-", citag): + citag = re.sub('-', '.', citag) + return citag + + +class VcsError(RpmbuilderError): + """ Exceptions for all version control error situations """ + pass + + +class GitError(RpmbuilderError): + """ Exceptions for git command errors """ + pass diff --git a/stashmakebuild.py b/stashmakebuild.py new file mode 100755 index 0000000..39f2c7b --- /dev/null +++ b/stashmakebuild.py @@ -0,0 +1,158 @@ +#! /usr/bin/python -tt +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Safebuild is capable of doing backup and restore of workspace. +This ensures that package repository gets incremental updates and only +minimal set of packages are created """ + +import argparse +import logging +import os +import re +import subprocess +import tarfile + +from rpmbuilder.log import configure_logging +from makebuild import Build, BuildingError, ArgumentMakebuild +from stashworkspace import ArgumentRemote, Stasher + + +class Safebuild(Build): + + """ Safebuild extends capabilities of Build by providing backup and + restore on top of normal building activities """ + + def __init__(self, args): + super(Safebuild, self).__init__(args) + self.logger = logging.getLogger(__name__) + self.args = args + + self.backupfilename = "configuration.tar.gz" + self.remotehost = args.remotehost + self.remotedir = args.remotedir + + def start_safebuilding(self): + """ Starting a build requires checking of workspace, doing build + and then backing up the state of build system """ + self.logger.info("Starting safe building by using remote %s:%s", + self.remotehost, self.remotedir) + self.prepare_workspace() + self.update_building_blocks() + if self.start_building(): + self.backup_workspace() + if self.args.remotefunction == "pullpush": + stasher = Stasher() + stasher.push_workspace_to_remote(toserver=self.remotehost, + todirectory=self.remotedir, + workspace=self.args.workspace) + else: + self.logger.info("Skipping updating remote host with new packages") + + def tar_file_from_workspace(self, tar, sourcefile): + """ Archiving file from under workspace without + workspace parent directory structure """ + arcfile = os.path.join(self.args.workspace, sourcefile) + # Remove workspace directory from file + arcnamestring = re.sub(self.args.workspace, '', arcfile) + self.logger.debug("Archiving %s", arcfile) + tar.add(arcfile, arcname=arcnamestring) + + def backup_workspace(self): + """ Backup status files and repositories """ + backuptarfile = os.path.join(self.args.workspace, self.backupfilename) + self.logger.debug("Creating backup of configuration: %s", + backuptarfile) + with tarfile.open(backuptarfile, 'w:gz') as tar: + # Project settings + projdir = os.path.join(self.args.workspace, "projects") + for occurence in os.listdir(projdir): + statusfile = os.path.join(projdir, occurence, 'status.txt') + self.logger.info("Backing up file: %s", statusfile) + if os.path.isfile(statusfile): + self.tar_file_from_workspace(tar, statusfile) + else: + self.logger.warning("No %s for archiving", statusfile) + + def prepare_workspace(self): + """ Check that workspace contains correct beginning state """ + projectsdir = os.path.join(self.args.workspace, "projects") + if os.path.isdir(projectsdir): + self.logger.info("Using existing Workspace %s", self.args.workspace) + else: + self.logger.info("Trying to restore workspace from remote") + self.restore_workspace_from_remote(self.remotehost, self.remotedir) + + def restore_workspace_from_remote(self, fromserver, fromdirectory): + """ Retrieve and restore workspace from remote server """ + self.logger.info("Restoring workspace from remote %s:%s", fromserver, fromdirectory) + source = fromserver + ":" + fromdirectory + sshoptions = 'ssh -o stricthostkeychecking=no -o userknownhostsfile=/dev/null -o batchmode=yes -o passwordauthentication=no' + cmd = ["/usr/bin/rsync", + "--archive", + "-e", sshoptions, + os.path.join(source, "buildrepository"), + os.path.join(source, self.backupfilename), + os.path.join(self.args.workspace)] + self.logger.debug("Running: %s", str(cmd)) + try: + subprocess.check_call(cmd, shell=False, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as err: + if err.returncode == 23: + self.logger.info("There is no remote backup.. doing initial build") + return True + else: + raise BuildingError("Rsync from remote server failed with exit code %d" % err.returncode) + except: + raise BuildingError("Unexpected error") + + backupfile = os.path.join(self.args.workspace, self.backupfilename) + with tarfile.open(backupfile, 'r:gz') as tar: + tar.extractall(path=self.args.workspace) + self.logger.info("Workspace restored from %s:%s", + fromserver, + fromdirectory) + + +class ArgumentStashMakebuild(object): + """ Default arguments which are always needed """ + def __init__(self): + """ init """ + self.parser = argparse.ArgumentParser(description=''' + RPM building tool for continuous integration and development usage. + Uses remote host to retrieve and store incremental building state. + ''') + ArgumentMakebuild().set_arguments(self.parser) + ArgumentRemote().set_arguments(self.parser) + self.parser.add_argument("--remotefunction", + choices=["pull", "pullpush"], + default="pull", + help="With \"pullpush\" remote is used to fetch previous" + " build state and on succesful build remote is updated with" + " new packages. With \"pull\" packages are fetched but " + " remote is not updated on succesful builds. (Default: pull)") + + +def main(): + """ Read arguments and start processing build configuration """ + args = ArgumentStashMakebuild().parser.parse_args() + + debugfiletarget = os.path.join(args.workspace, 'debug.log') + configure_logging(args.verbose, debugfiletarget) + building = Safebuild(args) + building.start_safebuilding() + + +if __name__ == "__main__": + main() diff --git a/stashworkspace.py b/stashworkspace.py new file mode 100755 index 0000000..375e4ff --- /dev/null +++ b/stashworkspace.py @@ -0,0 +1,84 @@ +#! /usr/bin/python -tt +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import subprocess + +class Stasher(object): + def __init__(self, args=None): + self.args = args + self.backupfilename = "configuration.tar.gz" + + def start(self): + self.push_workspace_to_remote(toserver=self.args.remotehost, + todirectory=self.args.remotedir, + workspace=self.args.workspace) + + def push_workspace_to_remote(self, toserver, todirectory, workspace): + """ Move workspace backup to remote host """ + destination = toserver + ":" + todirectory + sshoptions = 'ssh -o stricthostkeychecking=no -o userknownhostsfile=/dev/null -o batchmode=yes -o passwordauthentication=no' + sourceconfiguration = os.path.join(workspace, self.backupfilename) + sourcerpm = os.path.join(workspace, "buildrepository") + rsyncpathval = "mkdir -p " + todirectory + " && rsync" + cmd = ["/usr/bin/rsync", + "--verbose", + "--archive", + "--rsync-path", rsyncpathval, + "-e", sshoptions, + sourceconfiguration, sourcerpm, + destination] + try: + print subprocess.check_output(cmd, shell=False, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as err: + raise StasherError("Following command retured code %d: %s" % (err.returncode, + ' '.join(err.cmd))) + +class StasherError(Exception): + """ Exceptions originating from builder """ + pass + + + +class ArgumentRemote(object): + """ Default arguments which are always needed """ + def __init__(self): + """ Create parser for arguments """ + self.parser = argparse.ArgumentParser(description='Workspace stasher copies workspace to remote host.') + self.set_arguments(self.parser) + self.parser.add_argument("--workspace", + help="Local (source) directory", + required=True) + + def set_arguments(self, parser): + """ Add extra arguments to parser """ + parser.add_argument("--remotehost", + help="Remote host where script will ssh/rsync to store build", + required=True) + parser.add_argument("--remotedir", + help="Remote directory to use for storing build", + required=True) + + +def main(): + """ Get arguments required for stashing local workspace """ + args = ArgumentRemote().parser.parse_args() + + stasher = Stasher(args) + stasher.start() + +if __name__ == "__main__": + main() diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..ee1ac21 --- /dev/null +++ b/tox.ini @@ -0,0 +1,35 @@ +[tox] +envlist = py27 +skipsdist=True + +[testenv] +setenv = + COVERAGE_FILE = .coverage-{envname} +passenv = COVERAGE_FILE +basepython = python2.7 + +commands = pytest -v \ + --basetemp={envtmpdir} \ + --pep8 \ + --cov \ + --cov-branch \ + --cov-report term \ + --cov-report html:htmlcov \ + --cov-config .coveragerc \ + {posargs:rpmbuilder/executor_test.py rpmbuilder/executor.py} + +deps=pytest + pytest-cov + pytest-flakes + pytest-pep8 + mock + +[pytest] +cache_dir = .pytest-cache +pep8maxlinelength = 100 +basepython = python2.7 + +[testenv:clean] +deps= +whitelist_externals = rm +commands = rm -rf .coverage-py27 .pytest-cache __pycache__ htmlcov