makebuild.py

   1 #! /usr/bin/python -tt
   2 # Copyright 2019 Nokia
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 """
  17 This module loops through user given configuration and creates
  18 projects based on that information. Projects are then build
  19 """
  20 import argparse
  21 import logging
  22 import os
  23 import platform
  24 import random
  25 import re
  26 import shutil
  27 import sys
  28
  29 from rpmbuilder.baseerror import RpmbuilderError
  30 from rpmbuilder.buildhistory import Buildhistory
  31 from rpmbuilder.configfile import Configfilereader
  32 from rpmbuilder.log import configure_logging
  33 from rpmbuilder.mockbuilder import GitMockbuilder, LocalMockbuilder
  34 from rpmbuilder.packagebuilding import Packagebuilding
  35 from rpmbuilder.project import GitProject, LocalMountProject
  36 from rpmbuilder.prettyprinter import Prettyprint
  37 from rpmbuilder.rpmtools import Repotool
  38 from rpmbuilder.utils import find_files
  39
  40
  41 class Build(object):
  42
  43     """
  44     Build configuration module which creates projects and does building
  45     """
  46
  47     def __init__(self, args):
  48         self.logger = logging.getLogger(__name__)
  49         self.workspace = os.path.abspath(args.workspace)
  50         if hasattr(args, 'buildconfig') and args.buildconfig:
  51             self.configuration = Configfilereader(os.path.abspath(args.buildconfig))
  52         self.builder = None
  53         self.projects = {}
  54         self.args = args
  55         self.packagebuilder = Packagebuilding(args)
  56
  57     def update_building_blocks(self):
  58         """ Update version control system components and project configuration """
  59         # Mock building tools
  60         Prettyprint().print_heading("Initialize builders", 80)
  61         default_conf_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'defaults/lcc-epel-7-x86_64.cfg')
  62         if hasattr(self.args, 'mockconf') and self.args.mockconf:
  63             self.logger.debug("Loading Mock builder from local disk")
  64             self.builder = LocalMockbuilder(self.args.mockconf)
  65         elif hasattr(self, 'configuration') and self.configuration:
  66             self.logger.debug("Loading Mock builder from Git")
  67             self.builder = GitMockbuilder(self.workspace, self.configuration)
  68             if self.builder.check_builder_changed():
  69                 self.args.forcerebuild = True
  70         elif os.path.isfile(default_conf_file):
  71             self.logger.debug("Loading default Mock configuration from %s file", default_conf_file)
  72             self.builder = LocalMockbuilder(default_conf_file)
  73         else:
  74             self.logger.critical("No Mock builder configured. Define one in build config file or provide it with -m option.")
  75             raise BuildingError("No Mock builder configured.")
  76
  77         # Projects outside of project configuration
  78         if hasattr(self.args, 'localproj') and self.args.localproj:
  79             self.update_local_mount_projects()
  80
  81         # Projects from build configuration file
  82         if hasattr(self, 'configuration') and self.configuration:
  83             self.update_configini_projects()
  84
  85         if not self.projects:
  86             raise BuildingError("No projects defined. Nothing to build.")
  87
  88     def update_local_mount_projects(self):
  89         """ Create project objects and initialize project configuration.
  90         Project has been defined as argument """
  91
  92         Prettyprint().print_heading("Initialize local projects", 80)
  93         for projectdir in self.args.localproj:
  94             if not os.path.isdir(projectdir):
  95                 raise BuildingError("Given \"%s\" is not a directory" % projectdir)
  96             project_specs = list(find_files(os.path.abspath(projectdir), r'.*\.spec$'))
  97             for spec in project_specs:
  98                 projectname = os.path.basename(projectdir.rstrip('/'))
  99                 if len(list(project_specs)) > 1:
 100                     projectname = projectname + '_' + os.path.splitext(os.path.basename(spec))[0]
 101                 self.projects[projectname] = LocalMountProject(projectname,
 102                                                            os.path.abspath(projectdir),
 103                                                            self.workspace,
 104                                                            self.projects,
 105                                                            self.builder,
 106                                                            self.packagebuilder,
 107                                                            self.args,
 108                                                            spec_path=spec)
 109
 110     def update_configini_projects(self):
 111         """ Create project objects and initialize project configuration.
 112         Project has been defined in configuration file """
 113         Prettyprint().print_heading("Initialize projects", 80)
 114         for section in self.configuration.get_sections():
 115             if self.configuration.get_string(section, "type") == "project" \
 116                     and self.configuration.get_bool(section, "enabled", defaultvalue=True):
 117                 if section in self.projects:
 118                     self.logger.warning("Local %s project already configured. Skipping build config entry", section)
 119                 else:
 120                     self.projects[section] = GitProject(section,
 121                                                         self.workspace,
 122                                                         self.configuration,
 123                                                         self.projects,
 124                                                         self.builder,
 125                                                         self.packagebuilder,
 126                                                         self.args)
 127
 128     def start_building(self):
 129         """ search for changes and start building """
 130         Prettyprint().print_heading("Summary of changes", 80)
 131         projects_to_build = self.get_projects_to_build()
 132         self.logger.debug("Final list of projects to build: %s",
 133                           str(projects_to_build))
 134
 135         Prettyprint().print_heading("Projects to build", 80)
 136         if projects_to_build:
 137             self.logger.info("%-30s %10s %10s", "Name", "Changed", "Rebuild")
 138             for project in projects_to_build:
 139                 req_by = ""
 140                 if self.projects[project].buildrequires_upstream:
 141                     req_by = "(build requires: {})".format(
 142                         ', '.join(self.projects[project].buildrequires_upstream))
 143                 self.logger.info("%-30s %10s %10s    %s",
 144                                  self.projects[project].name,
 145                                  self.projects[project].project_changed,
 146                                  self.projects[project].project_rebuild_needed,
 147                                  req_by)
 148
 149             Prettyprint().print_heading("Building projects", 80)
 150
 151             if self.mock_projects(projects_to_build):
 152                 self.logger.info("All built succesfully..")
 153                 Prettyprint().print_heading("Running final steps", 80)
 154                 self.finalize(projects_to_build)
 155
 156                 # Clean mock chroot
 157                 for mockroot in self.builder.roots:
 158                     if self.args.scrub:
 159                         self.packagebuilder.scrub_mock_chroot(self.builder.get_configdir(),
 160                                                               mockroot)
 161                 return True
 162             else:
 163                 self.logger.critical("Problems while building")
 164                 raise BuildingError("Error during rpm mock")
 165         else:
 166             self.logger.info("No projects to build.. no changes")
 167         return None
 168
 169     def get_projects_to_build(self):
 170         """ Find which project are not built yet """
 171         buildlist = []
 172         # Find projects that need to be build because of change
 173         for project in self.projects:
 174             if self.projects[project].project_changed \
 175                     or self.projects[project].project_rebuild_needed:
 176                 self.logger.info("Project \"%s\": Need to build", project)
 177                 buildlist.append(project)
 178             else:
 179                 self.logger.info("Project \"%s\": OK. Already built", project)
 180
 181         # Find projects that have list changed projects in buildrequires
 182         if buildlist:
 183             self.logger.debug("Projects %s need building.", str(buildlist))
 184             self.logger.debug("Looking for projects that need rebuild")
 185             projects_to_rebuild = []
 186             for project in buildlist:
 187                 self.logger.debug("Project \"%s\" need building.", project)
 188                 self.logger.debug("Checking if downstream requires rebuilding")
 189                 need_rebuild = \
 190                     self.projects[
 191                         project].mark_downstream_for_rebuild(set(buildlist))
 192                 self.logger.debug("Rebuild needed for: %s", str(need_rebuild))
 193                 projects_to_rebuild.extend(need_rebuild)
 194             buildlist.extend(projects_to_rebuild)
 195         buildlist = list(set(buildlist))
 196         random.shuffle(buildlist)
 197         return buildlist
 198
 199     def mock_projects(self, build_list):
 200         """ Loop through all mock chroots to build projects """
 201         for mockroot in self.builder.roots:
 202             Prettyprint().print_heading("Processing chroot " + mockroot, 70)
 203             if self.args.init:
 204                 # Create mock chroot for project building
 205                 self.packagebuilder.init_mock_chroot(os.path.join(self.workspace, "mocksettings", "logs"),
 206                                                      self.builder.get_configdir(),
 207                                                      mockroot)
 208             # Restore local yum repository to Mock environment
 209             hostyumrepository = os.path.join(self.workspace, "buildrepository", mockroot, "rpm")
 210             if os.path.isdir(os.path.join(hostyumrepository, "repodata")):
 211                 logfile = os.path.join(self.workspace, 'restore-mock-env-yum-repository.log')
 212                 self.packagebuilder.restore_local_repository(hostyumrepository,
 213                                                              "/usr/localrepo",
 214                                                              self.builder.get_configdir(),
 215                                                              mockroot,
 216                                                              logfile=logfile)
 217
 218             # Mock projects
 219             if not self.build_projects(build_list, mockroot):
 220                 return False
 221         return True
 222
 223     def upstream_packages_in_buildlist(self, project, buildlist):
 224         for proj in self.projects[project].buildrequires_upstream:
 225             if proj in buildlist:
 226                 return True
 227         return False
 228
 229     def build_projects(self, build_list, mockroot):
 230         """ Build listed projects """
 231         self.logger.debug("%s: Projects to build=%s",
 232                           mockroot,
 233                           str(build_list))
 234         self.packagebuilder.update_local_repository(self.builder.get_configdir(), mockroot)
 235         something_was_built = True
 236         while something_was_built:
 237             something_was_built = False
 238             not_built = []
 239             for project in build_list:
 240                 self.logger.debug("Trying to build: {}".format(project))
 241                 self.logger.debug("Build list: {}".format(build_list))
 242                 if not self.upstream_packages_in_buildlist(project, build_list):
 243                     self.projects[project].resolve_dependencies(mockroot)
 244                     self.logger.debug("OK to build {}".format(project))
 245                     self.projects[project].build_project(mockroot)
 246                     something_was_built = True
 247                     self.packagebuilder.update_local_repository(self.builder.get_configdir(), mockroot)
 248                 else:
 249                     self.logger.debug("Skipping {} because upstream is not built yet".format(project))
 250                     not_built.append(project)
 251             build_list = not_built
 252
 253         if build_list:
 254             self.logger.warning("Requirements not available for \"%s\"",
 255                                 ", ".join(build_list))
 256             return False
 257         return True
 258
 259     def finalize(self, projectlist):
 260         """ Do final work such as create yum repositories """
 261         commonrepo = os.path.join(self.workspace, 'buildrepository')
 262         self.logger.info("Hard linking rpm packages to %s", commonrepo)
 263         for project in projectlist:
 264             self.projects[project].store_build_products(commonrepo)
 265
 266         for mockroot in self.builder.roots:
 267             Repotool().createrepo(os.path.join(self.workspace,
 268                                                'buildrepository',
 269                                                mockroot,
 270                                                'rpm'))
 271             Repotool().createrepo(os.path.join(self.workspace,
 272                                                'buildrepository',
 273                                                mockroot,
 274                                                'srpm'))
 275         # Store information of used builder
 276         # Next run then knows what was used in previous build
 277         self.builder.store_builder_status()
 278
 279         buildhistory = Buildhistory()
 280         historyfile = os.path.join(commonrepo, "buildhistory")
 281         buildhistory.update_history(historyfile,
 282                                     projectlist,
 283                                     self.projects)
 284         return True
 285
 286     def rm_obsolete_projectdirs(self):
 287         """ Clean projects which are not listed in configuration """
 288         self.logger.debug("Cleaning unused project directories")
 289         projects_directory = os.path.join(self.workspace, 'projects')
 290         if not os.path.isdir(projects_directory):
 291             return True
 292         for subdir in os.listdir(projects_directory):
 293             fulldir = os.path.join(projects_directory, subdir)
 294             if subdir in self.projects:
 295                 self.logger.debug("Project directory %s is active",
 296                                   fulldir)
 297             else:
 298                 self.logger.debug("Removing directory %s. No match in projects",
 299                                   fulldir)
 300                 shutil.rmtree(fulldir)
 301         return True
 302
 303
 304 class BuildingError(RpmbuilderError):
 305     """ Exceptions originating from builder """
 306     pass
 307
 308
 309 def warn_if_incompatible_distro():
 310     if platform.linux_distribution()[0].lower() not in ['fedora', 'redhat', 'rhel', 'centos']:
 311         logger = logging.getLogger()
 312         logger.warning("Distribution compatibility check failed.\n"
 313                        "If you use other than Fedora, RedHat or CentOS based Linux distribution, you might experience problems\n"
 314                        "in case there are BuildRequirements between your own packages. For more information, read README.md")
 315
 316
 317 class ArgumentMakebuild(object):
 318     """ Default arguments which are always needed """
 319
 320     def __init__(self):
 321         """ init """
 322         self.parser = argparse.ArgumentParser(description='''
 323             RPM building tool for continuous integration and development usage.
 324         ''')
 325         self.set_arguments(self.parser)
 326
 327     def set_arguments(self, parser):
 328         """ Add relevant arguments """
 329         parser.add_argument("localproj",
 330                             metavar="dir",
 331                             help="Local project directory outside of buildconfig. This option can be used multiple times.",
 332                             nargs="*")
 333         parser.add_argument("-w",
 334                             "--workspace",
 335                             help="Sandbox directory for builder. Used to store repository clones and built rpm files. Required option.",
 336                             required=True)
 337 #        parser.add_argument("-b",
 338 #                            "--buildconfig",
 339 #                            help="Build configuration file lists projects and mock configuration. Required option.")
 340         parser.add_argument("-m",
 341                             "--mockconf",
 342                             help="Local Mock configuration file. Overrides mock settings from build configuration.")
 343         parser.add_argument("--mockarguments",
 344                             help="Arguments to be passed to mock. Check possible arguments from mock man pages")
 345         parser.add_argument("-v",
 346                             "--verbose",
 347                             help="Verbosed printing.",
 348                             action="store_true")
 349         parser.add_argument("-f",
 350                             "--forcerebuild",
 351                             help="Force rebuilding of all projects.",
 352                             action="store_true")
 353         parser.add_argument("--nowipe",
 354                             help="Skip cleaning of Mock chroot if build fails. "
 355                             "Old chroot can be used for debugging but if you use this option, then you need to clean unused chroot manually.",
 356                             action="store_false",
 357                             dest="scrub")
 358         parser.add_argument("--nosrpm",
 359                             help="Skip source rpm creation.",
 360                             action="store_true")
 361         parser.add_argument("--noinit",
 362                             help="Skip initialization (cleaning) of mock chroot.",
 363                             default=True,
 364                             action="store_false",
 365                             dest="init")
 366         parser.add_argument("--uniqueext",
 367                             help="Unique extension used for cache.",
 368                             default=str(os.getpid()),
 369                             dest="uniqueext")
 370
 371
 372 def main():
 373     """ Read arguments and start processing build configuration """
 374     args = ArgumentMakebuild().parser.parse_args()
 375
 376     debugfiletarget = os.path.join(args.workspace, 'debug.log')
 377     configure_logging(args.verbose, debugfiletarget)
 378
 379     warn_if_incompatible_distro()
 380
 381     # Start the build system
 382     try:
 383         build = Build(args)
 384         build.update_building_blocks()
 385         build.start_building()
 386     except RpmbuilderError as err:
 387         logger = logging.getLogger()
 388         logger.error("Could not produce a build. %s", err)
 389         warn_if_incompatible_distro()
 390         raise
 391
 392 if __name__ == "__main__":
 393     try:
 394         main()
 395     except RpmbuilderError:
 396         sys.exit(1)