makebuild.py

   1 #! /usr/bin/python -tt
   2 # Copyright 2019 Nokia
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 """
  17 This module loops through user given configuration and creates
  18 projects based on that information. Projects are then build
  19 """
  20 import argparse
  21 import logging
  22 import os
  23 import platform
  24 import re
  25 import shutil
  26 import sys
  27
  28 from rpmbuilder.baseerror import RpmbuilderError
  29 from rpmbuilder.buildhistory import Buildhistory
  30 from rpmbuilder.configfile import Configfilereader
  31 from rpmbuilder.log import configure_logging
  32 from rpmbuilder.mockbuilder import GitMockbuilder, LocalMockbuilder
  33 from rpmbuilder.packagebuilding import Packagebuilding
  34 from rpmbuilder.project import GitProject, LocalMountProject
  35 from rpmbuilder.prettyprinter import Prettyprint
  36 from rpmbuilder.rpmtools import Repotool
  37 from rpmbuilder.utils import find_files
  38
  39
  40 class Build(object):
  41
  42     """
  43     Build configuration module which creates projects and does building
  44     """
  45
  46     def __init__(self, args):
  47         self.logger = logging.getLogger(__name__)
  48         self.workspace = os.path.abspath(args.workspace)
  49         if hasattr(args, 'buildconfig') and args.buildconfig:
  50             self.configuration = Configfilereader(os.path.abspath(args.buildconfig))
  51         self.builder = None
  52         self.projects = {}
  53         self.args = args
  54         self.packagebuilder = Packagebuilding(args)
  55
  56     def update_building_blocks(self):
  57         """ Update version control system components and project configuration """
  58         # Mock building tools
  59         Prettyprint().print_heading("Initialize builders", 80)
  60         default_conf_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'defaults/lcc-epel-7-x86_64.cfg')
  61         if hasattr(self.args, 'mockconf') and self.args.mockconf:
  62             self.logger.debug("Loading Mock builder from local disk")
  63             self.builder = LocalMockbuilder(self.args.mockconf)
  64         elif hasattr(self, 'configuration') and self.configuration:
  65             self.logger.debug("Loading Mock builder from Git")
  66             self.builder = GitMockbuilder(self.workspace, self.configuration)
  67             if self.builder.check_builder_changed():
  68                 self.args.forcerebuild = True
  69         elif os.path.isfile(default_conf_file):
  70             self.logger.debug("Loading default Mock configuration from %s file", default_conf_file)
  71             self.builder = LocalMockbuilder(default_conf_file)
  72         else:
  73             self.logger.critical("No Mock builder configured. Define one in build config file or provide it with -m option.")
  74             raise BuildingError("No Mock builder configured.")
  75
  76         # Projects outside of project configuration
  77         if hasattr(self.args, 'localproj') and self.args.localproj:
  78             self.update_local_mount_projects()
  79
  80         # Projects from build configuration file
  81         if hasattr(self, 'configuration') and self.configuration:
  82             self.update_configini_projects()
  83
  84         if not self.projects:
  85             raise BuildingError("No projects defined. Nothing to build.")
  86
  87     def update_local_mount_projects(self):
  88         """ Create project objects and initialize project configuration.
  89         Project has been defined as argument """
  90
  91         Prettyprint().print_heading("Initialize local projects", 80)
  92         for projectdir in self.args.localproj:
  93             if not os.path.isdir(projectdir):
  94                 raise BuildingError("Given \"%s\" is not a directory" % projectdir)
  95             project_specs = list(find_files(os.path.abspath(projectdir), r'.*\.spec$'))
  96             for spec in project_specs:
  97                 projectname = os.path.basename(projectdir.rstrip('/'))
  98                 if len(list(project_specs)) > 1:
  99                     projectname = projectname + '_' + os.path.splitext(os.path.basename(spec))[0]
 100                 self.projects[projectname] = LocalMountProject(projectname,
 101                                                            os.path.abspath(projectdir),
 102                                                            self.workspace,
 103                                                            self.projects,
 104                                                            self.builder,
 105                                                            self.packagebuilder,
 106                                                            self.args,
 107                                                            spec_path=spec)
 108
 109     def update_configini_projects(self):
 110         """ Create project objects and initialize project configuration.
 111         Project has been defined in configuration file """
 112         Prettyprint().print_heading("Initialize projects", 80)
 113         for section in self.configuration.get_sections():
 114             if self.configuration.get_string(section, "type") == "project" \
 115                     and self.configuration.get_bool(section, "enabled", defaultvalue=True):
 116                 if section in self.projects:
 117                     self.logger.warning("Local %s project already configured. Skipping build config entry", section)
 118                 else:
 119                     self.projects[section] = GitProject(section,
 120                                                         self.workspace,
 121                                                         self.configuration,
 122                                                         self.projects,
 123                                                         self.builder,
 124                                                         self.packagebuilder,
 125                                                         self.args)
 126
 127     def start_building(self):
 128         """ search for changes and start building """
 129         Prettyprint().print_heading("Summary of changes", 80)
 130         projects_to_build = self.get_projects_to_build()
 131         self.logger.debug("Final list of projects to build: %s",
 132                           str(projects_to_build))
 133
 134         Prettyprint().print_heading("Projects to build", 80)
 135         if projects_to_build:
 136             self.logger.info("%-30s %10s %10s", "Name", "Changed", "Rebuild")
 137             for project in projects_to_build:
 138                 req_by = ""
 139                 if self.projects[project].buildrequires_upstream:
 140                     req_by = "(build requires: {})".format(
 141                         ', '.join(self.projects[project].buildrequires_upstream))
 142                 self.logger.info("%-30s %10s %10s    %s",
 143                                  self.projects[project].name,
 144                                  self.projects[project].project_changed,
 145                                  self.projects[project].project_rebuild_needed,
 146                                  req_by)
 147
 148             Prettyprint().print_heading("Building projects", 80)
 149
 150             if self.mock_projects(projects_to_build):
 151                 self.logger.info("All built succesfully..")
 152                 Prettyprint().print_heading("Running final steps", 80)
 153                 self.finalize(projects_to_build)
 154
 155                 # Clean mock chroot
 156                 for mockroot in self.builder.roots:
 157                     if self.args.scrub:
 158                         self.packagebuilder.scrub_mock_chroot(self.builder.get_configdir(),
 159                                                               mockroot)
 160                 return True
 161             else:
 162                 self.logger.critical("Problems while building")
 163                 raise BuildingError("Error during rpm mock")
 164         else:
 165             self.logger.info("No projects to build.. no changes")
 166         return None
 167
 168     def get_projects_to_build(self):
 169         """ Find which project are not built yet """
 170         buildlist = []
 171         # Find projects that need to be build because of change
 172         for project in self.projects:
 173             if self.projects[project].project_changed \
 174                     or self.projects[project].project_rebuild_needed:
 175                 self.logger.info("Project \"%s\": Need to build", project)
 176                 buildlist.append(project)
 177             else:
 178                 self.logger.info("Project \"%s\": OK. Already built", project)
 179
 180         # Find projects that have list changed projects in buildrequires
 181         if buildlist:
 182             self.logger.debug("Projects %s need building.", str(buildlist))
 183             self.logger.debug("Looking for projects that need rebuild")
 184             projects_to_rebuild = []
 185             for project in buildlist:
 186                 self.logger.debug("Project \"%s\" need building.", project)
 187                 self.logger.debug("Checking if downstream requires rebuilding")
 188                 need_rebuild = \
 189                     self.projects[
 190                         project].mark_downstream_for_rebuild(set(buildlist))
 191                 self.logger.debug("Rebuild needed for: %s", str(need_rebuild))
 192                 projects_to_rebuild.extend(need_rebuild)
 193             buildlist.extend(projects_to_rebuild)
 194         buildlist = list(set(buildlist))
 195         buildlist.sort()
 196         return buildlist
 197
 198     def mock_projects(self, build_list):
 199         """ Loop through all mock chroots to build projects """
 200         for mockroot in self.builder.roots:
 201             Prettyprint().print_heading("Processing chroot " + mockroot, 70)
 202             if self.args.init:
 203                 # Create mock chroot for project building
 204                 self.packagebuilder.init_mock_chroot(os.path.join(self.workspace, "mocksettings", "logs"),
 205                                                      self.builder.get_configdir(),
 206                                                      mockroot)
 207             # Restore local yum repository to Mock environment
 208             hostyumrepository = os.path.join(self.workspace, "buildrepository", mockroot, "rpm")
 209             if os.path.isdir(os.path.join(hostyumrepository, "repodata")):
 210                 logfile = os.path.join(self.workspace, 'restore-mock-env-yum-repository.log')
 211                 self.packagebuilder.restore_local_repository(hostyumrepository,
 212                                                              "/usr/localrepo",
 213                                                              self.builder.get_configdir(),
 214                                                              mockroot,
 215                                                              logfile=logfile)
 216
 217             # Mock projects
 218             if not self.build_projects(build_list, mockroot):
 219                 return False
 220         return True
 221
 222     def upstream_packages_in_buildlist(self, project, buildlist):
 223         for proj in self.projects[project].buildrequires_upstream:
 224             if proj in buildlist:
 225                 return True
 226         return False
 227
 228     def build_projects(self, build_list, mockroot):
 229         """ Build listed projects """
 230         self.logger.debug("%s: Projects to build=%s",
 231                           mockroot,
 232                           str(build_list))
 233         self.packagebuilder.update_local_repository(self.builder.get_configdir(), mockroot)
 234         something_was_built = True
 235         while something_was_built:
 236             something_was_built = False
 237             not_built = []
 238             for project in build_list:
 239                 self.logger.debug("Trying to build: {}".format(project))
 240                 self.logger.debug("Build list: {}".format(build_list))
 241                 if not self.upstream_packages_in_buildlist(project, build_list):
 242                     if not self.projects[project].resolve_dependencies(mockroot):
 243                         self.logger.info("still unresolved dependencies: {}".format(project))
 244                         not_built.append(project)
 245                     else:
 246                         self.logger.debug("OK to build {}".format(project))
 247                         self.projects[project].build_project(mockroot)
 248                         something_was_built = True
 249                         self.packagebuilder.update_local_repository(self.builder.get_configdir(), mockroot)
 250                 else:
 251                     self.logger.debug("Skipping {} because upstream is not built yet".format(project))
 252                     not_built.append(project)
 253             build_list = not_built
 254
 255         if build_list:
 256             self.logger.warning("Requirements not available for \"%s\"",
 257                                 ", ".join(build_list))
 258             return False
 259         return True
 260
 261     def finalize(self, projectlist):
 262         """ Do final work such as create yum repositories """
 263         commonrepo = os.path.join(self.workspace, 'buildrepository')
 264         self.logger.info("Hard linking rpm packages to %s", commonrepo)
 265         for project in projectlist:
 266             self.projects[project].store_build_products(commonrepo)
 267
 268         for mockroot in self.builder.roots:
 269             Repotool().createrepo(os.path.join(self.workspace,
 270                                                'buildrepository',
 271                                                mockroot,
 272                                                'rpm'))
 273             Repotool().createrepo(os.path.join(self.workspace,
 274                                                'buildrepository',
 275                                                mockroot,
 276                                                'srpm'))
 277         # Store information of used builder
 278         # Next run then knows what was used in previous build
 279         self.builder.store_builder_status()
 280
 281         buildhistory = Buildhistory()
 282         historyfile = os.path.join(commonrepo, "buildhistory")
 283         buildhistory.update_history(historyfile,
 284                                     projectlist,
 285                                     self.projects)
 286         return True
 287
 288     def rm_obsolete_projectdirs(self):
 289         """ Clean projects which are not listed in configuration """
 290         self.logger.debug("Cleaning unused project directories")
 291         projects_directory = os.path.join(self.workspace, 'projects')
 292         if not os.path.isdir(projects_directory):
 293             return True
 294         for subdir in os.listdir(projects_directory):
 295             fulldir = os.path.join(projects_directory, subdir)
 296             if subdir in self.projects:
 297                 self.logger.debug("Project directory %s is active",
 298                                   fulldir)
 299             else:
 300                 self.logger.debug("Removing directory %s. No match in projects",
 301                                   fulldir)
 302                 shutil.rmtree(fulldir)
 303         return True
 304
 305
 306 class BuildingError(RpmbuilderError):
 307     """ Exceptions originating from builder """
 308     pass
 309
 310
 311 def warn_if_incompatible_distro():
 312     if platform.linux_distribution()[0].lower() not in ['fedora', 'redhat', 'rhel', 'centos']:
 313         logger = logging.getLogger()
 314         logger.warning("Distribution compatibility check failed.\n"
 315                        "If you use other than Fedora, RedHat or CentOS based Linux distribution, you might experience problems\n"
 316                        "in case there are BuildRequirements between your own packages. For more information, read README.md")
 317
 318
 319 class ArgumentMakebuild(object):
 320     """ Default arguments which are always needed """
 321
 322     def __init__(self):
 323         """ init """
 324         self.parser = argparse.ArgumentParser(description='''
 325             RPM building tool for continuous integration and development usage.
 326         ''')
 327         self.set_arguments(self.parser)
 328
 329     def set_arguments(self, parser):
 330         """ Add relevant arguments """
 331         parser.add_argument("localproj",
 332                             metavar="dir",
 333                             help="Local project directory outside of buildconfig. This option can be used multiple times.",
 334                             nargs="*")
 335         parser.add_argument("-w",
 336                             "--workspace",
 337                             help="Sandbox directory for builder. Used to store repository clones and built rpm files. Required option.",
 338                             required=True)
 339 #        parser.add_argument("-b",
 340 #                            "--buildconfig",
 341 #                            help="Build configuration file lists projects and mock configuration. Required option.")
 342         parser.add_argument("-m",
 343                             "--mockconf",
 344                             help="Local Mock configuration file. Overrides mock settings from build configuration.")
 345         parser.add_argument("--mockarguments",
 346                             help="Arguments to be passed to mock. Check possible arguments from mock man pages")
 347         parser.add_argument("-v",
 348                             "--verbose",
 349                             help="Verbosed printing.",
 350                             action="store_true")
 351         parser.add_argument("-f",
 352                             "--forcerebuild",
 353                             help="Force rebuilding of all projects.",
 354                             action="store_true")
 355         parser.add_argument("--nowipe",
 356                             help="Skip cleaning of Mock chroot if build fails. "
 357                             "Old chroot can be used for debugging but if you use this option, then you need to clean unused chroot manually.",
 358                             action="store_false",
 359                             dest="scrub")
 360         parser.add_argument("--nosrpm",
 361                             help="Skip source rpm creation.",
 362                             action="store_true")
 363         parser.add_argument("--noinit",
 364                             help="Skip initialization (cleaning) of mock chroot.",
 365                             default=True,
 366                             action="store_false",
 367                             dest="init")
 368         parser.add_argument("--uniqueext",
 369                             help="Unique extension used for cache.",
 370                             default=str(os.getpid()),
 371                             dest="uniqueext")
 372
 373
 374 def main():
 375     """ Read arguments and start processing build configuration """
 376     args = ArgumentMakebuild().parser.parse_args()
 377
 378     debugfiletarget = os.path.join(args.workspace, 'debug.log')
 379     configure_logging(args.verbose, debugfiletarget)
 380
 381     warn_if_incompatible_distro()
 382
 383     # Start the build system
 384     try:
 385         build = Build(args)
 386         build.update_building_blocks()
 387         build.start_building()
 388     except RpmbuilderError as err:
 389         logger = logging.getLogger()
 390         logger.error("Could not produce a build. %s", err)
 391         warn_if_incompatible_distro()
 392         raise
 393
 394 if __name__ == "__main__":
 395     try:
 396         main()
 397     except RpmbuilderError:
 398         sys.exit(1)