rpmbuilder/packagebuilding.py

   1 # Copyright 2019 Nokia
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 """ Module in charge of building a project """
  16 import glob
  17 import logging
  18 import os
  19 import pwd
  20 import shutil
  21 import subprocess
  22 from distutils.spawn import find_executable
  23
  24 import datetime
  25 from rpmbuilder.baseerror import RpmbuilderError
  26 from rpmbuilder.prettyprinter import Prettyprint
  27
  28 PIGZ_INSTALLED = False
  29 PBZIP2_INSTALLED = False
  30 PXZ_INSTALLED = False
  31
  32 class Packagebuilding(object):
  33
  34     """ Object for building rpm files with mock """
  35
  36     def __init__(self, masterargs):
  37         # Chroothousekeeping cleans chroot in case of mock errors. This should
  38         # keep /var/lib/mock from growing too much
  39         self.masterargs = masterargs
  40         self.logger = logging.getLogger(__name__)
  41         self.__check_tool_availability()
  42         self.chroot_installed_rpms = []
  43
  44         if find_executable("pigz"):
  45             global PIGZ_INSTALLED
  46             PIGZ_INSTALLED = True
  47             self.logger.debug("pigz is available")
  48         if find_executable("pbzip2"):
  49             global PBZIP2_INSTALLED
  50             PBZIP2_INSTALLED = True
  51             self.logger.debug("pbzip2 is available")
  52         if find_executable("pxz"):
  53             global PXZ_INSTALLED
  54             PXZ_INSTALLED = True
  55             self.logger.debug("pxz is available")
  56
  57     @staticmethod
  58     def __check_tool_availability():
  59         """ Verify that user belongs to mock group for things to work """
  60         username = pwd.getpwuid(os.getuid())[0]
  61         cmd = "id " + username + "| grep \\(mock\\) > /dev/null"
  62         if os.system(cmd) != 0:
  63             raise PackagebuildingError("Mock tool requires user to "
  64                                        "belong to group called mock")
  65         return True
  66
  67     def patch_specfile(self, origspecfile, outputdir, newversion, newrelease):
  68         """ Spec file is patched with version information from git describe """
  69         Prettyprint().print_heading("Patch spec", 50)
  70         self.logger.info("Patching new spec from %s", origspecfile)
  71         self.logger.debug(" - Version: %s", newversion)
  72         self.logger.debug(" - Release: %s", newrelease)
  73
  74         specfilebasename = os.path.basename(origspecfile)
  75         patchedspecfile = os.path.join(outputdir, specfilebasename)
  76         self.logger.debug("Writing new spec file to %s", patchedspecfile)
  77
  78         with open(origspecfile, 'r') as filepin:
  79             filepin_lines = filepin.readlines()
  80
  81         with open(patchedspecfile, 'w') as filepout:
  82             for line in filepin_lines:
  83                 linestripped = line.strip()
  84                 if not linestripped.startswith("#"):
  85                     # Check if version could be patched
  86                     if linestripped.lower().startswith("version:"):
  87                         filepout.write("Version: " + newversion + '\n')
  88                     elif linestripped.lower().startswith("release:"):
  89                         filepout.write("Release: " + newrelease + '\n')
  90                     else:
  91                         filepout.write(line)
  92         return patchedspecfile
  93
  94     def init_mock_chroot(self, resultdir, configdir, root):
  95         """
  96         Start a mock chroot where build requirements
  97         can be installed before building
  98         """
  99         Prettyprint().print_heading("Mock init in " + root, 50)
 100
 101         self.clean_directory(resultdir)
 102
 103         mock_arg_resultdir = "--resultdir=" + resultdir
 104
 105         mocklogfile = resultdir + '/mock-init-' + root + '.log'
 106
 107         arguments = [mock_arg_resultdir,
 108                      "--scrub=all"]
 109         self.run_mock_command(arguments, mocklogfile, configdir, root)
 110
 111         #Allow the builder to run sudo without terminal and without password
 112         #This makes it possible to run disk image builder needed by ipa-builder
 113         allow_sudo_str = "mockbuild ALL=(ALL) NOPASSWD: ALL"
 114         notty_str = "Defaults:mockbuild !requiretty"
 115         sudoers_file = "/etc/sudoers"
 116         command = "grep \'%s\' %s || echo -e \'%s\n%s\' >> %s" %(allow_sudo_str, sudoers_file, allow_sudo_str, notty_str, sudoers_file)
 117         arguments=["--chroot",
 118                    command ]
 119         self.run_mock_command(arguments, mocklogfile, configdir, root)
 120
 121         return True
 122
 123     def restore_local_repository(self, localdir, destdir, configdir, root, logfile):
 124         """
 125         Mock copying local yum repository to mock environment so that it can
 126         be used during building of other RPM packages.
 127         """
 128         Prettyprint().print_heading("Restoring local repository", 50)
 129         arguments = ["--copyin",
 130                      localdir,
 131                      destdir]
 132         self.run_mock_command(arguments, logfile, configdir, root)
 133
 134     def mock_source_rpm(self, hostsourcedir, specfile, resultdir, configdir, root):
 135         """ Mock SRPM file which can be used to build rpm """
 136         Prettyprint().print_heading("Mock source rpm in " + root, 50)
 137         self.logger.info("Build from:")
 138         self.logger.info(" - source directory %s", hostsourcedir)
 139         self.logger.info(" - spec %s", specfile)
 140
 141         self.clean_directory(resultdir)
 142
 143         mock_arg_resultdir = "--resultdir=" + resultdir
 144         mock_arg_spec = "--spec=" + specfile
 145         mock_arg_sources = "--sources=" + hostsourcedir
 146         arguments = [mock_arg_resultdir,
 147                      "--no-clean",
 148                      "--no-cleanup-after",
 149                      "--buildsrpm",
 150                      mock_arg_sources,
 151                      mock_arg_spec]
 152
 153         mocklogfile = resultdir + '/mock.log'
 154         self.run_mock_command(arguments, mocklogfile, configdir, root)
 155
 156         # Find source rpm and return the path
 157         globstring = resultdir + '/*.src.rpm'
 158         globmatches = glob.glob(globstring)
 159         assert len(globmatches) == 1, "Too many source rpm files"
 160
 161         return globmatches[0]
 162
 163     def mock_rpm(self, sourcerpm, resultdir, configdir, root):
 164         """ Mock RPM binary file from SRPM """
 165         Prettyprint().print_heading("Mock rpm in " + root, 50)
 166         self.logger.info("Building from:")
 167         self.logger.info(" - source rpm %s", sourcerpm)
 168
 169         self.clean_directory(resultdir)
 170
 171         mock_arg_resultdir = "--resultdir=" + resultdir
 172         arguments = [mock_arg_resultdir,
 173                      "--no-clean",
 174                      "--no-cleanup-after",
 175                      "--rebuild",
 176                      sourcerpm]
 177
 178         mocklogfile = resultdir + '/mock.log'
 179         self.run_mock_command(arguments, mocklogfile, configdir, root)
 180
 181         self.logger.debug("RPM files build to: %s", resultdir)
 182         return True
 183
 184     def mock_rpm_from_archive(self, source_tar_packages, resultdir, configdir, root):
 185         """ Mock rpm binary file straight from archive file """
 186         self.clean_directory(resultdir)
 187
 188         # Copy source archive to chroot
 189         chroot_sourcedir = "/builddir/build/SOURCES/"
 190         self.copy_to_chroot(configdir, root, resultdir, source_tar_packages, chroot_sourcedir)
 191
 192         # Create rpm from source archive
 193         sourcebasename = os.path.basename(source_tar_packages[0])
 194         chrootsourcefile = os.path.join(chroot_sourcedir, sourcebasename)
 195
 196         Prettyprint().print_heading("Mock rpm in " + root, 50)
 197         self.logger.info("Building from:")
 198         self.logger.info(" - source archive %s", chrootsourcefile)
 199
 200         mock_arg_resultdir = "--resultdir=" + resultdir
 201         rpmbuildcommand = "/usr/bin/rpmbuild --noclean -tb -v "
 202         rpmbuildcommand += os.path.join(chroot_sourcedir, chrootsourcefile)
 203         arguments = [mock_arg_resultdir,
 204                      "--chroot",
 205                      rpmbuildcommand]
 206         mocklogfile = resultdir + '/mock-rpmbuild.log'
 207         self.run_mock_command(arguments, mocklogfile, configdir, root)
 208
 209     def mock_rpm_from_filesystem(self, path, spec, resultdir, configdir, root, srpm_resultdir):
 210         """ Mock rpm binary file straight from archive file """
 211         self.clean_directory(resultdir)
 212         # Copy source archive to chroot
 213         chroot_sourcedir = "/builddir/build/"
 214         self.copy_to_chroot(configdir, root, resultdir, [os.path.join(path, 'SPECS', spec)], os.path.join(chroot_sourcedir, 'SPECS'))
 215         self.copy_to_chroot(configdir, root, resultdir, [os.path.join(path, 'SOURCES', f) for f in os.listdir(os.path.join(path, 'SOURCES'))], os.path.join(chroot_sourcedir, 'SOURCES'))
 216
 217         Prettyprint().print_heading("Mock rpm in " + root, 50)
 218         mocklogfile = resultdir + '/mock-rpmbuild.log'
 219         mock_arg_resultdir = "--resultdir=" + resultdir
 220         arguments = [mock_arg_resultdir,
 221                      "--chroot",
 222                      "chown -R root:root "+chroot_sourcedir]
 223         self.run_mock_command(arguments, mocklogfile, configdir, root)
 224         rpmbuildcommand = "/usr/bin/rpmbuild --noclean -ba -v "
 225         rpmbuildcommand += os.path.join(chroot_sourcedir, 'SPECS', spec)
 226         arguments = [mock_arg_resultdir,
 227                      "--chroot",
 228                      rpmbuildcommand]
 229         mocklogfile = resultdir + '/mock-rpmbuild.log'
 230         self.run_mock_command(arguments, mocklogfile, configdir, root)
 231
 232         arguments = ["--copyout",
 233                      "/builddir/build", resultdir+"/tmp/packages"]
 234         mocklogfile = resultdir + '/mock-copyout.log'
 235         self.run_mock_command(arguments, mocklogfile, configdir, root)
 236
 237         for filename in glob.glob(resultdir+"/tmp/packages/RPMS/*"):
 238             shutil.move(filename, resultdir)
 239
 240         for filename in glob.glob(resultdir+"/tmp/packages/SRPMS/*"):
 241             shutil.move(filename, srpm_resultdir)
 242
 243     def mock_wipe_buildroot(self, resultdir, configdir, root):
 244         """ Wipe buildroot clean """
 245         Prettyprint().print_heading("Wiping buildroot", 50)
 246         arguments = ["--chroot",
 247                      "mkdir -pv /usr/localrepo && " \
 248                      "cp -v /builddir/build/RPMS/*.rpm /usr/localrepo/. ;" \
 249                      "rm -rf /builddir/build/{BUILD,RPMS,SOURCES,SPECS,SRPMS}/*"]
 250         mocklogfile = resultdir + '/mock-wipe-buildroot.log'
 251         self.run_mock_command(arguments, mocklogfile, configdir, root)
 252
 253     def update_local_repository(self, configdir, root):
 254         Prettyprint().print_heading("Update repository " + root, 50)
 255
 256         arguments = ["--chroot",
 257                      "mkdir -pv /usr/localrepo && " \
 258                      "createrepo --update /usr/localrepo && yum clean expire-cache"]
 259         self.run_mock_command(arguments, configdir+"/log", configdir, root)
 260
 261     def copy_to_chroot(self, configdir, root, resultdir, source_files, destination):
 262         # Copy source archive to chroot
 263         Prettyprint().print_heading("Copy source archive to " + root, 50)
 264         self.logger.info(" - Copy from %s", source_files)
 265         self.logger.info(" - Copy to   %s", destination)
 266
 267         mock_arg_resultdir = "--resultdir=" + resultdir
 268         arguments = [mock_arg_resultdir,
 269                      "--copyin"]
 270         arguments.extend(source_files)
 271         arguments.append(destination)
 272
 273         mocklogfile = resultdir + '/mock-copyin.log'
 274         self.run_mock_command(arguments, mocklogfile, configdir, root)
 275
 276     def scrub_mock_chroot(self, configdir, root):
 277         time_start = datetime.datetime.now()
 278         Prettyprint().print_heading("Scrub mock chroot " + root, 50)
 279         mock_clean_command = ["/usr/bin/mock",
 280                               "--configdir=" + configdir,
 281                               "--root=" + root,
 282                               "--uniqueext=" + self.masterargs.uniqueext,
 283                               "--orphanskill",
 284                               "--scrub=chroot"]
 285         self.logger.info("Removing mock chroot.")
 286         self.logger.debug(" ".join(mock_clean_command))
 287         try:
 288             subprocess.check_call(mock_clean_command,
 289                                   shell=False,
 290                                   stderr=subprocess.STDOUT)
 291         except subprocess.CalledProcessError as err:
 292             raise PackagebuildingError("Mock chroot removal failed. Error code %s" % (err.returncode))
 293         time_delta = datetime.datetime.now() - time_start
 294         self.logger.debug('[mock-end] cmd="%s" took=%s (%s sec)', mock_clean_command, time_delta, time_delta.seconds)
 295
 296     def run_builddep(self, specfile, resultdir, configdir, root):
 297         arguments = ["--copyin"]
 298         arguments.append(specfile)
 299         arguments.append("/builddir/"+os.path.basename(specfile))
 300
 301         mocklogfile = resultdir + '/mock-builddep.log'
 302         self.run_mock_command(arguments, mocklogfile, configdir, root)
 303
 304         builddepcommand = "/usr/bin/yum-builddep -y "+"/builddir/"+os.path.basename(specfile)
 305         arguments = ["--chroot",
 306                      builddepcommand]
 307         mocklogfile = resultdir + '/mock-builddep.log'
 308         return self.run_mock_command(arguments, mocklogfile, configdir, root, True) == 0
 309
 310     def run_mock_command(self, arguments, outputfile, configdir, root, return_error=False):
 311         """ Mock binary rpm package """
 312         mock_command = ["/usr/bin/mock",
 313                         "--configdir=" + configdir,
 314                         "--root=" + root,
 315                         "--uniqueext=" + self.masterargs.uniqueext,
 316                         "--verbose",
 317                         "--old-chroot",
 318                         "--enable-network"]
 319         mock_command.extend(arguments)
 320         if self.masterargs.mockarguments:
 321             mock_command.extend([self.masterargs.mockarguments])
 322         self.logger.info("Running mock. Log goes to %s", outputfile)
 323         self.logger.debug('[mock-start] cmd="%s"', mock_command)
 324         time_start = datetime.datetime.now()
 325         self.logger.debug(" ".join(mock_command))
 326         with open(outputfile, 'a') as filep:
 327             try:
 328                 mockproc = subprocess.Popen(mock_command,
 329                                             shell=False,
 330                                             stdout=subprocess.PIPE,
 331                                             stderr=subprocess.STDOUT)
 332                 for line in iter(mockproc.stdout.readline, b''):
 333                     if self.masterargs.verbose:
 334                         self.logger.debug("mock-%s", line.rstrip('\n'))
 335                     filep.write(line)
 336                 _, stderr = mockproc.communicate()  # wait for the subprocess to exit
 337                 if return_error:
 338                     return mockproc.returncode
 339                 if mockproc.returncode != 0:
 340                     raise Mockcommanderror(returncode=mockproc.returncode)
 341             except Mockcommanderror as err:
 342                 self.logger.error("There was a failure during mocking")
 343                 if self.masterargs.scrub:
 344                     self.scrub_mock_chroot(configdir, root)
 345                     guidance_message = ""
 346                 else:
 347                     mock_shell_command = ["/usr/bin/mock",
 348                                           "--configdir=" + configdir,
 349                                           "--root=" + root,
 350                                           "--uniqueext=" + self.masterargs.uniqueext,
 351                                           "--shell"]
 352                     guidance_message = ". To open mock shell, run the following: " + " ".join(mock_shell_command)
 353                 raise PackagebuildingError("Mock exited with value \"%s\". "
 354                                            "Log for debuging: %s %s" % (err.returncode, outputfile, guidance_message))
 355             except OSError:
 356                 raise PackagebuildingError("Mock executable not found. "
 357                                            "Have you installed mock?")
 358             except:
 359                 raise
 360         time_delta = datetime.datetime.now() - time_start
 361         self.logger.debug('[mock-end] cmd="%s" took=%s (%s sec)', mock_command, time_delta, time_delta.seconds)
 362
 363     def clean_directory(self, directory):
 364         """ Make sure given directory exists and is clean """
 365         if os.path.isdir(directory):
 366             shutil.rmtree(directory)
 367         os.makedirs(directory)
 368
 369     def tar_filter(self, tarinfo):
 370         """ Filter git related and spec files away """
 371         if tarinfo.name.endswith('.spec') or tarinfo.name.endswith('.git'):
 372             self.logger.debug("Ignore %s", tarinfo.name)
 373             return None
 374         self.logger.debug("Archiving %s", tarinfo.name)
 375         return tarinfo
 376
 377     def create_source_archive(self,
 378                               package_name,
 379                               sourcedir,
 380                               outputdir,
 381                               project_changed,
 382                               archive_file_extension):
 383         """
 384         Create tar file. Example helloworld-2.4.tar.gz
 385         Tar file has naming <name>-<version>.tar.gz
 386         """
 387         Prettyprint().print_heading("Tar package creation", 50)
 388
 389         tar_file = package_name + '.' + 'tar'
 390         # Directory where tar should be stored.
 391         # Example /var/mybuild/workspace/sources
 392
 393         tarfilefullpath = os.path.join(outputdir, tar_file)
 394         if os.path.isfile(tarfilefullpath) and not project_changed:
 395             self.logger.info("Using cached %s", tarfilefullpath)
 396             return tarfilefullpath
 397
 398         self.logger.info("Creating tar file %s", tarfilefullpath)
 399         # sourcedir          = /var/mybuild/helloworld/checkout
 400         # sourcedir_dirname  = /var/mybuild/helloworld
 401         # sourcedir_basename =                         checkout
 402         sourcedir_dirname = os.path.dirname(sourcedir)
 403
 404         os.chdir(sourcedir_dirname)
 405
 406         tar_params = ["tar", "cf", tarfilefullpath, "--directory="+os.path.dirname(sourcedir)]
 407         tar_params = tar_params+["--exclude-vcs"]
 408         tar_params = tar_params+["--transform=s/" + os.path.basename(sourcedir) + "/" + os.path.join(package_name) + "/"]
 409         tar_params = tar_params+[os.path.basename(sourcedir)]
 410         self.logger.debug("Running: %s", " ".join(tar_params))
 411         ret = subprocess.call(tar_params)
 412         if ret > 0:
 413             raise PackagebuildingError("Tar error: %s", ret)
 414
 415         git_dir = os.path.join(os.path.basename(sourcedir), '.git')
 416         if os.path.exists(git_dir):
 417             tar_params = ["tar", "rf", tarfilefullpath, "--directory="+os.path.dirname(sourcedir)]
 418             tar_params += ["--transform=s/" + os.path.basename(sourcedir) + "/" + os.path.join(package_name) + "/"]
 419             tar_params += ['--dereference', git_dir]
 420             self.logger.debug("Running: %s", " ".join(tar_params))
 421             ret = subprocess.call(tar_params)
 422             if ret > 1:
 423                 self.logger.warning("Git dir tar failed")
 424
 425         if archive_file_extension == "tar.gz":
 426             if PIGZ_INSTALLED:
 427                 cmd = ['pigz', '-f']
 428             else:
 429                 cmd = ['gzip', '-f']
 430             resultfile = tarfilefullpath + '.gz'
 431         else:
 432             raise PackagebuildingError("Unknown source archive format: %s" % archive_file_extension)
 433         cmd += [tarfilefullpath]
 434         self.logger.debug("Running: %s", " ".join(cmd))
 435         ret = subprocess.call(cmd)
 436         if ret > 0:
 437             raise PackagebuildingError("Cmd error: %s", ret)
 438
 439         return resultfile
 440
 441 class Mockcommanderror(RpmbuilderError):
 442     def __init__(self, returncode):
 443         self.returncode = returncode
 444
 445 class PackagebuildingError(RpmbuilderError):
 446
 447     """ Exceptions originating from Builder and main level """
 448     pass