rpmbuilder/project.py

   1 # Copyright 2019 Nokia
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 """
  16 Project is a subsystem which contains one spec file which
  17 defines how it is build. Every project has one git
  18 repository from where it is cloned from.
  19 """
  20 import glob
  21 import json
  22 import logging
  23 import os
  24 import shutil
  25 import subprocess
  26
  27 import re
  28
  29 import datetime
  30 from rpmbuilder.baseerror import RpmbuilderError
  31 from rpmbuilder.prettyprinter import Prettyprint
  32 from rpmbuilder.rpmtools import Repotool, Specworker, RepotoolError, SpecError
  33 from rpmbuilder.utils import find_files
  34 from rpmbuilder.version_control import VersionControlSystem, VcsError
  35 from rpmbuilder.get_sources import get_sources
  36
  37
  38 class Project(object):
  39
  40     """ Instance of a project """
  41
  42     def __init__(self, name, workspace, projects, builders, packagebuilder, chrootscrub=True, nosrpm=False):
  43         self.name = name
  44
  45         self.logger = logging.getLogger(__name__ + "." + self.name)
  46
  47         self.project_rebuild_needed = False
  48
  49         self.project_workspace = os.path.join(workspace,
  50                                               'projects',
  51                                               self.name)
  52
  53         self.projects = projects
  54         self.builders = builders
  55         self.directory_of_specpatch = os.path.join(self.project_workspace,
  56                                                    'rpmbuild',
  57                                                    'spec')
  58         self.directory_of_sourcepackage = os.path.join(self.project_workspace,
  59                                                        'rpmbuild',
  60                                                        'sources')
  61         self.directory_of_srpms = os.path.join(self.project_workspace,
  62                                                'rpmbuild',
  63                                                'srpm')
  64         self.directory_of_rpm = os.path.join(self.project_workspace,
  65                                              'rpmbuild',
  66                                              'rpm')
  67         self.directory_of_commonrepo = os.path.join(workspace,
  68                                                     'buildrepository')
  69
  70         self.directory_of_builder = self.builders.get_configdir()
  71
  72         self.__create_directories([self.directory_of_specpatch,
  73                                    self.directory_of_srpms],
  74                                   verify_empty=True)
  75         self.__create_directories([self.directory_of_sourcepackage],
  76                                   verify_empty=False)
  77
  78         self.packagebuilder = packagebuilder
  79
  80         self.chrootscrub = chrootscrub
  81         self.built = {}
  82         for mockroot in builders.roots:
  83             self.built[mockroot] = False
  84
  85         self.project_changed = False
  86         self.projconf = None
  87         self.spec = None
  88         self.useversion = None
  89         self.directory_of_checkout = None
  90         self.nosrpm = nosrpm
  91         self.centos_style = False
  92         self.buildrequires_downstream = set()
  93         self.buildrequires_upstream = set()
  94
  95     def mark_for_rebuild(self):
  96         """ Marking project for rebuild only if project has not changed """
  97         if not self.project_changed:
  98             self.logger.debug("Marking project %s for rebuild.", self.name)
  99             self.project_rebuild_needed = True
 100
 101     def mark_downstream_for_rebuild(self, marked_for_build=None):
 102         """
 103         Recursively mark downstream projects for rebuilding.
 104         Return set of projects marked for rebuild
 105         """
 106         if marked_for_build is None:
 107             marked_for_build = set()
 108         self.logger.debug("Marking downstream for rebuild in \"%s\"",
 109             self.name)
 110         for project in self.who_buildrequires_me():
 111             self.logger.debug("BuildRequires to \"%s\" found in \"%s\"",
 112                 self.name, project)
 113             if project in marked_for_build:
 114                 self.logger.debug("\"%s\" already marked for build", project)
 115             elif self.projects[project].project_rebuild_needed:
 116                 self.logger.debug("\"%s\" already marked for rebuild", project)
 117             else:
 118                 self.projects[project].mark_for_rebuild()
 119                 marked_for_build.add(project)
 120                 # Check if downstream has downstream projects
 121                 tmpset = self.projects[project].mark_downstream_for_rebuild(
 122                     marked_for_build)
 123                 marked_for_build.update(tmpset)
 124         return marked_for_build
 125
 126     def build_project(self, mockroot):
 127         """ Do building of SRPM and RPM files """
 128         time_start = datetime.datetime.now()
 129         Prettyprint().print_heading("Build " + self.name, 60)
 130         assert not self.built[mockroot], "Project already built"
 131
 132         # Produce spec file
 133         if self.spec.version == '%{_version}':
 134             self.logger.debug("patching spec file")
 135             self.logger.debug("Version in spec is going to be %s", self.useversion)
 136
 137             rpm = Repotool()
 138             userelease = rpm.next_release_of_package(
 139                 os.path.join(self.directory_of_commonrepo,
 140                              self.builders.roots[0],
 141                              "rpm"),
 142                 self.spec.name,
 143                 self.useversion,
 144                 self.spec.release)
 145             self.logger.debug("Release in spec is going to be %s", userelease)
 146
 147             specfile = self.packagebuilder.patch_specfile(self.spec.specfilefullpath,
 148                                                           self.directory_of_specpatch,
 149                                                           self.useversion,
 150                                                           userelease)
 151         else:
 152             self.logger.debug("Skipping spec patching")
 153             specfile = self.spec.specfilefullpath
 154
 155         # Start mocking
 156         self.logger.debug("Starting building in root \"%s\"", mockroot)
 157         if self.centos_style:
 158             shutil.rmtree(self.directory_of_sourcepackage)
 159             ignore_git = shutil.ignore_patterns('.git')
 160             shutil.copytree(self.directory_of_checkout, self.directory_of_sourcepackage, ignore=ignore_git)
 161             sources_key = 'CENTOS_SOURCES'
 162             if sources_key not in os.environ:
 163                 raise RpmbuilderError('Cannot build CentOS style RPM, %s not defined in the environment' % sources_key)
 164             get_sources(self.directory_of_sourcepackage, os.environ[sources_key].split(','), self.logger)
 165             self.create_rpm_from_filesystem(self.directory_of_sourcepackage, mockroot)
 166         elif self.nosrpm:
 167             list_of_source_packages = self.get_source_package()
 168             self.create_rpm_from_archive(list_of_source_packages, mockroot)
 169         else:
 170             self.get_source_package()
 171             # Create source RPM file
 172             sourcerpm = self.get_source_rpm(self.directory_of_sourcepackage, specfile, mockroot)
 173
 174             # Create final RPM file(s)
 175             self.create_rpm_from_srpm(sourcerpm, mockroot)
 176
 177         # Mark build completed
 178         self.built[mockroot] = True
 179         time_delta = datetime.datetime.now() - time_start
 180         self.logger.info('Building success: %s (took %s [%s sec])', self.name, time_delta, time_delta.seconds)
 181
 182         # We wipe buildroot of previously built rpm, source etc. packages
 183         # This is custom cleaning which does not remove chroot
 184         self.packagebuilder.mock_wipe_buildroot(self.project_workspace, self.directory_of_builder, mockroot)
 185
 186     def pull_source_packages(self, target_dir):
 187         cmd = ['/usr/bin/spectool', '-d', 'KVERSION a.b', '-g', '--directory', target_dir, self.spec.specfilefullpath]
 188         self.logger.info('Pulling source packages: %s', cmd)
 189         try:
 190             subprocess.check_call(cmd, shell=False)
 191             self.logger.info('Pulling source packages ok')
 192         except OSError as err:
 193             self.logger.info('Pulling source packages nok %s', err.strerror)
 194             raise RepotoolError("Calling of command spectool caused: \"%s\"" % err.strerror)
 195         except:
 196             self.logger.info('Pulling source packages nok ??', err.strerror)
 197             raise RepotoolError("There was error pulling source content")
 198
 199     def get_source_package(self):
 200         # Produce source package
 201         source_package_list = []
 202         for source_file_hit in self.spec.source_files:
 203             self.logger.info("Acquiring source file \"%s\"", source_file_hit)
 204             if re.match(r'^(http[s]*|ftp)://', source_file_hit):
 205                 self.logger.info("PULL %s", self.directory_of_sourcepackage)
 206                 self.pull_source_packages(self.directory_of_sourcepackage)
 207                 source_package_list.append(self.directory_of_sourcepackage + '/' + source_file_hit.split('/')[-1])
 208                 continue
 209             for subdir in ["", "SOURCES"]:
 210                 if os.path.isfile(os.path.join(self.directory_of_checkout, subdir, source_file_hit)):
 211                     shutil.copy(os.path.join(self.directory_of_checkout, subdir, source_file_hit), self.directory_of_sourcepackage)
 212                     source_package_list.append(os.path.join(self.directory_of_sourcepackage, source_file_hit))
 213                     break
 214             else:
 215                 tarname = self.spec.name + '-' + self.useversion
 216                 source_package_list.append(self.packagebuilder.create_source_archive(tarname,
 217                                                                                      self.directory_of_checkout,
 218                                                                                      self.directory_of_sourcepackage,
 219                                                                                      self.project_changed,
 220                                                                                      self.spec.source_file_extension))
 221
 222         for patch_file_hit in self.spec.patch_files:
 223             self.logger.info("Copying %s to directory %s", patch_file_hit, self.directory_of_sourcepackage)
 224             for subdir in ["", "SOURCES"]:
 225                 if os.path.isfile(os.path.join(self.directory_of_checkout, subdir, patch_file_hit)):
 226                     shutil.copy(os.path.join(self.directory_of_checkout, subdir, patch_file_hit), self.directory_of_sourcepackage)
 227                     break
 228             else:
 229                 raise ProjectError("Spec file lists patch \"%s\" but no file found" % patch_file_hit)
 230         return source_package_list
 231
 232
 233     def get_source_rpm(self, hostsourcedir, specfile, mockroot):
 234         return self.packagebuilder.mock_source_rpm(hostsourcedir,
 235                                                    specfile,
 236                                                    self.directory_of_srpms,
 237                                                    self.directory_of_builder,
 238                                                    mockroot)
 239
 240     def create_rpm_from_srpm(self, sourcerpm, mockroot):
 241         directory_of_rpm = os.path.join(self.directory_of_rpm, mockroot)
 242         self.packagebuilder.mock_rpm(sourcerpm,
 243                                      directory_of_rpm,
 244                                      self.directory_of_builder,
 245                                      mockroot)
 246         # Delete duplicated src.rpm which is returned by rpm creation
 247         os.remove(os.path.join(directory_of_rpm, os.path.basename(sourcerpm)))
 248
 249     def create_rpm_from_archive(self, source_tar_packages, mockroot):
 250         directory_of_rpm = os.path.join(self.directory_of_rpm, mockroot)
 251         self.packagebuilder.mock_rpm_from_archive(source_tar_packages, directory_of_rpm, self.directory_of_builder, mockroot)
 252
 253     def create_rpm_from_filesystem(self, path, mockroot):
 254         directory_of_rpm = os.path.join(self.directory_of_rpm, mockroot)
 255         self.packagebuilder.mock_rpm_from_filesystem(path,
 256                                                      self.spec.specfilename,
 257                                                      directory_of_rpm,
 258                                                      self.directory_of_builder,
 259                                                      mockroot,
 260                                                      self.directory_of_srpms)
 261
 262     def list_buildproducts_for_mockroot(self, mockroot):
 263         """ List both source and final rpm packages """
 264         srpmlist = []
 265         rpmlist = []
 266         for occurence in os.listdir(os.path.join(self.directory_of_rpm, mockroot)):
 267             if occurence.endswith(".rpm"):
 268                 rpmlist.append(occurence)
 269         for occurence in os.listdir(self.directory_of_srpms):
 270             if occurence.endswith(".src.rpm"):
 271                 srpmlist.append(occurence)
 272         return rpmlist, srpmlist
 273
 274     def resolve_dependencies(self, mockroot):
 275         return self.packagebuilder.run_builddep(self.spec.specfilefullpath,
 276                                                 self.directory_of_srpms,
 277                                                 self.directory_of_builder,
 278                                                 mockroot)
 279
 280     def store_build_products(self, commonrepo):
 281         """ Save build products under common yum repository """
 282         self.__create_directories([commonrepo])
 283         for mockroot in self.builders.roots:
 284             srpmtargetdir = os.path.join(commonrepo, mockroot, 'srpm')
 285             rpmtargetdir = os.path.join(commonrepo, mockroot, 'rpm')
 286             self.__create_directories([srpmtargetdir, rpmtargetdir])
 287             (rpmlist, srpmlist) = self.list_buildproducts_for_mockroot(mockroot)
 288             build_product_dir = os.path.join(self.directory_of_rpm, mockroot)
 289             self.logger.debug("Hard linking %s rpm packages to %s", self.name, rpmtargetdir)
 290             for rpm_file in rpmlist:
 291                 self.logger.info("Hard linking %s", rpm_file)
 292                 try:
 293                     os.link(os.path.join(build_product_dir, rpm_file),
 294                             os.path.join(rpmtargetdir, os.path.basename(rpm_file)))
 295                 except OSError:
 296                     pass
 297             self.logger.debug("Hard linking %s srpm packages to %s", self.name, srpmtargetdir)
 298             for srpm_file in srpmlist:
 299                 self.logger.info("Hard linking %s", srpm_file)
 300                 try:
 301                     os.link(os.path.join(self.directory_of_srpms, srpm_file),
 302                             os.path.join(srpmtargetdir, srpm_file))
 303                 except OSError:
 304                     pass
 305
 306         # Store info of latest build
 307         self.store_project_status()
 308
 309
 310     def who_buildrequires_me(self):
 311         """
 312         Return a list of projects which directly buildrequires this project (non-recursive)
 313         """
 314         downstream_projects = set()
 315         # Loop through my packages
 316         for package in self.spec.packages:
 317             # Loop other projects and check if they need me
 318             # To need me, they have my package in buildrequires
 319             for project in self.projects:
 320                 if package in self.projects[project].spec.buildrequires:
 321                     self.logger.debug("Found dependency in {}: my package {} is required by project {}".format(self.name, package, project))
 322                     self.projects[project].buildrequires_upstream.add(self.name)
 323                     self.projects[self.name].buildrequires_downstream.add(project)
 324                     downstream_projects.add(project)
 325         return downstream_projects
 326
 327
 328     def who_requires_me(self, recursive=False, depth=0):
 329         """
 330         Return a list of projects which have requirement to this project
 331         """
 332         if depth > 10:
 333             self.logger.warn("Hit infinite recursion limiter in {}".format(self.name))
 334             recursive = False
 335         # Loop through my packages
 336         downstream_projects = set()
 337         for package in self.spec.packages:
 338             # Loop other projects and check if they need me
 339             # To need me, they have my package in buildrequires or requires
 340             for project in self.projects:
 341                 if package in self.projects[project].spec.buildrequires \
 342                 or package in self.projects[project].spec.requires:
 343                     downstream_projects.add(project)
 344                     if recursive:
 345                         downstream_projects.update(
 346                             self.projects[project].who_requires_me(True, depth+1))
 347         self.logger.debug("Returning who_requires_me for %s: %s",
 348             self.name, ', '.join(downstream_projects))
 349         return downstream_projects
 350
 351     def get_project_changed(self):
 352         raise NotImplementedError
 353
 354     def store_project_status(self):
 355         raise NotImplementedError
 356
 357     def __create_directories(self, directories, verify_empty=False):
 358         """ Directory creation """
 359         for directory in directories:
 360             if os.path.isdir(directory):
 361                 if verify_empty and os.listdir(directory) != []:
 362                     self.logger.debug("Cleaning directory %s", directory)
 363                     globstring = directory + "/*"
 364                     files = glob.glob(globstring)
 365                     for foundfile in files:
 366                         self.logger.debug("Removing file %s", foundfile)
 367                         os.remove(foundfile)
 368             else:
 369                 self.logger.debug("Creating directory %s", directory)
 370                 try:
 371                     os.makedirs(directory)
 372                 except OSError:
 373                     raise
 374         return True
 375
 376 class LocalMountProject(Project):
 377     """ Projects coming from local disk mount """
 378     def __init__(self, name, directory, workspace, projects, builders, packagebuilder, masterargs, spec_path):
 379         chrootscrub = masterargs.scrub
 380         nosrpm = masterargs.nosrpm
 381         forcebuild = masterargs.forcerebuild
 382
 383         Prettyprint().print_heading("Initializing %s from disk" % name, 60)
 384         super(LocalMountProject, self).__init__(name, workspace, projects, builders, packagebuilder)
 385
 386         if not os.path.isdir(directory):
 387             raise ProjectError("No directory %s found", directory)
 388
 389         self.vcs = VersionControlSystem(directory)
 390         self.directory_of_checkout = directory
 391
 392         # Values from build configuration file
 393         self.projconf = {}
 394         # Read spec
 395         if len(list(find_files(directory, r'\..+\.metadata$'))) > 0 and \
 396                 os.path.isdir(os.path.join(directory, 'SOURCES')) and \
 397                 os.path.isdir(os.path.join(directory, 'SPECS')):
 398             self.centos_style = True
 399             self.logger.debug('CentOS stype RPM detected')
 400         self.spec = Specworker(os.path.dirname(spec_path), os.path.basename(spec_path))
 401
 402         self.gitversioned = False
 403         try:
 404             citag = self.vcs.get_citag()
 405             self.gitversioned = True
 406         except VcsError:
 407             self.logger.debug("Project does not come from Git")
 408         except:
 409             raise
 410
 411         if self.spec.version == '%{_version}':
 412             if self.gitversioned:
 413                 self.logger.debug("Using Git describe for package version")
 414                 self.useversion = citag
 415             else:
 416                 self.logger.debug("Project not from Git. Using a.b package version")
 417                 self.useversion = 'a.b'
 418         else:
 419             self.logger.debug("Using spec definition for package version")
 420             self.useversion = self.spec.version
 421
 422         self.packageversion = self.useversion
 423         self.project_changed = self.get_project_changed()
 424         self.nosrpm = nosrpm
 425
 426         if forcebuild:
 427             self.mark_for_rebuild()
 428
 429         self.chrootscrub = chrootscrub
 430
 431     def get_project_changed(self):
 432         """
 433         Project status is read from status.txt file. Dirty git clones always require rebuild.
 434         """
 435         statusfile = os.path.join(self.project_workspace, 'status.txt')
 436
 437         if os.path.isfile(statusfile):
 438             with open(statusfile, 'r') as filep:
 439                 previousprojectstatus = json.load(filep)
 440             # Compare old values against new values
 441             if not self.gitversioned:
 442                 self.logger.warning("Project %s is not git versioned. Forcing rebuild.", self.name)
 443                 return True
 444             elif self.vcs.is_dirty():
 445                 self.logger.warning("Project %s contains unversioned changes and is \"dirty\". Forcing rebuild.", self.name)
 446                 return True
 447             elif previousprojectstatus['sha'] != self.vcs.commitsha:
 448                 self.logger.info("Project %s log has new hash. Rebuild needed.", self.name)
 449                 return True
 450             else:
 451                 self.logger.info("Project %s has NO new changes.", self.name)
 452             return False
 453         else:
 454             # No configuration means that project has not been compiled
 455             self.logger.warning("No previous build found for %s. Building initial version.", self.name)
 456         return True
 457
 458     def store_project_status(self):
 459         """ Write information of project version to status.txt
 460         This can only be done for git versioned projects """
 461         if self.gitversioned:
 462             # Save information of the last compilation
 463             statusfile = os.path.join(self.project_workspace, 'status.txt')
 464             self.logger.debug("Updating status file %s", statusfile)
 465
 466             projectstatus = {"packageversion": self.packageversion,
 467                              "sha": self.vcs.commitsha,
 468                              "project": self.name}
 469
 470             with open(statusfile, 'w') as outfile:
 471                 json.dump(projectstatus, outfile)
 472
 473 class GitProject(Project):
 474     """ Projects cloned from Git version control system """
 475     def __init__(self, name, workspace, conf, projects, builders, packagebuilder, masterargs):
 476         forcebuild = masterargs.forcerebuild
 477         chrootscrub = masterargs.scrub
 478
 479         Prettyprint().print_heading("Initializing %s from Git" % name, 60)
 480         super(GitProject, self).__init__(name, workspace, projects, builders, packagebuilder)
 481
 482         # Values from build configuration file
 483         self.projconf = {'url': conf.get_string(name, "url", mandatory=True),
 484                          'ref': conf.get_string(name, "ref", mandatory=True),
 485                          'spec': conf.get_string(name, "spec", mandatory=False, defaultvalue=None)}
 486
 487         # Do version control updates
 488         self.directory_of_checkout = os.path.join(self.project_workspace,
 489                                                   'checkout')
 490         self.vcs = VersionControlSystem(self.directory_of_checkout)
 491         self.vcs.update_git_project(self.projconf["url"], self.projconf["ref"])
 492         self.useversion = self.vcs.get_citag()
 493
 494         # Read spec
 495         try:
 496             self.spec = Specworker(self.directory_of_checkout,
 497                                    self.projconf["spec"])
 498         except SpecError:
 499             self.spec = Specworker(os.path.join(self.directory_of_checkout, "SPEC"), None)
 500             self.centos_style = True
 501
 502         # Define what version shall be used in spec file
 503         if self.spec.version == '%{_version}':
 504             self.packageversion = self.vcs.get_citag()
 505             self.logger.debug("Taking package version from VCS")
 506         else:
 507             self.packageversion = self.spec.version
 508             self.logger.debug("Taking package version from spec")
 509         self.logger.debug("Package version: %s", self.packageversion)
 510
 511         self.project_changed = self.get_project_changed()
 512         if forcebuild:
 513             self.mark_for_rebuild()
 514
 515         self.chrootscrub = chrootscrub
 516
 517     def get_project_changed(self):
 518         """
 519         Check if there has been changes in the project
 520         if project has not been compiled -> return = True
 521         if project has GIT/VCS changes   -> return = True
 522         if project has not changed       -> return = False
 523         """
 524         statusfile = os.path.join(self.project_workspace, 'status.txt')
 525
 526         if os.path.isfile(statusfile):
 527             with open(statusfile, 'r') as filep:
 528                 previousprojectstatus = json.load(filep)
 529             # Compare old values against new values
 530             if previousprojectstatus['url'] != self.projconf["url"] \
 531                 or previousprojectstatus['ref'] != self.projconf["ref"] \
 532                 or previousprojectstatus['sha'] != self.vcs.commitsha:
 533                 self.logger.debug("Returning info that changes found")
 534                 return True
 535             else:
 536                 self.logger.debug("Returning info of NO changes")
 537             return False
 538         else:
 539             # No configuration means that project has not been compiled
 540             self.logger.debug("Doing first build of this project")
 541         return True
 542
 543     def store_project_status(self):
 544         """ Save information of the last compilation """
 545         statusfile = os.path.join(self.project_workspace, 'status.txt')
 546         self.logger.debug("Updating status file %s", statusfile)
 547
 548         projectstatus = {"url": self.projconf["url"],
 549                          "ref": self.projconf["ref"],
 550                          "spec": self.projconf["spec"],
 551                          "packageversion": self.packageversion,
 552                          "sha": self.vcs.commitsha,
 553                          "project": self.name}
 554
 555         with open(statusfile, 'w') as outfile:
 556             json.dump(projectstatus, outfile)
 557
 558 class ProjectError(RpmbuilderError):
 559
 560     """ Exceptions originating from Project """
 561     pass