+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Project is a subsystem which contains one spec file which
+defines how it is build. Every project has one git
+repository from where it is cloned from.
+"""
+import glob
+import json
+import logging
+import os
+import shutil
+import subprocess
+
+import re
+
+import datetime
+from rpmbuilder.baseerror import RpmbuilderError
+from rpmbuilder.prettyprinter import Prettyprint
+from rpmbuilder.rpmtools import Repotool, Specworker, RepotoolError, SpecError
+from rpmbuilder.utils import find_files
+from rpmbuilder.version_control import VersionControlSystem, VcsError
+from rpmbuilder.get_sources import get_sources
+
+
+class Project(object):
+
+ """ Instance of a project """
+
+ def __init__(self, name, workspace, projects, builders, packagebuilder, chrootscrub=True, nosrpm=False):
+ self.name = name
+
+ self.logger = logging.getLogger(__name__ + "." + self.name)
+
+ self.project_rebuild_needed = False
+
+ self.project_workspace = os.path.join(workspace,
+ 'projects',
+ self.name)
+
+ self.projects = projects
+ self.builders = builders
+ self.directory_of_specpatch = os.path.join(self.project_workspace,
+ 'rpmbuild',
+ 'spec')
+ self.directory_of_sourcepackage = os.path.join(self.project_workspace,
+ 'rpmbuild',
+ 'sources')
+ self.directory_of_srpms = os.path.join(self.project_workspace,
+ 'rpmbuild',
+ 'srpm')
+ self.directory_of_rpm = os.path.join(self.project_workspace,
+ 'rpmbuild',
+ 'rpm')
+ self.directory_of_commonrepo = os.path.join(workspace,
+ 'buildrepository')
+
+ self.directory_of_builder = self.builders.get_configdir()
+
+ self.__create_directories([self.directory_of_specpatch,
+ self.directory_of_srpms],
+ verify_empty=True)
+ self.__create_directories([self.directory_of_sourcepackage],
+ verify_empty=False)
+
+ self.packagebuilder = packagebuilder
+
+ self.chrootscrub = chrootscrub
+ self.built = {}
+ for mockroot in builders.roots:
+ self.built[mockroot] = False
+
+ self.project_changed = False
+ self.projconf = None
+ self.spec = None
+ self.useversion = None
+ self.directory_of_checkout = None
+ self.nosrpm = nosrpm
+ self.centos_style = False
+ self.buildrequires_downstream = set()
+ self.buildrequires_upstream = set()
+
+ def mark_for_rebuild(self):
+ """ Marking project for rebuild only if project has not changed """
+ if not self.project_changed:
+ self.logger.debug("Marking project %s for rebuild.", self.name)
+ self.project_rebuild_needed = True
+
+ def mark_downstream_for_rebuild(self, marked_for_build=None):
+ """
+ Recursively mark downstream projects for rebuilding.
+ Return set of projects marked for rebuild
+ """
+ if marked_for_build is None:
+ marked_for_build = set()
+ self.logger.debug("Marking downstream for rebuild in \"%s\"",
+ self.name)
+ for project in self.who_buildrequires_me():
+ self.logger.debug("BuildRequires to \"%s\" found in \"%s\"",
+ self.name, project)
+ if project in marked_for_build:
+ self.logger.debug("\"%s\" already marked for build", project)
+ elif self.projects[project].project_rebuild_needed:
+ self.logger.debug("\"%s\" already marked for rebuild", project)
+ else:
+ self.projects[project].mark_for_rebuild()
+ marked_for_build.add(project)
+ # Check if downstream has downstream projects
+ tmpset = self.projects[project].mark_downstream_for_rebuild(
+ marked_for_build)
+ marked_for_build.update(tmpset)
+ return marked_for_build
+
+ def build_project(self, mockroot):
+ """ Do building of SRPM and RPM files """
+ time_start = datetime.datetime.now()
+ Prettyprint().print_heading("Build " + self.name, 60)
+ assert not self.built[mockroot], "Project already built"
+
+ # Produce spec file
+ if self.spec.version == '%{_version}':
+ self.logger.debug("patching spec file")
+ self.logger.debug("Version in spec is going to be %s", self.useversion)
+
+ rpm = Repotool()
+ userelease = rpm.next_release_of_package(
+ os.path.join(self.directory_of_commonrepo,
+ self.builders.roots[0],
+ "rpm"),
+ self.spec.name,
+ self.useversion,
+ self.spec.release)
+ self.logger.debug("Release in spec is going to be %s", userelease)
+
+ specfile = self.packagebuilder.patch_specfile(self.spec.specfilefullpath,
+ self.directory_of_specpatch,
+ self.useversion,
+ userelease)
+ else:
+ self.logger.debug("Skipping spec patching")
+ specfile = self.spec.specfilefullpath
+
+ # Start mocking
+ self.logger.debug("Starting building in root \"%s\"", mockroot)
+ if self.centos_style:
+ shutil.rmtree(self.directory_of_sourcepackage)
+ ignore_git = shutil.ignore_patterns('.git')
+ shutil.copytree(self.directory_of_checkout, self.directory_of_sourcepackage, ignore=ignore_git)
+ sources_key = 'CENTOS_SOURCES'
+ if sources_key not in os.environ:
+ raise RpmbuilderError('Cannot build CentOS style RPM, %s not defined in the environment' % sources_key)
+ get_sources(self.directory_of_sourcepackage, os.environ[sources_key].split(','), self.logger)
+ self.create_rpm_from_filesystem(self.directory_of_sourcepackage, mockroot)
+ elif self.nosrpm:
+ list_of_source_packages = self.get_source_package()
+ self.create_rpm_from_archive(list_of_source_packages, mockroot)
+ else:
+ self.get_source_package()
+ # Create source RPM file
+ sourcerpm = self.get_source_rpm(self.directory_of_sourcepackage, specfile, mockroot)
+
+ # Create final RPM file(s)
+ self.create_rpm_from_srpm(sourcerpm, mockroot)
+
+ # Mark build completed
+ self.built[mockroot] = True
+ time_delta = datetime.datetime.now() - time_start
+ self.logger.info('Building success: %s (took %s [%s sec])', self.name, time_delta, time_delta.seconds)
+
+ # We wipe buildroot of previously built rpm, source etc. packages
+ # This is custom cleaning which does not remove chroot
+ self.packagebuilder.mock_wipe_buildroot(self.project_workspace, self.directory_of_builder, mockroot)
+
+ def pull_source_packages(self, target_dir):
+ cmd = ['/usr/bin/spectool', '-d', 'KVERSION a.b', '-g', '--directory', target_dir, self.spec.specfilefullpath]
+ self.logger.info('Pulling source packages: %s', cmd)
+ try:
+ subprocess.check_call(cmd, shell=False)
+ self.logger.info('Pulling source packages ok')
+ except OSError as err:
+ self.logger.info('Pulling source packages nok %s', err.strerror)
+ raise RepotoolError("Calling of command spectool caused: \"%s\"" % err.strerror)
+ except:
+ self.logger.info('Pulling source packages nok ??', err.strerror)
+ raise RepotoolError("There was error pulling source content")
+
+ def get_source_package(self):
+ # Produce source package
+ source_package_list = []
+ for source_file_hit in self.spec.source_files:
+ self.logger.info("Acquiring source file \"%s\"", source_file_hit)
+ if re.match(r'^(http[s]*|ftp)://', source_file_hit):
+ self.logger.info("PULL %s", self.directory_of_sourcepackage)
+ self.pull_source_packages(self.directory_of_sourcepackage)
+ source_package_list.append(self.directory_of_sourcepackage + '/' + source_file_hit.split('/')[-1])
+ continue
+ for subdir in ["", "SOURCES"]:
+ if os.path.isfile(os.path.join(self.directory_of_checkout, subdir, source_file_hit)):
+ shutil.copy(os.path.join(self.directory_of_checkout, subdir, source_file_hit), self.directory_of_sourcepackage)
+ source_package_list.append(os.path.join(self.directory_of_sourcepackage, source_file_hit))
+ break
+ else:
+ tarname = self.spec.name + '-' + self.useversion
+ source_package_list.append(self.packagebuilder.create_source_archive(tarname,
+ self.directory_of_checkout,
+ self.directory_of_sourcepackage,
+ self.project_changed,
+ self.spec.source_file_extension))
+
+ for patch_file_hit in self.spec.patch_files:
+ self.logger.info("Copying %s to directory %s", patch_file_hit, self.directory_of_sourcepackage)
+ for subdir in ["", "SOURCES"]:
+ if os.path.isfile(os.path.join(self.directory_of_checkout, subdir, patch_file_hit)):
+ shutil.copy(os.path.join(self.directory_of_checkout, subdir, patch_file_hit), self.directory_of_sourcepackage)
+ break
+ else:
+ raise ProjectError("Spec file lists patch \"%s\" but no file found" % patch_file_hit)
+ return source_package_list
+
+
+ def get_source_rpm(self, hostsourcedir, specfile, mockroot):
+ return self.packagebuilder.mock_source_rpm(hostsourcedir,
+ specfile,
+ self.directory_of_srpms,
+ self.directory_of_builder,
+ mockroot)
+
+ def create_rpm_from_srpm(self, sourcerpm, mockroot):
+ directory_of_rpm = os.path.join(self.directory_of_rpm, mockroot)
+ self.packagebuilder.mock_rpm(sourcerpm,
+ directory_of_rpm,
+ self.directory_of_builder,
+ mockroot)
+ # Delete duplicated src.rpm which is returned by rpm creation
+ os.remove(os.path.join(directory_of_rpm, os.path.basename(sourcerpm)))
+
+ def create_rpm_from_archive(self, source_tar_packages, mockroot):
+ directory_of_rpm = os.path.join(self.directory_of_rpm, mockroot)
+ self.packagebuilder.mock_rpm_from_archive(source_tar_packages, directory_of_rpm, self.directory_of_builder, mockroot)
+
+ def create_rpm_from_filesystem(self, path, mockroot):
+ directory_of_rpm = os.path.join(self.directory_of_rpm, mockroot)
+ self.packagebuilder.mock_rpm_from_filesystem(path,
+ self.spec.specfilename,
+ directory_of_rpm,
+ self.directory_of_builder,
+ mockroot,
+ self.directory_of_srpms)
+
+ def list_buildproducts_for_mockroot(self, mockroot):
+ """ List both source and final rpm packages """
+ srpmlist = []
+ rpmlist = []
+ for occurence in os.listdir(os.path.join(self.directory_of_rpm, mockroot)):
+ if occurence.endswith(".rpm"):
+ rpmlist.append(occurence)
+ for occurence in os.listdir(self.directory_of_srpms):
+ if occurence.endswith(".src.rpm"):
+ srpmlist.append(occurence)
+ return rpmlist, srpmlist
+
+ def resolve_dependencies(self, mockroot):
+ return self.packagebuilder.run_builddep(self.spec.specfilefullpath,
+ self.directory_of_srpms,
+ self.directory_of_builder,
+ mockroot)
+
+ def store_build_products(self, commonrepo):
+ """ Save build products under common yum repository """
+ self.__create_directories([commonrepo])
+ for mockroot in self.builders.roots:
+ srpmtargetdir = os.path.join(commonrepo, mockroot, 'srpm')
+ rpmtargetdir = os.path.join(commonrepo, mockroot, 'rpm')
+ self.__create_directories([srpmtargetdir, rpmtargetdir])
+ (rpmlist, srpmlist) = self.list_buildproducts_for_mockroot(mockroot)
+ build_product_dir = os.path.join(self.directory_of_rpm, mockroot)
+ self.logger.debug("Hard linking %s rpm packages to %s", self.name, rpmtargetdir)
+ for rpm_file in rpmlist:
+ self.logger.info("Hard linking %s", rpm_file)
+ try:
+ os.link(os.path.join(build_product_dir, rpm_file),
+ os.path.join(rpmtargetdir, os.path.basename(rpm_file)))
+ except OSError:
+ pass
+ self.logger.debug("Hard linking %s srpm packages to %s", self.name, srpmtargetdir)
+ for srpm_file in srpmlist:
+ self.logger.info("Hard linking %s", srpm_file)
+ try:
+ os.link(os.path.join(self.directory_of_srpms, srpm_file),
+ os.path.join(srpmtargetdir, srpm_file))
+ except OSError:
+ pass
+
+ # Store info of latest build
+ self.store_project_status()
+
+
+ def who_buildrequires_me(self):
+ """
+ Return a list of projects which directly buildrequires this project (non-recursive)
+ """
+ downstream_projects = set()
+ # Loop through my packages
+ for package in self.spec.packages:
+ # Loop other projects and check if they need me
+ # To need me, they have my package in buildrequires
+ for project in self.projects:
+ if package in self.projects[project].spec.buildrequires:
+ self.logger.debug("Found dependency in {}: my package {} is required by project {}".format(self.name, package, project))
+ self.projects[project].buildrequires_upstream.add(self.name)
+ self.projects[self.name].buildrequires_downstream.add(project)
+ downstream_projects.add(project)
+ return downstream_projects
+
+
+ def who_requires_me(self, recursive=False, depth=0):
+ """
+ Return a list of projects which have requirement to this project
+ """
+ if depth > 10:
+ self.logger.warn("Hit infinite recursion limiter in {}".format(self.name))
+ recursive = False
+ # Loop through my packages
+ downstream_projects = set()
+ for package in self.spec.packages:
+ # Loop other projects and check if they need me
+ # To need me, they have my package in buildrequires or requires
+ for project in self.projects:
+ if package in self.projects[project].spec.buildrequires \
+ or package in self.projects[project].spec.requires:
+ downstream_projects.add(project)
+ if recursive:
+ downstream_projects.update(
+ self.projects[project].who_requires_me(True, depth+1))
+ self.logger.debug("Returning who_requires_me for %s: %s",
+ self.name, ', '.join(downstream_projects))
+ return downstream_projects
+
+ def get_project_changed(self):
+ raise NotImplementedError
+
+ def store_project_status(self):
+ raise NotImplementedError
+
+ def __create_directories(self, directories, verify_empty=False):
+ """ Directory creation """
+ for directory in directories:
+ if os.path.isdir(directory):
+ if verify_empty and os.listdir(directory) != []:
+ self.logger.debug("Cleaning directory %s", directory)
+ globstring = directory + "/*"
+ files = glob.glob(globstring)
+ for foundfile in files:
+ self.logger.debug("Removing file %s", foundfile)
+ os.remove(foundfile)
+ else:
+ self.logger.debug("Creating directory %s", directory)
+ try:
+ os.makedirs(directory)
+ except OSError:
+ raise
+ return True
+
+class LocalMountProject(Project):
+ """ Projects coming from local disk mount """
+ def __init__(self, name, directory, workspace, projects, builders, packagebuilder, masterargs, spec_path):
+ chrootscrub = masterargs.scrub
+ nosrpm = masterargs.nosrpm
+ forcebuild = masterargs.forcerebuild
+
+ Prettyprint().print_heading("Initializing %s from disk" % name, 60)
+ super(LocalMountProject, self).__init__(name, workspace, projects, builders, packagebuilder)
+
+ if not os.path.isdir(directory):
+ raise ProjectError("No directory %s found", directory)
+
+ self.vcs = VersionControlSystem(directory)
+ self.directory_of_checkout = directory
+
+ # Values from build configuration file
+ self.projconf = {}
+ # Read spec
+ if len(list(find_files(directory, r'\..+\.metadata$'))) > 0 and \
+ os.path.isdir(os.path.join(directory, 'SOURCES')) and \
+ os.path.isdir(os.path.join(directory, 'SPECS')):
+ self.centos_style = True
+ self.logger.debug('CentOS stype RPM detected')
+ self.spec = Specworker(os.path.dirname(spec_path), os.path.basename(spec_path))
+
+ self.gitversioned = False
+ try:
+ citag = self.vcs.get_citag()
+ self.gitversioned = True
+ except VcsError:
+ self.logger.debug("Project does not come from Git")
+ except:
+ raise
+
+ if self.spec.version == '%{_version}':
+ if self.gitversioned:
+ self.logger.debug("Using Git describe for package version")
+ self.useversion = citag
+ else:
+ self.logger.debug("Project not from Git. Using a.b package version")
+ self.useversion = 'a.b'
+ else:
+ self.logger.debug("Using spec definition for package version")
+ self.useversion = self.spec.version
+
+ self.packageversion = self.useversion
+ self.project_changed = self.get_project_changed()
+ self.nosrpm = nosrpm
+
+ if forcebuild:
+ self.mark_for_rebuild()
+
+ self.chrootscrub = chrootscrub
+
+ def get_project_changed(self):
+ """
+ Project status is read from status.txt file. Dirty git clones always require rebuild.
+ """
+ statusfile = os.path.join(self.project_workspace, 'status.txt')
+
+ if os.path.isfile(statusfile):
+ with open(statusfile, 'r') as filep:
+ previousprojectstatus = json.load(filep)
+ # Compare old values against new values
+ if not self.gitversioned:
+ self.logger.warning("Project %s is not git versioned. Forcing rebuild.", self.name)
+ return True
+ elif self.vcs.is_dirty():
+ self.logger.warning("Project %s contains unversioned changes and is \"dirty\". Forcing rebuild.", self.name)
+ return True
+ elif previousprojectstatus['sha'] != self.vcs.commitsha:
+ self.logger.info("Project %s log has new hash. Rebuild needed.", self.name)
+ return True
+ else:
+ self.logger.info("Project %s has NO new changes.", self.name)
+ return False
+ else:
+ # No configuration means that project has not been compiled
+ self.logger.warning("No previous build found for %s. Building initial version.", self.name)
+ return True
+
+ def store_project_status(self):
+ """ Write information of project version to status.txt
+ This can only be done for git versioned projects """
+ if self.gitversioned:
+ # Save information of the last compilation
+ statusfile = os.path.join(self.project_workspace, 'status.txt')
+ self.logger.debug("Updating status file %s", statusfile)
+
+ projectstatus = {"packageversion": self.packageversion,
+ "sha": self.vcs.commitsha,
+ "project": self.name}
+
+ with open(statusfile, 'w') as outfile:
+ json.dump(projectstatus, outfile)
+
+class GitProject(Project):
+ """ Projects cloned from Git version control system """
+ def __init__(self, name, workspace, conf, projects, builders, packagebuilder, masterargs):
+ forcebuild = masterargs.forcerebuild
+ chrootscrub = masterargs.scrub
+
+ Prettyprint().print_heading("Initializing %s from Git" % name, 60)
+ super(GitProject, self).__init__(name, workspace, projects, builders, packagebuilder)
+
+ # Values from build configuration file
+ self.projconf = {'url': conf.get_string(name, "url", mandatory=True),
+ 'ref': conf.get_string(name, "ref", mandatory=True),
+ 'spec': conf.get_string(name, "spec", mandatory=False, defaultvalue=None)}
+
+ # Do version control updates
+ self.directory_of_checkout = os.path.join(self.project_workspace,
+ 'checkout')
+ self.vcs = VersionControlSystem(self.directory_of_checkout)
+ self.vcs.update_git_project(self.projconf["url"], self.projconf["ref"])
+ self.useversion = self.vcs.get_citag()
+
+ # Read spec
+ try:
+ self.spec = Specworker(self.directory_of_checkout,
+ self.projconf["spec"])
+ except SpecError:
+ self.spec = Specworker(os.path.join(self.directory_of_checkout, "SPEC"), None)
+ self.centos_style = True
+
+ # Define what version shall be used in spec file
+ if self.spec.version == '%{_version}':
+ self.packageversion = self.vcs.get_citag()
+ self.logger.debug("Taking package version from VCS")
+ else:
+ self.packageversion = self.spec.version
+ self.logger.debug("Taking package version from spec")
+ self.logger.debug("Package version: %s", self.packageversion)
+
+ self.project_changed = self.get_project_changed()
+ if forcebuild:
+ self.mark_for_rebuild()
+
+ self.chrootscrub = chrootscrub
+
+ def get_project_changed(self):
+ """
+ Check if there has been changes in the project
+ if project has not been compiled -> return = True
+ if project has GIT/VCS changes -> return = True
+ if project has not changed -> return = False
+ """
+ statusfile = os.path.join(self.project_workspace, 'status.txt')
+
+ if os.path.isfile(statusfile):
+ with open(statusfile, 'r') as filep:
+ previousprojectstatus = json.load(filep)
+ # Compare old values against new values
+ if previousprojectstatus['url'] != self.projconf["url"] \
+ or previousprojectstatus['ref'] != self.projconf["ref"] \
+ or previousprojectstatus['sha'] != self.vcs.commitsha:
+ self.logger.debug("Returning info that changes found")
+ return True
+ else:
+ self.logger.debug("Returning info of NO changes")
+ return False
+ else:
+ # No configuration means that project has not been compiled
+ self.logger.debug("Doing first build of this project")
+ return True
+
+ def store_project_status(self):
+ """ Save information of the last compilation """
+ statusfile = os.path.join(self.project_workspace, 'status.txt')
+ self.logger.debug("Updating status file %s", statusfile)
+
+ projectstatus = {"url": self.projconf["url"],
+ "ref": self.projconf["ref"],
+ "spec": self.projconf["spec"],
+ "packageversion": self.packageversion,
+ "sha": self.vcs.commitsha,
+ "project": self.name}
+
+ with open(statusfile, 'w') as outfile:
+ json.dump(projectstatus, outfile)
+
+class ProjectError(RpmbuilderError):
+
+ """ Exceptions originating from Project """
+ pass
Code Review / ta / rpmbuilder.git / blobdiff