X-Git-Url: https://gerrit.akraino.org/r/gitweb?p=ta%2Frpmbuilder.git;a=blobdiff_plain;f=rpmbuilder%2Fproject.py;fp=rpmbuilder%2Fproject.py;h=e100d945ad32fc01bdb12b1de4070af33de19d84;hp=0000000000000000000000000000000000000000;hb=876631a959303430aafc0be7897b086ee9b921fe;hpb=d8468e0423a9af0d3fd5bf30d45ebe18ba8b1801

diff --git a/rpmbuilder/project.py b/rpmbuilder/project.py
new file mode 100644
index 0000000..e100d94
--- /dev/null
+++ b/rpmbuilder/project.py
@@ -0,0 +1,561 @@
+# Copyright 2019 Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Project is a subsystem which contains one spec file which
+defines how it is build. Every project has one git
+repository from where it is cloned from.
+"""
+import glob
+import json
+import logging
+import os
+import shutil
+import subprocess
+
+import re
+
+import datetime
+from rpmbuilder.baseerror import RpmbuilderError
+from rpmbuilder.prettyprinter import Prettyprint
+from rpmbuilder.rpmtools import Repotool, Specworker, RepotoolError, SpecError
+from rpmbuilder.utils import find_files
+from rpmbuilder.version_control import VersionControlSystem, VcsError
+from rpmbuilder.get_sources import get_sources
+
+
+class Project(object):
+
+    """ Instance of a project """
+
+    def __init__(self, name, workspace, projects, builders, packagebuilder, chrootscrub=True, nosrpm=False):
+        self.name = name
+
+        self.logger = logging.getLogger(__name__ + "." + self.name)
+
+        self.project_rebuild_needed = False
+
+        self.project_workspace = os.path.join(workspace,
+                                              'projects',
+                                              self.name)
+
+        self.projects = projects
+        self.builders = builders
+        self.directory_of_specpatch = os.path.join(self.project_workspace,
+                                                   'rpmbuild',
+                                                   'spec')
+        self.directory_of_sourcepackage = os.path.join(self.project_workspace,
+                                                       'rpmbuild',
+                                                       'sources')
+        self.directory_of_srpms = os.path.join(self.project_workspace,
+                                               'rpmbuild',
+                                               'srpm')
+        self.directory_of_rpm = os.path.join(self.project_workspace,
+                                             'rpmbuild',
+                                             'rpm')
+        self.directory_of_commonrepo = os.path.join(workspace,
+                                                    'buildrepository')
+
+        self.directory_of_builder = self.builders.get_configdir()
+
+        self.__create_directories([self.directory_of_specpatch,
+                                   self.directory_of_srpms],
+                                  verify_empty=True)
+        self.__create_directories([self.directory_of_sourcepackage],
+                                  verify_empty=False)
+
+        self.packagebuilder = packagebuilder
+
+        self.chrootscrub = chrootscrub
+        self.built = {}
+        for mockroot in builders.roots:
+            self.built[mockroot] = False
+
+        self.project_changed = False
+        self.projconf = None
+        self.spec = None
+        self.useversion = None
+        self.directory_of_checkout = None
+        self.nosrpm = nosrpm
+        self.centos_style = False
+        self.buildrequires_downstream = set()
+        self.buildrequires_upstream = set()
+
+    def mark_for_rebuild(self):
+        """ Marking project for rebuild only if project has not changed """
+        if not self.project_changed:
+            self.logger.debug("Marking project %s for rebuild.", self.name)
+            self.project_rebuild_needed = True
+
+    def mark_downstream_for_rebuild(self, marked_for_build=None):
+        """
+        Recursively mark downstream projects for rebuilding.
+        Return set of projects marked for rebuild
+        """
+        if marked_for_build is None:
+            marked_for_build = set()
+        self.logger.debug("Marking downstream for rebuild in \"%s\"",
+            self.name)
+        for project in self.who_buildrequires_me():
+            self.logger.debug("BuildRequires to \"%s\" found in \"%s\"",
+                self.name, project)
+            if project in marked_for_build:
+                self.logger.debug("\"%s\" already marked for build", project)
+            elif self.projects[project].project_rebuild_needed:
+                self.logger.debug("\"%s\" already marked for rebuild", project)
+            else:
+                self.projects[project].mark_for_rebuild()
+                marked_for_build.add(project)
+                # Check if downstream has downstream projects
+                tmpset = self.projects[project].mark_downstream_for_rebuild(
+                    marked_for_build)
+                marked_for_build.update(tmpset)
+        return marked_for_build
+
+    def build_project(self, mockroot):
+        """ Do building of SRPM and RPM files """
+        time_start = datetime.datetime.now()
+        Prettyprint().print_heading("Build " + self.name, 60)
+        assert not self.built[mockroot], "Project already built"
+
+        # Produce spec file
+        if self.spec.version == '%{_version}':
+            self.logger.debug("patching spec file")
+            self.logger.debug("Version in spec is going to be %s", self.useversion)
+
+            rpm = Repotool()
+            userelease = rpm.next_release_of_package(
+                os.path.join(self.directory_of_commonrepo,
+                             self.builders.roots[0],
+                             "rpm"),
+                self.spec.name,
+                self.useversion,
+                self.spec.release)
+            self.logger.debug("Release in spec is going to be %s", userelease)
+
+            specfile = self.packagebuilder.patch_specfile(self.spec.specfilefullpath,
+                                                          self.directory_of_specpatch,
+                                                          self.useversion,
+                                                          userelease)
+        else:
+            self.logger.debug("Skipping spec patching")
+            specfile = self.spec.specfilefullpath
+
+        # Start mocking
+        self.logger.debug("Starting building in root \"%s\"", mockroot)
+        if self.centos_style:
+            shutil.rmtree(self.directory_of_sourcepackage)
+            ignore_git = shutil.ignore_patterns('.git')
+            shutil.copytree(self.directory_of_checkout, self.directory_of_sourcepackage, ignore=ignore_git)
+            sources_key = 'CENTOS_SOURCES'
+            if sources_key not in os.environ:
+                raise RpmbuilderError('Cannot build CentOS style RPM, %s not defined in the environment' % sources_key)
+            get_sources(self.directory_of_sourcepackage, os.environ[sources_key].split(','), self.logger)
+            self.create_rpm_from_filesystem(self.directory_of_sourcepackage, mockroot)
+        elif self.nosrpm:
+            list_of_source_packages = self.get_source_package()
+            self.create_rpm_from_archive(list_of_source_packages, mockroot)
+        else:
+            self.get_source_package()
+            # Create source RPM file
+            sourcerpm = self.get_source_rpm(self.directory_of_sourcepackage, specfile, mockroot)
+
+            # Create final RPM file(s)
+            self.create_rpm_from_srpm(sourcerpm, mockroot)
+
+        # Mark build completed
+        self.built[mockroot] = True
+        time_delta = datetime.datetime.now() - time_start
+        self.logger.info('Building success: %s (took %s [%s sec])', self.name, time_delta, time_delta.seconds)
+
+        # We wipe buildroot of previously built rpm, source etc. packages
+        # This is custom cleaning which does not remove chroot
+        self.packagebuilder.mock_wipe_buildroot(self.project_workspace, self.directory_of_builder, mockroot)
+
+    def pull_source_packages(self, target_dir):
+        cmd = ['/usr/bin/spectool', '-d', 'KVERSION a.b', '-g', '--directory', target_dir, self.spec.specfilefullpath]
+        self.logger.info('Pulling source packages: %s', cmd)
+        try:
+            subprocess.check_call(cmd, shell=False)
+            self.logger.info('Pulling source packages ok')
+        except OSError as err:
+            self.logger.info('Pulling source packages nok %s', err.strerror)
+            raise RepotoolError("Calling of command spectool caused: \"%s\"" % err.strerror)
+        except:
+            self.logger.info('Pulling source packages nok ??', err.strerror)
+            raise RepotoolError("There was error pulling source content")
+
+    def get_source_package(self):
+        # Produce source package
+        source_package_list = []
+        for source_file_hit in self.spec.source_files:
+            self.logger.info("Acquiring source file \"%s\"", source_file_hit)
+            if re.match(r'^(http[s]*|ftp)://', source_file_hit):
+                self.logger.info("PULL %s", self.directory_of_sourcepackage)
+                self.pull_source_packages(self.directory_of_sourcepackage)
+                source_package_list.append(self.directory_of_sourcepackage + '/' + source_file_hit.split('/')[-1])
+                continue
+            for subdir in ["", "SOURCES"]:
+                if os.path.isfile(os.path.join(self.directory_of_checkout, subdir, source_file_hit)):
+                    shutil.copy(os.path.join(self.directory_of_checkout, subdir, source_file_hit), self.directory_of_sourcepackage)
+                    source_package_list.append(os.path.join(self.directory_of_sourcepackage, source_file_hit))
+                    break
+            else:
+                tarname = self.spec.name + '-' + self.useversion
+                source_package_list.append(self.packagebuilder.create_source_archive(tarname,
+                                                                                     self.directory_of_checkout,
+                                                                                     self.directory_of_sourcepackage,
+                                                                                     self.project_changed,
+                                                                                     self.spec.source_file_extension))
+
+        for patch_file_hit in self.spec.patch_files:
+            self.logger.info("Copying %s to directory %s", patch_file_hit, self.directory_of_sourcepackage)
+            for subdir in ["", "SOURCES"]:
+                if os.path.isfile(os.path.join(self.directory_of_checkout, subdir, patch_file_hit)):
+                    shutil.copy(os.path.join(self.directory_of_checkout, subdir, patch_file_hit), self.directory_of_sourcepackage)
+                    break
+            else:
+                raise ProjectError("Spec file lists patch \"%s\" but no file found" % patch_file_hit)
+        return source_package_list
+
+
+    def get_source_rpm(self, hostsourcedir, specfile, mockroot):
+        return self.packagebuilder.mock_source_rpm(hostsourcedir,
+                                                   specfile,
+                                                   self.directory_of_srpms,
+                                                   self.directory_of_builder,
+                                                   mockroot)
+
+    def create_rpm_from_srpm(self, sourcerpm, mockroot):
+        directory_of_rpm = os.path.join(self.directory_of_rpm, mockroot)
+        self.packagebuilder.mock_rpm(sourcerpm,
+                                     directory_of_rpm,
+                                     self.directory_of_builder,
+                                     mockroot)
+        # Delete duplicated src.rpm which is returned by rpm creation
+        os.remove(os.path.join(directory_of_rpm, os.path.basename(sourcerpm)))
+
+    def create_rpm_from_archive(self, source_tar_packages, mockroot):
+        directory_of_rpm = os.path.join(self.directory_of_rpm, mockroot)
+        self.packagebuilder.mock_rpm_from_archive(source_tar_packages, directory_of_rpm, self.directory_of_builder, mockroot)
+
+    def create_rpm_from_filesystem(self, path, mockroot):
+        directory_of_rpm = os.path.join(self.directory_of_rpm, mockroot)
+        self.packagebuilder.mock_rpm_from_filesystem(path,
+                                                     self.spec.specfilename,
+                                                     directory_of_rpm,
+                                                     self.directory_of_builder,
+                                                     mockroot,
+                                                     self.directory_of_srpms)
+
+    def list_buildproducts_for_mockroot(self, mockroot):
+        """ List both source and final rpm packages """
+        srpmlist = []
+        rpmlist = []
+        for occurence in os.listdir(os.path.join(self.directory_of_rpm, mockroot)):
+            if occurence.endswith(".rpm"):
+                rpmlist.append(occurence)
+        for occurence in os.listdir(self.directory_of_srpms):
+            if occurence.endswith(".src.rpm"):
+                srpmlist.append(occurence)
+        return rpmlist, srpmlist
+
+    def resolve_dependencies(self, mockroot):
+        return self.packagebuilder.run_builddep(self.spec.specfilefullpath,
+                                                self.directory_of_srpms,
+                                                self.directory_of_builder,
+                                                mockroot)
+
+    def store_build_products(self, commonrepo):
+        """ Save build products under common yum repository """
+        self.__create_directories([commonrepo])
+        for mockroot in self.builders.roots:
+            srpmtargetdir = os.path.join(commonrepo, mockroot, 'srpm')
+            rpmtargetdir = os.path.join(commonrepo, mockroot, 'rpm')
+            self.__create_directories([srpmtargetdir, rpmtargetdir])
+            (rpmlist, srpmlist) = self.list_buildproducts_for_mockroot(mockroot)
+            build_product_dir = os.path.join(self.directory_of_rpm, mockroot)
+            self.logger.debug("Hard linking %s rpm packages to %s", self.name, rpmtargetdir)
+            for rpm_file in rpmlist:
+                self.logger.info("Hard linking %s", rpm_file)
+                try:
+                    os.link(os.path.join(build_product_dir, rpm_file),
+                            os.path.join(rpmtargetdir, os.path.basename(rpm_file)))
+                except OSError:
+                    pass
+            self.logger.debug("Hard linking %s srpm packages to %s", self.name, srpmtargetdir)
+            for srpm_file in srpmlist:
+                self.logger.info("Hard linking %s", srpm_file)
+                try:
+                    os.link(os.path.join(self.directory_of_srpms, srpm_file),
+                            os.path.join(srpmtargetdir, srpm_file))
+                except OSError:
+                    pass
+
+        # Store info of latest build
+        self.store_project_status()
+
+
+    def who_buildrequires_me(self):
+        """
+        Return a list of projects which directly buildrequires this project (non-recursive)
+        """
+        downstream_projects = set()
+        # Loop through my packages
+        for package in self.spec.packages:
+            # Loop other projects and check if they need me
+            # To need me, they have my package in buildrequires
+            for project in self.projects:
+                if package in self.projects[project].spec.buildrequires:
+                    self.logger.debug("Found dependency in {}: my package {} is required by project {}".format(self.name, package, project))
+                    self.projects[project].buildrequires_upstream.add(self.name)
+                    self.projects[self.name].buildrequires_downstream.add(project)
+                    downstream_projects.add(project)
+        return downstream_projects
+
+
+    def who_requires_me(self, recursive=False, depth=0):
+        """
+        Return a list of projects which have requirement to this project
+        """
+        if depth > 10:
+            self.logger.warn("Hit infinite recursion limiter in {}".format(self.name))
+            recursive = False
+        # Loop through my packages
+        downstream_projects = set()
+        for package in self.spec.packages:
+            # Loop other projects and check if they need me
+            # To need me, they have my package in buildrequires or requires
+            for project in self.projects:
+                if package in self.projects[project].spec.buildrequires \
+                or package in self.projects[project].spec.requires:
+                    downstream_projects.add(project)
+                    if recursive:
+                        downstream_projects.update(
+                            self.projects[project].who_requires_me(True, depth+1))
+        self.logger.debug("Returning who_requires_me for %s: %s",
+            self.name, ', '.join(downstream_projects))
+        return downstream_projects
+
+    def get_project_changed(self):
+        raise NotImplementedError
+
+    def store_project_status(self):
+        raise NotImplementedError
+
+    def __create_directories(self, directories, verify_empty=False):
+        """ Directory creation """
+        for directory in directories:
+            if os.path.isdir(directory):
+                if verify_empty and os.listdir(directory) != []:
+                    self.logger.debug("Cleaning directory %s", directory)
+                    globstring = directory + "/*"
+                    files = glob.glob(globstring)
+                    for foundfile in files:
+                        self.logger.debug("Removing file %s", foundfile)
+                        os.remove(foundfile)
+            else:
+                self.logger.debug("Creating directory %s", directory)
+                try:
+                    os.makedirs(directory)
+                except OSError:
+                    raise
+        return True
+
+class LocalMountProject(Project):
+    """ Projects coming from local disk mount """
+    def __init__(self, name, directory, workspace, projects, builders, packagebuilder, masterargs, spec_path):
+        chrootscrub = masterargs.scrub
+        nosrpm = masterargs.nosrpm
+        forcebuild = masterargs.forcerebuild
+
+        Prettyprint().print_heading("Initializing %s from disk" % name, 60)
+        super(LocalMountProject, self).__init__(name, workspace, projects, builders, packagebuilder)
+
+        if not os.path.isdir(directory):
+            raise ProjectError("No directory %s found", directory)
+
+        self.vcs = VersionControlSystem(directory)
+        self.directory_of_checkout = directory
+
+        # Values from build configuration file
+        self.projconf = {}
+        # Read spec
+        if len(list(find_files(directory, r'\..+\.metadata$'))) > 0 and \
+                os.path.isdir(os.path.join(directory, 'SOURCES')) and \
+                os.path.isdir(os.path.join(directory, 'SPECS')):
+            self.centos_style = True
+            self.logger.debug('CentOS stype RPM detected')
+        self.spec = Specworker(os.path.dirname(spec_path), os.path.basename(spec_path))
+
+        self.gitversioned = False
+        try:
+            citag = self.vcs.get_citag()
+            self.gitversioned = True
+        except VcsError:
+            self.logger.debug("Project does not come from Git")
+        except:
+            raise
+
+        if self.spec.version == '%{_version}':
+            if self.gitversioned:
+                self.logger.debug("Using Git describe for package version")
+                self.useversion = citag
+            else:
+                self.logger.debug("Project not from Git. Using a.b package version")
+                self.useversion = 'a.b'
+        else:
+            self.logger.debug("Using spec definition for package version")
+            self.useversion = self.spec.version
+
+        self.packageversion = self.useversion
+        self.project_changed = self.get_project_changed()
+        self.nosrpm = nosrpm
+
+        if forcebuild:
+            self.mark_for_rebuild()
+
+        self.chrootscrub = chrootscrub
+
+    def get_project_changed(self):
+        """
+        Project status is read from status.txt file. Dirty git clones always require rebuild.
+        """
+        statusfile = os.path.join(self.project_workspace, 'status.txt')
+
+        if os.path.isfile(statusfile):
+            with open(statusfile, 'r') as filep:
+                previousprojectstatus = json.load(filep)
+            # Compare old values against new values
+            if not self.gitversioned:
+                self.logger.warning("Project %s is not git versioned. Forcing rebuild.", self.name)
+                return True
+            elif self.vcs.is_dirty():
+                self.logger.warning("Project %s contains unversioned changes and is \"dirty\". Forcing rebuild.", self.name)
+                return True
+            elif previousprojectstatus['sha'] != self.vcs.commitsha:
+                self.logger.info("Project %s log has new hash. Rebuild needed.", self.name)
+                return True
+            else:
+                self.logger.info("Project %s has NO new changes.", self.name)
+            return False
+        else:
+            # No configuration means that project has not been compiled
+            self.logger.warning("No previous build found for %s. Building initial version.", self.name)
+        return True
+
+    def store_project_status(self):
+        """ Write information of project version to status.txt
+        This can only be done for git versioned projects """
+        if self.gitversioned:
+            # Save information of the last compilation
+            statusfile = os.path.join(self.project_workspace, 'status.txt')
+            self.logger.debug("Updating status file %s", statusfile)
+
+            projectstatus = {"packageversion": self.packageversion,
+                             "sha": self.vcs.commitsha,
+                             "project": self.name}
+
+            with open(statusfile, 'w') as outfile:
+                json.dump(projectstatus, outfile)
+
+class GitProject(Project):
+    """ Projects cloned from Git version control system """
+    def __init__(self, name, workspace, conf, projects, builders, packagebuilder, masterargs):
+        forcebuild = masterargs.forcerebuild
+        chrootscrub = masterargs.scrub
+
+        Prettyprint().print_heading("Initializing %s from Git" % name, 60)
+        super(GitProject, self).__init__(name, workspace, projects, builders, packagebuilder)
+
+        # Values from build configuration file
+        self.projconf = {'url': conf.get_string(name, "url", mandatory=True),
+                         'ref': conf.get_string(name, "ref", mandatory=True),
+                         'spec': conf.get_string(name, "spec", mandatory=False, defaultvalue=None)}
+
+        # Do version control updates
+        self.directory_of_checkout = os.path.join(self.project_workspace,
+                                                  'checkout')
+        self.vcs = VersionControlSystem(self.directory_of_checkout)
+        self.vcs.update_git_project(self.projconf["url"], self.projconf["ref"])
+        self.useversion = self.vcs.get_citag()
+
+        # Read spec
+        try:
+            self.spec = Specworker(self.directory_of_checkout,
+                                   self.projconf["spec"])
+        except SpecError:
+            self.spec = Specworker(os.path.join(self.directory_of_checkout, "SPEC"), None)
+            self.centos_style = True
+
+        # Define what version shall be used in spec file
+        if self.spec.version == '%{_version}':
+            self.packageversion = self.vcs.get_citag()
+            self.logger.debug("Taking package version from VCS")
+        else:
+            self.packageversion = self.spec.version
+            self.logger.debug("Taking package version from spec")
+        self.logger.debug("Package version: %s", self.packageversion)
+
+        self.project_changed = self.get_project_changed()
+        if forcebuild:
+            self.mark_for_rebuild()
+
+        self.chrootscrub = chrootscrub
+
+    def get_project_changed(self):
+        """
+        Check if there has been changes in the project
+        if project has not been compiled -> return = True
+        if project has GIT/VCS changes   -> return = True
+        if project has not changed       -> return = False
+        """
+        statusfile = os.path.join(self.project_workspace, 'status.txt')
+
+        if os.path.isfile(statusfile):
+            with open(statusfile, 'r') as filep:
+                previousprojectstatus = json.load(filep)
+            # Compare old values against new values
+            if previousprojectstatus['url'] != self.projconf["url"] \
+                or previousprojectstatus['ref'] != self.projconf["ref"] \
+                or previousprojectstatus['sha'] != self.vcs.commitsha:
+                self.logger.debug("Returning info that changes found")
+                return True
+            else:
+                self.logger.debug("Returning info of NO changes")
+            return False
+        else:
+            # No configuration means that project has not been compiled
+            self.logger.debug("Doing first build of this project")
+        return True
+
+    def store_project_status(self):
+        """ Save information of the last compilation """
+        statusfile = os.path.join(self.project_workspace, 'status.txt')
+        self.logger.debug("Updating status file %s", statusfile)
+
+        projectstatus = {"url": self.projconf["url"],
+                         "ref": self.projconf["ref"],
+                         "spec": self.projconf["spec"],
+                         "packageversion": self.packageversion,
+                         "sha": self.vcs.commitsha,
+                         "project": self.name}
+
+        with open(statusfile, 'w') as outfile:
+            json.dump(projectstatus, outfile)
+
+class ProjectError(RpmbuilderError):
+
+    """ Exceptions originating from Project """
+    pass