3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
16 Project is a subsystem which contains one spec file which
17 defines how it is build. Every project has one git
18 repository from where it is cloned from.
30 from rpmbuilder.baseerror import RpmbuilderError
31 from rpmbuilder.prettyprinter import Prettyprint
32 from rpmbuilder.rpmtools import Repotool, Specworker, RepotoolError, SpecError
33 from rpmbuilder.utils import find_files
34 from rpmbuilder.version_control import VersionControlSystem, VcsError
35 from rpmbuilder.get_sources import get_sources
38 class Project(object):
40 """ Instance of a project """
42 def __init__(self, name, workspace, projects, builders, packagebuilder, chrootscrub=True, nosrpm=False):
45 self.logger = logging.getLogger(__name__ + "." + self.name)
47 self.project_rebuild_needed = False
49 self.project_workspace = os.path.join(workspace,
53 self.projects = projects
54 self.builders = builders
55 self.directory_of_specpatch = os.path.join(self.project_workspace,
58 self.directory_of_sourcepackage = os.path.join(self.project_workspace,
61 self.directory_of_srpms = os.path.join(self.project_workspace,
64 self.directory_of_rpm = os.path.join(self.project_workspace,
67 self.directory_of_commonrepo = os.path.join(workspace,
70 self.directory_of_builder = self.builders.get_configdir()
72 self.__create_directories([self.directory_of_specpatch,
73 self.directory_of_srpms],
75 self.__create_directories([self.directory_of_sourcepackage],
78 self.packagebuilder = packagebuilder
80 self.chrootscrub = chrootscrub
82 for mockroot in builders.roots:
83 self.built[mockroot] = False
85 self.project_changed = False
88 self.useversion = None
89 self.directory_of_checkout = None
91 self.centos_style = False
92 self.buildrequires_downstream = set()
93 self.buildrequires_upstream = set()
95 def mark_for_rebuild(self):
96 """ Marking project for rebuild only if project has not changed """
97 if not self.project_changed:
98 self.logger.debug("Marking project %s for rebuild.", self.name)
99 self.project_rebuild_needed = True
101 def mark_downstream_for_rebuild(self, marked_for_build=None):
103 Recursively mark downstream projects for rebuilding.
104 Return set of projects marked for rebuild
106 if marked_for_build is None:
107 marked_for_build = set()
108 self.logger.debug("Marking downstream for rebuild in \"%s\"",
110 for project in self.who_buildrequires_me():
111 self.logger.debug("BuildRequires to \"%s\" found in \"%s\"",
113 if project in marked_for_build:
114 self.logger.debug("\"%s\" already marked for build", project)
115 elif self.projects[project].project_rebuild_needed:
116 self.logger.debug("\"%s\" already marked for rebuild", project)
118 self.projects[project].mark_for_rebuild()
119 marked_for_build.add(project)
120 # Check if downstream has downstream projects
121 tmpset = self.projects[project].mark_downstream_for_rebuild(
123 marked_for_build.update(tmpset)
124 return marked_for_build
126 def build_project(self, mockroot):
127 """ Do building of SRPM and RPM files """
128 time_start = datetime.datetime.now()
129 Prettyprint().print_heading("Build " + self.name, 60)
130 assert not self.built[mockroot], "Project already built"
133 if self.spec.version == '%{_version}':
134 self.logger.debug("patching spec file")
135 self.logger.debug("Version in spec is going to be %s", self.useversion)
138 userelease = rpm.next_release_of_package(
139 os.path.join(self.directory_of_commonrepo,
140 self.builders.roots[0],
145 self.logger.debug("Release in spec is going to be %s", userelease)
147 specfile = self.packagebuilder.patch_specfile(self.spec.specfilefullpath,
148 self.directory_of_specpatch,
152 self.logger.debug("Skipping spec patching")
153 specfile = self.spec.specfilefullpath
156 self.logger.debug("Starting building in root \"%s\"", mockroot)
157 if self.centos_style:
158 shutil.rmtree(self.directory_of_sourcepackage)
159 ignore_git = shutil.ignore_patterns('.git')
160 shutil.copytree(self.directory_of_checkout, self.directory_of_sourcepackage, ignore=ignore_git)
161 sources_key = 'CENTOS_SOURCES'
162 if sources_key not in os.environ:
163 raise RpmbuilderError('Cannot build CentOS style RPM, %s not defined in the environment' % sources_key)
164 get_sources(self.directory_of_sourcepackage, os.environ[sources_key].split(','), self.logger)
165 self.create_rpm_from_filesystem(self.directory_of_sourcepackage, mockroot)
167 list_of_source_packages = self.get_source_package()
168 self.create_rpm_from_archive(list_of_source_packages, mockroot)
170 self.get_source_package()
171 # Create source RPM file
172 sourcerpm = self.get_source_rpm(self.directory_of_sourcepackage, specfile, mockroot)
174 # Create final RPM file(s)
175 self.create_rpm_from_srpm(sourcerpm, mockroot)
177 # Mark build completed
178 self.built[mockroot] = True
179 time_delta = datetime.datetime.now() - time_start
180 self.logger.info('Building success: %s (took %s [%s sec])', self.name, time_delta, time_delta.seconds)
182 # We wipe buildroot of previously built rpm, source etc. packages
183 # This is custom cleaning which does not remove chroot
184 self.packagebuilder.mock_wipe_buildroot(self.project_workspace, self.directory_of_builder, mockroot)
186 def pull_source_packages(self, target_dir):
187 cmd = ['/usr/bin/spectool', '-d', 'KVERSION a.b', '-g', '--directory', target_dir, self.spec.specfilefullpath]
188 self.logger.info('Pulling source packages: %s', cmd)
190 subprocess.check_call(cmd, shell=False)
191 self.logger.info('Pulling source packages ok')
192 except OSError as err:
193 self.logger.info('Pulling source packages nok %s', err.strerror)
194 raise RepotoolError("Calling of command spectool caused: \"%s\"" % err.strerror)
196 self.logger.info('Pulling source packages nok ??', err.strerror)
197 raise RepotoolError("There was error pulling source content")
199 def get_source_package(self):
200 # Produce source package
201 source_package_list = []
202 for source_file_hit in self.spec.source_files:
203 self.logger.info("Acquiring source file \"%s\"", source_file_hit)
204 if re.match(r'^(http[s]*|ftp)://', source_file_hit):
205 self.logger.info("PULL %s", self.directory_of_sourcepackage)
206 self.pull_source_packages(self.directory_of_sourcepackage)
207 source_package_list.append(self.directory_of_sourcepackage + '/' + source_file_hit.split('/')[-1])
209 for subdir in ["", "SOURCES"]:
210 if os.path.isfile(os.path.join(self.directory_of_checkout, subdir, source_file_hit)):
211 shutil.copy(os.path.join(self.directory_of_checkout, subdir, source_file_hit), self.directory_of_sourcepackage)
212 source_package_list.append(os.path.join(self.directory_of_sourcepackage, source_file_hit))
215 tarname = self.spec.name + '-' + self.useversion
216 source_package_list.append(self.packagebuilder.create_source_archive(tarname,
217 self.directory_of_checkout,
218 self.directory_of_sourcepackage,
219 self.project_changed,
220 self.spec.source_file_extension))
222 for patch_file_hit in self.spec.patch_files:
223 self.logger.info("Copying %s to directory %s", patch_file_hit, self.directory_of_sourcepackage)
224 for subdir in ["", "SOURCES"]:
225 if os.path.isfile(os.path.join(self.directory_of_checkout, subdir, patch_file_hit)):
226 shutil.copy(os.path.join(self.directory_of_checkout, subdir, patch_file_hit), self.directory_of_sourcepackage)
229 raise ProjectError("Spec file lists patch \"%s\" but no file found" % patch_file_hit)
230 return source_package_list
233 def get_source_rpm(self, hostsourcedir, specfile, mockroot):
234 return self.packagebuilder.mock_source_rpm(hostsourcedir,
236 self.directory_of_srpms,
237 self.directory_of_builder,
240 def create_rpm_from_srpm(self, sourcerpm, mockroot):
241 directory_of_rpm = os.path.join(self.directory_of_rpm, mockroot)
242 self.packagebuilder.mock_rpm(sourcerpm,
244 self.directory_of_builder,
246 # Delete duplicated src.rpm which is returned by rpm creation
247 os.remove(os.path.join(directory_of_rpm, os.path.basename(sourcerpm)))
249 def create_rpm_from_archive(self, source_tar_packages, mockroot):
250 directory_of_rpm = os.path.join(self.directory_of_rpm, mockroot)
251 self.packagebuilder.mock_rpm_from_archive(source_tar_packages, directory_of_rpm, self.directory_of_builder, mockroot)
253 def create_rpm_from_filesystem(self, path, mockroot):
254 directory_of_rpm = os.path.join(self.directory_of_rpm, mockroot)
255 self.packagebuilder.mock_rpm_from_filesystem(path,
256 self.spec.specfilename,
258 self.directory_of_builder,
260 self.directory_of_srpms)
262 def list_buildproducts_for_mockroot(self, mockroot):
263 """ List both source and final rpm packages """
266 for occurence in os.listdir(os.path.join(self.directory_of_rpm, mockroot)):
267 if occurence.endswith(".rpm"):
268 rpmlist.append(occurence)
269 for occurence in os.listdir(self.directory_of_srpms):
270 if occurence.endswith(".src.rpm"):
271 srpmlist.append(occurence)
272 return rpmlist, srpmlist
274 def resolve_dependencies(self, mockroot):
275 return self.packagebuilder.run_builddep(self.spec.specfilefullpath,
276 self.directory_of_srpms,
277 self.directory_of_builder,
280 def store_build_products(self, commonrepo):
281 """ Save build products under common yum repository """
282 self.__create_directories([commonrepo])
283 for mockroot in self.builders.roots:
284 srpmtargetdir = os.path.join(commonrepo, mockroot, 'srpm')
285 rpmtargetdir = os.path.join(commonrepo, mockroot, 'rpm')
286 self.__create_directories([srpmtargetdir, rpmtargetdir])
287 (rpmlist, srpmlist) = self.list_buildproducts_for_mockroot(mockroot)
288 build_product_dir = os.path.join(self.directory_of_rpm, mockroot)
289 self.logger.debug("Hard linking %s rpm packages to %s", self.name, rpmtargetdir)
290 for rpm_file in rpmlist:
291 self.logger.info("Hard linking %s", rpm_file)
293 os.link(os.path.join(build_product_dir, rpm_file),
294 os.path.join(rpmtargetdir, os.path.basename(rpm_file)))
297 self.logger.debug("Hard linking %s srpm packages to %s", self.name, srpmtargetdir)
298 for srpm_file in srpmlist:
299 self.logger.info("Hard linking %s", srpm_file)
301 os.link(os.path.join(self.directory_of_srpms, srpm_file),
302 os.path.join(srpmtargetdir, srpm_file))
306 # Store info of latest build
307 self.store_project_status()
310 def who_buildrequires_me(self):
312 Return a list of projects which directly buildrequires this project (non-recursive)
314 downstream_projects = set()
315 # Loop through my packages
316 for package in self.spec.packages:
317 # Loop other projects and check if they need me
318 # To need me, they have my package in buildrequires
319 for project in self.projects:
320 if package in self.projects[project].spec.buildrequires:
321 self.logger.debug("Found dependency in {}: my package {} is required by project {}".format(self.name, package, project))
322 self.projects[project].buildrequires_upstream.add(self.name)
323 self.projects[self.name].buildrequires_downstream.add(project)
324 downstream_projects.add(project)
325 return downstream_projects
328 def who_requires_me(self, recursive=False, depth=0):
330 Return a list of projects which have requirement to this project
333 self.logger.warn("Hit infinite recursion limiter in {}".format(self.name))
335 # Loop through my packages
336 downstream_projects = set()
337 for package in self.spec.packages:
338 # Loop other projects and check if they need me
339 # To need me, they have my package in buildrequires or requires
340 for project in self.projects:
341 if package in self.projects[project].spec.buildrequires \
342 or package in self.projects[project].spec.requires:
343 downstream_projects.add(project)
345 downstream_projects.update(
346 self.projects[project].who_requires_me(True, depth+1))
347 self.logger.debug("Returning who_requires_me for %s: %s",
348 self.name, ', '.join(downstream_projects))
349 return downstream_projects
351 def get_project_changed(self):
352 raise NotImplementedError
354 def store_project_status(self):
355 raise NotImplementedError
357 def __create_directories(self, directories, verify_empty=False):
358 """ Directory creation """
359 for directory in directories:
360 if os.path.isdir(directory):
361 if verify_empty and os.listdir(directory) != []:
362 self.logger.debug("Cleaning directory %s", directory)
363 globstring = directory + "/*"
364 files = glob.glob(globstring)
365 for foundfile in files:
366 self.logger.debug("Removing file %s", foundfile)
369 self.logger.debug("Creating directory %s", directory)
371 os.makedirs(directory)
376 class LocalMountProject(Project):
377 """ Projects coming from local disk mount """
378 def __init__(self, name, directory, workspace, projects, builders, packagebuilder, masterargs, spec_path):
379 chrootscrub = masterargs.scrub
380 nosrpm = masterargs.nosrpm
381 forcebuild = masterargs.forcerebuild
383 Prettyprint().print_heading("Initializing %s from disk" % name, 60)
384 super(LocalMountProject, self).__init__(name, workspace, projects, builders, packagebuilder)
386 if not os.path.isdir(directory):
387 raise ProjectError("No directory %s found", directory)
389 self.vcs = VersionControlSystem(directory)
390 self.directory_of_checkout = directory
392 # Values from build configuration file
395 if len(list(find_files(directory, r'\..+\.metadata$'))) > 0 and \
396 os.path.isdir(os.path.join(directory, 'SOURCES')) and \
397 os.path.isdir(os.path.join(directory, 'SPECS')):
398 self.centos_style = True
399 self.logger.debug('CentOS stype RPM detected')
400 self.spec = Specworker(os.path.dirname(spec_path), os.path.basename(spec_path))
402 self.gitversioned = False
404 citag = self.vcs.get_citag()
405 self.gitversioned = True
407 self.logger.debug("Project does not come from Git")
411 if self.spec.version == '%{_version}':
412 if self.gitversioned:
413 self.logger.debug("Using Git describe for package version")
414 self.useversion = citag
416 self.logger.debug("Project not from Git. Using a.b package version")
417 self.useversion = 'a.b'
419 self.logger.debug("Using spec definition for package version")
420 self.useversion = self.spec.version
422 self.packageversion = self.useversion
423 self.project_changed = self.get_project_changed()
427 self.mark_for_rebuild()
429 self.chrootscrub = chrootscrub
431 def get_project_changed(self):
433 Project status is read from status.txt file. Dirty git clones always require rebuild.
435 statusfile = os.path.join(self.project_workspace, 'status.txt')
437 if os.path.isfile(statusfile):
438 with open(statusfile, 'r') as filep:
439 previousprojectstatus = json.load(filep)
440 # Compare old values against new values
441 if not self.gitversioned:
442 self.logger.warning("Project %s is not git versioned. Forcing rebuild.", self.name)
444 elif self.vcs.is_dirty():
445 self.logger.warning("Project %s contains unversioned changes and is \"dirty\". Forcing rebuild.", self.name)
447 elif previousprojectstatus['sha'] != self.vcs.commitsha:
448 self.logger.info("Project %s log has new hash. Rebuild needed.", self.name)
451 self.logger.info("Project %s has NO new changes.", self.name)
454 # No configuration means that project has not been compiled
455 self.logger.warning("No previous build found for %s. Building initial version.", self.name)
458 def store_project_status(self):
459 """ Write information of project version to status.txt
460 This can only be done for git versioned projects """
461 if self.gitversioned:
462 # Save information of the last compilation
463 statusfile = os.path.join(self.project_workspace, 'status.txt')
464 self.logger.debug("Updating status file %s", statusfile)
466 projectstatus = {"packageversion": self.packageversion,
467 "sha": self.vcs.commitsha,
468 "project": self.name}
470 with open(statusfile, 'w') as outfile:
471 json.dump(projectstatus, outfile)
473 class GitProject(Project):
474 """ Projects cloned from Git version control system """
475 def __init__(self, name, workspace, conf, projects, builders, packagebuilder, masterargs):
476 forcebuild = masterargs.forcerebuild
477 chrootscrub = masterargs.scrub
479 Prettyprint().print_heading("Initializing %s from Git" % name, 60)
480 super(GitProject, self).__init__(name, workspace, projects, builders, packagebuilder)
482 # Values from build configuration file
483 self.projconf = {'url': conf.get_string(name, "url", mandatory=True),
484 'ref': conf.get_string(name, "ref", mandatory=True),
485 'spec': conf.get_string(name, "spec", mandatory=False, defaultvalue=None)}
487 # Do version control updates
488 self.directory_of_checkout = os.path.join(self.project_workspace,
490 self.vcs = VersionControlSystem(self.directory_of_checkout)
491 self.vcs.update_git_project(self.projconf["url"], self.projconf["ref"])
492 self.useversion = self.vcs.get_citag()
496 self.spec = Specworker(self.directory_of_checkout,
497 self.projconf["spec"])
499 self.spec = Specworker(os.path.join(self.directory_of_checkout, "SPEC"), None)
500 self.centos_style = True
502 # Define what version shall be used in spec file
503 if self.spec.version == '%{_version}':
504 self.packageversion = self.vcs.get_citag()
505 self.logger.debug("Taking package version from VCS")
507 self.packageversion = self.spec.version
508 self.logger.debug("Taking package version from spec")
509 self.logger.debug("Package version: %s", self.packageversion)
511 self.project_changed = self.get_project_changed()
513 self.mark_for_rebuild()
515 self.chrootscrub = chrootscrub
517 def get_project_changed(self):
519 Check if there has been changes in the project
520 if project has not been compiled -> return = True
521 if project has GIT/VCS changes -> return = True
522 if project has not changed -> return = False
524 statusfile = os.path.join(self.project_workspace, 'status.txt')
526 if os.path.isfile(statusfile):
527 with open(statusfile, 'r') as filep:
528 previousprojectstatus = json.load(filep)
529 # Compare old values against new values
530 if previousprojectstatus['url'] != self.projconf["url"] \
531 or previousprojectstatus['ref'] != self.projconf["ref"] \
532 or previousprojectstatus['sha'] != self.vcs.commitsha:
533 self.logger.debug("Returning info that changes found")
536 self.logger.debug("Returning info of NO changes")
539 # No configuration means that project has not been compiled
540 self.logger.debug("Doing first build of this project")
543 def store_project_status(self):
544 """ Save information of the last compilation """
545 statusfile = os.path.join(self.project_workspace, 'status.txt')
546 self.logger.debug("Updating status file %s", statusfile)
548 projectstatus = {"url": self.projconf["url"],
549 "ref": self.projconf["ref"],
550 "spec": self.projconf["spec"],
551 "packageversion": self.packageversion,
552 "sha": self.vcs.commitsha,
553 "project": self.name}
555 with open(statusfile, 'w') as outfile:
556 json.dump(projectstatus, outfile)
558 class ProjectError(RpmbuilderError):
560 """ Exceptions originating from Project """