3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 """ Module in charge of building a project """
22 from distutils.spawn import find_executable
25 from rpmbuilder.baseerror import RpmbuilderError
26 from rpmbuilder.prettyprinter import Prettyprint
28 PIGZ_INSTALLED = False
29 PBZIP2_INSTALLED = False
32 class Packagebuilding(object):
34 """ Object for building rpm files with mock """
36 def __init__(self, masterargs):
37 # Chroothousekeeping cleans chroot in case of mock errors. This should
38 # keep /var/lib/mock from growing too much
39 self.masterargs = masterargs
40 self.logger = logging.getLogger(__name__)
41 self.__check_tool_availability()
42 self.chroot_installed_rpms = []
44 if find_executable("pigz"):
47 self.logger.debug("pigz is available")
48 if find_executable("pbzip2"):
49 global PBZIP2_INSTALLED
50 PBZIP2_INSTALLED = True
51 self.logger.debug("pbzip2 is available")
52 if find_executable("pxz"):
55 self.logger.debug("pxz is available")
58 def __check_tool_availability():
59 """ Verify that user belongs to mock group for things to work """
60 username = pwd.getpwuid(os.getuid())[0]
61 cmd = "id " + username + "| grep \\(mock\\) > /dev/null"
62 if os.system(cmd) != 0:
63 raise PackagebuildingError("Mock tool requires user to "
64 "belong to group called mock")
67 def patch_specfile(self, origspecfile, outputdir, newversion, newrelease):
68 """ Spec file is patched with version information from git describe """
69 Prettyprint().print_heading("Patch spec", 50)
70 self.logger.info("Patching new spec from %s", origspecfile)
71 self.logger.debug(" - Version: %s", newversion)
72 self.logger.debug(" - Release: %s", newrelease)
74 specfilebasename = os.path.basename(origspecfile)
75 patchedspecfile = os.path.join(outputdir, specfilebasename)
76 self.logger.debug("Writing new spec file to %s", patchedspecfile)
78 with open(origspecfile, 'r') as filepin:
79 filepin_lines = filepin.readlines()
81 with open(patchedspecfile, 'w') as filepout:
82 for line in filepin_lines:
83 linestripped = line.strip()
84 if not linestripped.startswith("#"):
85 # Check if version could be patched
86 if linestripped.lower().startswith("version:"):
87 filepout.write("Version: " + newversion + '\n')
88 elif linestripped.lower().startswith("release:"):
89 filepout.write("Release: " + newrelease + '\n')
92 return patchedspecfile
94 def init_mock_chroot(self, resultdir, configdir, root):
96 Start a mock chroot where build requirements
97 can be installed before building
99 Prettyprint().print_heading("Mock init in " + root, 50)
101 self.clean_directory(resultdir)
103 mock_arg_resultdir = "--resultdir=" + resultdir
105 mocklogfile = resultdir + '/mock-init-' + root + '.log'
107 arguments = [mock_arg_resultdir,
109 self.run_mock_command(arguments, mocklogfile, configdir, root)
111 #Allow the builder to run sudo without terminal and without password
112 #This makes it possible to run disk image builder needed by ipa-builder
113 allow_sudo_str = "mockbuild ALL=(ALL) NOPASSWD: ALL"
114 notty_str = "Defaults:mockbuild !requiretty"
115 sudoers_file = "/etc/sudoers"
116 command = "grep \'%s\' %s || echo -e \'%s\n%s\' >> %s" %(allow_sudo_str, sudoers_file, allow_sudo_str, notty_str, sudoers_file)
117 arguments=["--chroot",
119 self.run_mock_command(arguments, mocklogfile, configdir, root)
123 def restore_local_repository(self, localdir, destdir, configdir, root, logfile):
125 Mock copying local yum repository to mock environment so that it can
126 be used during building of other RPM packages.
128 Prettyprint().print_heading("Restoring local repository", 50)
129 arguments = ["--copyin",
132 self.run_mock_command(arguments, logfile, configdir, root)
134 def mock_source_rpm(self, hostsourcedir, specfile, resultdir, configdir, root):
135 """ Mock SRPM file which can be used to build rpm """
136 Prettyprint().print_heading("Mock source rpm in " + root, 50)
137 self.logger.info("Build from:")
138 self.logger.info(" - source directory %s", hostsourcedir)
139 self.logger.info(" - spec %s", specfile)
141 self.clean_directory(resultdir)
143 mock_arg_resultdir = "--resultdir=" + resultdir
144 mock_arg_spec = "--spec=" + specfile
145 mock_arg_sources = "--sources=" + hostsourcedir
146 arguments = [mock_arg_resultdir,
148 "--no-cleanup-after",
153 mocklogfile = resultdir + '/mock.log'
154 self.run_mock_command(arguments, mocklogfile, configdir, root)
156 # Find source rpm and return the path
157 globstring = resultdir + '/*.src.rpm'
158 globmatches = glob.glob(globstring)
159 assert len(globmatches) == 1, "Too many source rpm files"
161 return globmatches[0]
163 def mock_rpm(self, sourcerpm, resultdir, configdir, root):
164 """ Mock RPM binary file from SRPM """
165 Prettyprint().print_heading("Mock rpm in " + root, 50)
166 self.logger.info("Building from:")
167 self.logger.info(" - source rpm %s", sourcerpm)
169 self.clean_directory(resultdir)
171 mock_arg_resultdir = "--resultdir=" + resultdir
172 arguments = [mock_arg_resultdir,
174 "--no-cleanup-after",
178 mocklogfile = resultdir + '/mock.log'
179 self.run_mock_command(arguments, mocklogfile, configdir, root)
181 self.logger.debug("RPM files build to: %s", resultdir)
184 def mock_rpm_from_archive(self, source_tar_packages, resultdir, configdir, root):
185 """ Mock rpm binary file straight from archive file """
186 self.clean_directory(resultdir)
188 # Copy source archive to chroot
189 chroot_sourcedir = "/builddir/build/SOURCES/"
190 self.copy_to_chroot(configdir, root, resultdir, source_tar_packages, chroot_sourcedir)
192 # Create rpm from source archive
193 sourcebasename = os.path.basename(source_tar_packages[0])
194 chrootsourcefile = os.path.join(chroot_sourcedir, sourcebasename)
196 Prettyprint().print_heading("Mock rpm in " + root, 50)
197 self.logger.info("Building from:")
198 self.logger.info(" - source archive %s", chrootsourcefile)
200 mock_arg_resultdir = "--resultdir=" + resultdir
201 rpmbuildcommand = "/usr/bin/rpmbuild --noclean -tb -v "
202 rpmbuildcommand += os.path.join(chroot_sourcedir, chrootsourcefile)
203 arguments = [mock_arg_resultdir,
206 mocklogfile = resultdir + '/mock-rpmbuild.log'
207 self.run_mock_command(arguments, mocklogfile, configdir, root)
209 def mock_rpm_from_filesystem(self, path, spec, resultdir, configdir, root, srpm_resultdir):
210 """ Mock rpm binary file straight from archive file """
211 self.clean_directory(resultdir)
212 # Copy source archive to chroot
213 chroot_sourcedir = "/builddir/build/"
214 self.copy_to_chroot(configdir, root, resultdir, [os.path.join(path, 'SPECS', spec)], os.path.join(chroot_sourcedir, 'SPECS'))
215 self.copy_to_chroot(configdir, root, resultdir, [os.path.join(path, 'SOURCES', f) for f in os.listdir(os.path.join(path, 'SOURCES'))], os.path.join(chroot_sourcedir, 'SOURCES'))
217 Prettyprint().print_heading("Mock rpm in " + root, 50)
218 mocklogfile = resultdir + '/mock-rpmbuild.log'
219 mock_arg_resultdir = "--resultdir=" + resultdir
220 arguments = [mock_arg_resultdir,
222 "chown -R root:root "+chroot_sourcedir]
223 self.run_mock_command(arguments, mocklogfile, configdir, root)
224 rpmbuildcommand = "/usr/bin/rpmbuild --noclean -ba -v "
225 rpmbuildcommand += os.path.join(chroot_sourcedir, 'SPECS', spec)
226 arguments = [mock_arg_resultdir,
229 mocklogfile = resultdir + '/mock-rpmbuild.log'
230 self.run_mock_command(arguments, mocklogfile, configdir, root)
232 arguments = ["--copyout",
233 "/builddir/build", resultdir+"/tmp/packages"]
234 mocklogfile = resultdir + '/mock-copyout.log'
235 self.run_mock_command(arguments, mocklogfile, configdir, root)
237 for filename in glob.glob(resultdir+"/tmp/packages/RPMS/*"):
238 shutil.move(filename, resultdir)
240 for filename in glob.glob(resultdir+"/tmp/packages/SRPMS/*"):
241 shutil.move(filename, srpm_resultdir)
243 def mock_wipe_buildroot(self, resultdir, configdir, root):
244 """ Wipe buildroot clean """
245 Prettyprint().print_heading("Wiping buildroot", 50)
246 arguments = ["--chroot",
247 "mkdir -pv /usr/localrepo && " \
248 "cp -v /builddir/build/RPMS/*.rpm /usr/localrepo/. ;" \
249 "rm -rf /builddir/build/{BUILD,RPMS,SOURCES,SPECS,SRPMS}/*"]
250 mocklogfile = resultdir + '/mock-wipe-buildroot.log'
251 self.run_mock_command(arguments, mocklogfile, configdir, root)
253 def update_local_repository(self, configdir, root):
254 Prettyprint().print_heading("Update repository " + root, 50)
256 arguments = ["--chroot",
257 "mkdir -pv /usr/localrepo && " \
258 "createrepo --update /usr/localrepo && yum clean expire-cache"]
259 self.run_mock_command(arguments, configdir+"/log", configdir, root)
261 def copy_to_chroot(self, configdir, root, resultdir, source_files, destination):
262 # Copy source archive to chroot
263 Prettyprint().print_heading("Copy source archive to " + root, 50)
264 self.logger.info(" - Copy from %s", source_files)
265 self.logger.info(" - Copy to %s", destination)
267 mock_arg_resultdir = "--resultdir=" + resultdir
268 arguments = [mock_arg_resultdir,
270 arguments.extend(source_files)
271 arguments.append(destination)
273 mocklogfile = resultdir + '/mock-copyin.log'
274 self.run_mock_command(arguments, mocklogfile, configdir, root)
276 def scrub_mock_chroot(self, configdir, root):
277 time_start = datetime.datetime.now()
278 Prettyprint().print_heading("Scrub mock chroot " + root, 50)
279 mock_clean_command = ["/usr/bin/mock",
280 "--configdir=" + configdir,
282 "--uniqueext=" + self.masterargs.uniqueext,
285 self.logger.info("Removing mock chroot.")
286 self.logger.debug(" ".join(mock_clean_command))
288 subprocess.check_call(mock_clean_command,
290 stderr=subprocess.STDOUT)
291 except subprocess.CalledProcessError as err:
292 raise PackagebuildingError("Mock chroot removal failed. Error code %s" % (err.returncode))
293 time_delta = datetime.datetime.now() - time_start
294 self.logger.debug('[mock-end] cmd="%s" took=%s (%s sec)', mock_clean_command, time_delta, time_delta.seconds)
296 def run_builddep(self, specfile, resultdir, configdir, root):
297 arguments = ["--copyin"]
298 arguments.append(specfile)
299 arguments.append("/builddir/"+os.path.basename(specfile))
301 mocklogfile = resultdir + '/mock-builddep.log'
302 self.run_mock_command(arguments, mocklogfile, configdir, root)
304 builddepcommand = "/usr/bin/yum-builddep -y "+"/builddir/"+os.path.basename(specfile)
305 arguments = ["--chroot",
307 mocklogfile = resultdir + '/mock-builddep.log'
308 return self.run_mock_command(arguments, mocklogfile, configdir, root, True) == 0
310 def run_mock_command(self, arguments, outputfile, configdir, root, return_error=False):
311 """ Mock binary rpm package """
312 mock_command = ["/usr/bin/mock",
313 "--configdir=" + configdir,
315 "--uniqueext=" + self.masterargs.uniqueext,
319 mock_command.extend(arguments)
320 if self.masterargs.mockarguments:
321 mock_command.extend([self.masterargs.mockarguments])
322 self.logger.info("Running mock. Log goes to %s", outputfile)
323 self.logger.debug('[mock-start] cmd="%s"', mock_command)
324 time_start = datetime.datetime.now()
325 self.logger.debug(" ".join(mock_command))
326 with open(outputfile, 'a') as filep:
328 mockproc = subprocess.Popen(mock_command,
330 stdout=subprocess.PIPE,
331 stderr=subprocess.STDOUT)
332 for line in iter(mockproc.stdout.readline, b''):
333 if self.masterargs.verbose:
334 self.logger.debug("mock-%s", line.rstrip('\n'))
336 _, stderr = mockproc.communicate() # wait for the subprocess to exit
338 return mockproc.returncode
339 if mockproc.returncode != 0:
340 raise Mockcommanderror(returncode=mockproc.returncode)
341 except Mockcommanderror as err:
342 self.logger.error("There was a failure during mocking")
343 if self.masterargs.scrub:
344 self.scrub_mock_chroot(configdir, root)
345 guidance_message = ""
347 mock_shell_command = ["/usr/bin/mock",
348 "--configdir=" + configdir,
350 "--uniqueext=" + self.masterargs.uniqueext,
352 guidance_message = ". To open mock shell, run the following: " + " ".join(mock_shell_command)
353 raise PackagebuildingError("Mock exited with value \"%s\". "
354 "Log for debuging: %s %s" % (err.returncode, outputfile, guidance_message))
356 raise PackagebuildingError("Mock executable not found. "
357 "Have you installed mock?")
360 time_delta = datetime.datetime.now() - time_start
361 self.logger.debug('[mock-end] cmd="%s" took=%s (%s sec)', mock_command, time_delta, time_delta.seconds)
363 def clean_directory(self, directory):
364 """ Make sure given directory exists and is clean """
365 if os.path.isdir(directory):
366 shutil.rmtree(directory)
367 os.makedirs(directory)
369 def tar_filter(self, tarinfo):
370 """ Filter git related and spec files away """
371 if tarinfo.name.endswith('.spec') or tarinfo.name.endswith('.git'):
372 self.logger.debug("Ignore %s", tarinfo.name)
374 self.logger.debug("Archiving %s", tarinfo.name)
377 def create_source_archive(self,
382 archive_file_extension):
384 Create tar file. Example helloworld-2.4.tar.gz
385 Tar file has naming <name>-<version>.tar.gz
387 Prettyprint().print_heading("Tar package creation", 50)
389 tar_file = package_name + '.' + 'tar'
390 # Directory where tar should be stored.
391 # Example /var/mybuild/workspace/sources
393 tarfilefullpath = os.path.join(outputdir, tar_file)
394 if os.path.isfile(tarfilefullpath) and not project_changed:
395 self.logger.info("Using cached %s", tarfilefullpath)
396 return tarfilefullpath
398 self.logger.info("Creating tar file %s", tarfilefullpath)
399 # sourcedir = /var/mybuild/helloworld/checkout
400 # sourcedir_dirname = /var/mybuild/helloworld
401 # sourcedir_basename = checkout
402 sourcedir_dirname = os.path.dirname(sourcedir)
404 os.chdir(sourcedir_dirname)
406 tar_params = ["tar", "cf", tarfilefullpath, "--directory="+os.path.dirname(sourcedir)]
407 tar_params = tar_params+["--exclude-vcs"]
408 tar_params = tar_params+["--transform=s/" + os.path.basename(sourcedir) + "/" + os.path.join(package_name) + "/"]
409 tar_params = tar_params+[os.path.basename(sourcedir)]
410 self.logger.debug("Running: %s", " ".join(tar_params))
411 ret = subprocess.call(tar_params)
413 raise PackagebuildingError("Tar error: %s", ret)
415 git_dir = os.path.join(os.path.basename(sourcedir), '.git')
416 if os.path.exists(git_dir):
417 tar_params = ["tar", "rf", tarfilefullpath, "--directory="+os.path.dirname(sourcedir)]
418 tar_params += ["--transform=s/" + os.path.basename(sourcedir) + "/" + os.path.join(package_name) + "/"]
419 tar_params += ['--dereference', git_dir]
420 self.logger.debug("Running: %s", " ".join(tar_params))
421 ret = subprocess.call(tar_params)
423 self.logger.warning("Git dir tar failed")
425 if archive_file_extension == "tar.gz":
430 resultfile = tarfilefullpath + '.gz'
432 raise PackagebuildingError("Unknown source archive format: %s" % archive_file_extension)
433 cmd += [tarfilefullpath]
434 self.logger.debug("Running: %s", " ".join(cmd))
435 ret = subprocess.call(cmd)
437 raise PackagebuildingError("Cmd error: %s", ret)
441 class Mockcommanderror(RpmbuilderError):
442 def __init__(self, returncode):
443 self.returncode = returncode
445 class PackagebuildingError(RpmbuilderError):
447 """ Exceptions originating from Builder and main level """