X-Git-Url: https://gerrit.akraino.org/r/gitweb?p=ta%2Frpmbuilder.git;a=blobdiff_plain;f=stashmakebuild.py;fp=stashmakebuild.py;h=39f2c7bcb61d33b43920fc11f09ccd2b67926381;hp=0000000000000000000000000000000000000000;hb=876631a959303430aafc0be7897b086ee9b921fe;hpb=d8468e0423a9af0d3fd5bf30d45ebe18ba8b1801 diff --git a/stashmakebuild.py b/stashmakebuild.py new file mode 100755 index 0000000..39f2c7b --- /dev/null +++ b/stashmakebuild.py @@ -0,0 +1,158 @@ +#! /usr/bin/python -tt +# Copyright 2019 Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Safebuild is capable of doing backup and restore of workspace. +This ensures that package repository gets incremental updates and only +minimal set of packages are created """ + +import argparse +import logging +import os +import re +import subprocess +import tarfile + +from rpmbuilder.log import configure_logging +from makebuild import Build, BuildingError, ArgumentMakebuild +from stashworkspace import ArgumentRemote, Stasher + + +class Safebuild(Build): + + """ Safebuild extends capabilities of Build by providing backup and + restore on top of normal building activities """ + + def __init__(self, args): + super(Safebuild, self).__init__(args) + self.logger = logging.getLogger(__name__) + self.args = args + + self.backupfilename = "configuration.tar.gz" + self.remotehost = args.remotehost + self.remotedir = args.remotedir + + def start_safebuilding(self): + """ Starting a build requires checking of workspace, doing build + and then backing up the state of build system """ + self.logger.info("Starting safe building by using remote %s:%s", + self.remotehost, self.remotedir) + self.prepare_workspace() + self.update_building_blocks() + if self.start_building(): + self.backup_workspace() + if self.args.remotefunction == "pullpush": + stasher = Stasher() + stasher.push_workspace_to_remote(toserver=self.remotehost, + todirectory=self.remotedir, + workspace=self.args.workspace) + else: + self.logger.info("Skipping updating remote host with new packages") + + def tar_file_from_workspace(self, tar, sourcefile): + """ Archiving file from under workspace without + workspace parent directory structure """ + arcfile = os.path.join(self.args.workspace, sourcefile) + # Remove workspace directory from file + arcnamestring = re.sub(self.args.workspace, '', arcfile) + self.logger.debug("Archiving %s", arcfile) + tar.add(arcfile, arcname=arcnamestring) + + def backup_workspace(self): + """ Backup status files and repositories """ + backuptarfile = os.path.join(self.args.workspace, self.backupfilename) + self.logger.debug("Creating backup of configuration: %s", + backuptarfile) + with tarfile.open(backuptarfile, 'w:gz') as tar: + # Project settings + projdir = os.path.join(self.args.workspace, "projects") + for occurence in os.listdir(projdir): + statusfile = os.path.join(projdir, occurence, 'status.txt') + self.logger.info("Backing up file: %s", statusfile) + if os.path.isfile(statusfile): + self.tar_file_from_workspace(tar, statusfile) + else: + self.logger.warning("No %s for archiving", statusfile) + + def prepare_workspace(self): + """ Check that workspace contains correct beginning state """ + projectsdir = os.path.join(self.args.workspace, "projects") + if os.path.isdir(projectsdir): + self.logger.info("Using existing Workspace %s", self.args.workspace) + else: + self.logger.info("Trying to restore workspace from remote") + self.restore_workspace_from_remote(self.remotehost, self.remotedir) + + def restore_workspace_from_remote(self, fromserver, fromdirectory): + """ Retrieve and restore workspace from remote server """ + self.logger.info("Restoring workspace from remote %s:%s", fromserver, fromdirectory) + source = fromserver + ":" + fromdirectory + sshoptions = 'ssh -o stricthostkeychecking=no -o userknownhostsfile=/dev/null -o batchmode=yes -o passwordauthentication=no' + cmd = ["/usr/bin/rsync", + "--archive", + "-e", sshoptions, + os.path.join(source, "buildrepository"), + os.path.join(source, self.backupfilename), + os.path.join(self.args.workspace)] + self.logger.debug("Running: %s", str(cmd)) + try: + subprocess.check_call(cmd, shell=False, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as err: + if err.returncode == 23: + self.logger.info("There is no remote backup.. doing initial build") + return True + else: + raise BuildingError("Rsync from remote server failed with exit code %d" % err.returncode) + except: + raise BuildingError("Unexpected error") + + backupfile = os.path.join(self.args.workspace, self.backupfilename) + with tarfile.open(backupfile, 'r:gz') as tar: + tar.extractall(path=self.args.workspace) + self.logger.info("Workspace restored from %s:%s", + fromserver, + fromdirectory) + + +class ArgumentStashMakebuild(object): + """ Default arguments which are always needed """ + def __init__(self): + """ init """ + self.parser = argparse.ArgumentParser(description=''' + RPM building tool for continuous integration and development usage. + Uses remote host to retrieve and store incremental building state. + ''') + ArgumentMakebuild().set_arguments(self.parser) + ArgumentRemote().set_arguments(self.parser) + self.parser.add_argument("--remotefunction", + choices=["pull", "pullpush"], + default="pull", + help="With \"pullpush\" remote is used to fetch previous" + " build state and on succesful build remote is updated with" + " new packages. With \"pull\" packages are fetched but " + " remote is not updated on succesful builds. (Default: pull)") + + +def main(): + """ Read arguments and start processing build configuration """ + args = ArgumentStashMakebuild().parser.parse_args() + + debugfiletarget = os.path.join(args.workspace, 'debug.log') + configure_logging(args.verbose, debugfiletarget) + building = Safebuild(args) + building.start_safebuilding() + + +if __name__ == "__main__": + main()