stashmakebuild.py

   1 #! /usr/bin/python -tt
   2 # Copyright 2019 Nokia
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 """ Safebuild is capable of doing backup and restore of workspace.
  17 This ensures that package repository gets incremental updates and only
  18 minimal set of packages are created """
  19
  20 import argparse
  21 import logging
  22 import os
  23 import re
  24 import subprocess
  25 import tarfile
  26
  27 from rpmbuilder.log import configure_logging
  28 from makebuild import Build, BuildingError, ArgumentMakebuild
  29 from stashworkspace import ArgumentRemote, Stasher
  30
  31
  32 class Safebuild(Build):
  33
  34     """ Safebuild extends capabilities of Build by providing backup and
  35     restore on top of normal building activities """
  36
  37     def __init__(self, args):
  38         super(Safebuild, self).__init__(args)
  39         self.logger = logging.getLogger(__name__)
  40         self.args = args
  41
  42         self.backupfilename = "configuration.tar.gz"
  43         self.remotehost = args.remotehost
  44         self.remotedir = args.remotedir
  45
  46     def start_safebuilding(self):
  47         """ Starting a build requires checking of workspace, doing build
  48         and then backing up the state of build system """
  49         self.logger.info("Starting safe building by using remote %s:%s",
  50                          self.remotehost, self.remotedir)
  51         self.prepare_workspace()
  52         self.update_building_blocks()
  53         if self.start_building():
  54             self.backup_workspace()
  55             if self.args.remotefunction == "pullpush":
  56                 stasher = Stasher()
  57                 stasher.push_workspace_to_remote(toserver=self.remotehost,
  58                                                  todirectory=self.remotedir,
  59                                                  workspace=self.args.workspace)
  60             else:
  61                 self.logger.info("Skipping updating remote host with new packages")
  62
  63     def tar_file_from_workspace(self, tar, sourcefile):
  64         """ Archiving file from under workspace without
  65         workspace parent directory structure """
  66         arcfile = os.path.join(self.args.workspace, sourcefile)
  67         # Remove workspace directory from file
  68         arcnamestring = re.sub(self.args.workspace, '', arcfile)
  69         self.logger.debug("Archiving %s", arcfile)
  70         tar.add(arcfile, arcname=arcnamestring)
  71
  72     def backup_workspace(self):
  73         """ Backup status files and repositories """
  74         backuptarfile = os.path.join(self.args.workspace, self.backupfilename)
  75         self.logger.debug("Creating backup of configuration: %s",
  76                           backuptarfile)
  77         with tarfile.open(backuptarfile, 'w:gz') as tar:
  78             # Project settings
  79             projdir = os.path.join(self.args.workspace, "projects")
  80             for occurence in os.listdir(projdir):
  81                 statusfile = os.path.join(projdir, occurence, 'status.txt')
  82                 self.logger.info("Backing up file: %s", statusfile)
  83                 if os.path.isfile(statusfile):
  84                     self.tar_file_from_workspace(tar, statusfile)
  85                 else:
  86                     self.logger.warning("No %s for archiving", statusfile)
  87
  88     def prepare_workspace(self):
  89         """ Check that workspace contains correct beginning state """
  90         projectsdir = os.path.join(self.args.workspace, "projects")
  91         if os.path.isdir(projectsdir):
  92             self.logger.info("Using existing Workspace %s", self.args.workspace)
  93         else:
  94             self.logger.info("Trying to restore workspace from remote")
  95             self.restore_workspace_from_remote(self.remotehost, self.remotedir)
  96
  97     def restore_workspace_from_remote(self, fromserver, fromdirectory):
  98         """ Retrieve and restore workspace from remote server """
  99         self.logger.info("Restoring workspace from remote %s:%s", fromserver, fromdirectory)
 100         source = fromserver + ":" + fromdirectory
 101         sshoptions = 'ssh -o stricthostkeychecking=no -o userknownhostsfile=/dev/null -o batchmode=yes -o passwordauthentication=no'
 102         cmd = ["/usr/bin/rsync",
 103                "--archive",
 104                "-e", sshoptions,
 105                os.path.join(source, "buildrepository"),
 106                os.path.join(source, self.backupfilename),
 107                os.path.join(self.args.workspace)]
 108         self.logger.debug("Running: %s", str(cmd))
 109         try:
 110             subprocess.check_call(cmd, shell=False, stderr=subprocess.STDOUT)
 111         except subprocess.CalledProcessError as err:
 112             if err.returncode == 23:
 113                 self.logger.info("There is no remote backup.. doing initial build")
 114                 return True
 115             else:
 116                 raise BuildingError("Rsync from remote server failed with exit code %d" % err.returncode)
 117         except:
 118             raise BuildingError("Unexpected error")
 119
 120         backupfile = os.path.join(self.args.workspace, self.backupfilename)
 121         with tarfile.open(backupfile, 'r:gz') as tar:
 122             tar.extractall(path=self.args.workspace)
 123         self.logger.info("Workspace restored from %s:%s",
 124                          fromserver,
 125                          fromdirectory)
 126
 127
 128 class ArgumentStashMakebuild(object):
 129     """ Default arguments which are always needed """
 130     def __init__(self):
 131         """ init """
 132         self.parser = argparse.ArgumentParser(description='''
 133             RPM building tool for continuous integration and development usage.
 134             Uses remote host to retrieve and store incremental building state.
 135         ''')
 136         ArgumentMakebuild().set_arguments(self.parser)
 137         ArgumentRemote().set_arguments(self.parser)
 138         self.parser.add_argument("--remotefunction",
 139                                  choices=["pull", "pullpush"],
 140                                  default="pull",
 141                                  help="With \"pullpush\" remote is used to fetch previous"
 142                                  " build state and on succesful build remote  is updated with"
 143                                  " new packages. With \"pull\" packages are fetched but "
 144                                  " remote is not updated on succesful builds. (Default: pull)")
 145
 146
 147 def main():
 148     """ Read arguments and start processing build configuration """
 149     args = ArgumentStashMakebuild().parser.parse_args()
 150
 151     debugfiletarget = os.path.join(args.workspace, 'debug.log')
 152     configure_logging(args.verbose, debugfiletarget)
 153     building = Safebuild(args)
 154     building.start_safebuilding()
 155
 156
 157 if __name__ == "__main__":
 158     main()