src/remoteinstaller/installer/install.py

   1 # Copyright 2019 Nokia
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import logging
  16 import sys
  17 import argparse
  18 import subprocess
  19 import os
  20 import importlib
  21 import time
  22 import distutils.util
  23
  24 from yaml import load
  25 from netaddr import IPNetwork
  26 from netaddr import IPAddress
  27
  28 import hw_detector.hw_detect_lib as hw_detect
  29 from hw_detector.hw_exception import HWException
  30 from remoteinstaller.installer.bmc_management.bmctools import BMCException
  31 from remoteinstaller.installer.catfile import CatFile
  32 from remoteinstaller.installer.catfile import CatFileException
  33
  34 class InstallException(Exception):
  35     pass
  36
  37 class Installer(object):
  38     SSH_OPTS = ('-o StrictHostKeyChecking=no '
  39                 '-o UserKnownHostsFile=/dev/null '
  40                 '-o ServerAliveInterval=60')
  41
  42     def __init__(self, callback_server, callback_uuid, yaml, logdir, args=None):
  43         self._callback_server = callback_server
  44         self._callback_uuid = callback_uuid
  45         self._yaml_path = yaml
  46         self._uc = self._read_user_config(self._yaml_path)
  47         self._logdir = logdir
  48
  49         self._boot_iso_path = None
  50         self._iso_url = None
  51         self._callback_url = None
  52         self._client_key = None
  53         self._client_cert = None
  54         self._ca_cert = None
  55         self._own_ip = None
  56         self._tag = None
  57         self._disable_bmc_initial_reset = False
  58         self._disable_other_bmc_reset = True
  59
  60         if args:
  61             self._set_arguments(args)
  62
  63         self._vip = None
  64         self._first_controller = None
  65         self._first_controller_ip = None
  66         self._first_controller_bmc = None
  67
  68         self._define_first_controller()
  69
  70     def _get_bool_arg(self, args, arg, default):
  71         if hasattr(args, arg):
  72             arg_value = vars(args)[arg]
  73             if not isinstance(arg_value, bool):
  74                 if isinstance(arg_value, basestring):
  75                     try:
  76                         arg_value = bool(distutils.util.strtobool(arg_value))
  77                         return arg_value
  78                     except ValueError:
  79                         logging.warning('Invalid value for %s: %s', arg, arg_value)
  80             else:
  81                 return arg_value
  82
  83         return default
  84
  85     def _set_arguments(self, args):
  86         self._disable_bmc_initial_reset = self._get_bool_arg(args, 'disable_bmc_initial_reset', self._disable_bmc_initial_reset)
  87         self._disable_other_bmc_reset = self._get_bool_arg(args, 'disable_other_bmc_reset', self._disable_other_bmc_reset)
  88
  89         self._boot_iso_path = args.boot_iso
  90         self._iso_url = args.iso
  91         self._callback_url = args.callback_url
  92         self._client_key = args.client_key
  93         self._client_cert = args.client_cert
  94         self._ca_cert = args.ca_cert
  95         self._own_ip = args.host_ip
  96         self._tag = args.tag
  97
  98     @staticmethod
  99     def _read_user_config(config_file_path):
 100         logging.debug('Read user config from %s', config_file_path)
 101
 102         try:
 103             with open(config_file_path, 'r') as f:
 104                 y = load(f)
 105
 106             return y
 107         except Exception as ex:
 108             raise InstallException(str(ex))
 109
 110     @staticmethod
 111     def _execute_shell(command, desc=''):
 112         logging.debug('Execute %s with command: %s', desc, command)
 113
 114         p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 115         out, _ = p.communicate()
 116         if p.returncode:
 117             logging.warning('Failed to %s: %s (rc=%s)', desc, out, p.returncode)
 118             raise InstallException('Failed to {}'.format(desc))
 119
 120         return (p.returncode, out)
 121
 122     def _attach_iso_as_virtual_media(self, file_list):
 123         logging.info('Attach ISO as virtual media')
 124
 125         nfs_mount = os.path.dirname(self._boot_iso_path)
 126         boot_iso_filename = os.path.basename(self._boot_iso_path)
 127         patched_iso_filename = '{}/{}_{}'.format(nfs_mount, self._tag, boot_iso_filename)
 128
 129         self._patch_iso(patched_iso_filename, file_list)
 130
 131         self._first_controller_bmc.attach_virtual_cd(self._own_ip, nfs_mount, os.path.basename(patched_iso_filename))
 132
 133     def _setup_bmc_for_node(self, hw):
 134         bmc_log_path = '{}/{}.log'.format(self._logdir, hw)
 135
 136         host = self._uc['hosts'][hw]['hwmgmt']['address']
 137         user = self._uc['hosts'][hw]['hwmgmt']['user']
 138         passwd = self._uc['hosts'][hw]['hwmgmt']['password']
 139         priv_level = self._uc['hosts'][hw]['hwmgmt'].get('priv_level', 'ADMINISTRATOR')
 140
 141         try:
 142             hw_data = hw_detect.get_hw_data(host, user, passwd, priv_level, False)
 143         except HWException as e:
 144             error = "Hardware not detected for {}: {}".format(hw, str(e))
 145             logging.error(error)
 146             raise BMCException(error)
 147
 148         logging.debug("Hardware belongs to %s product family", (hw_data['product_family']))
 149         if 'Unknown' in hw_data['product_family']:
 150             error = "Hardware not detected for %s" % hw
 151             logging.error(error)
 152             raise BMCException(error)
 153
 154         bmc_mod_name = 'remoteinstaller.installer.bmc_management.{}'.format(hw_data['product_family'].lower())
 155         bmc_mod = importlib.import_module(bmc_mod_name)
 156         bmc_class = getattr(bmc_mod, hw_data['product_family'])
 157         bmc = bmc_class(host, user, passwd, priv_level, bmc_log_path)
 158         bmc.set_host_name(hw)
 159
 160         return bmc
 161
 162     def _define_first_controller(self):
 163         for hw in sorted(self._uc['hosts']):
 164             logging.debug('HW node name is %s', hw)
 165
 166             if 'controller' in self._uc['hosts'][hw]['service_profiles'] or \
 167             'caas_master' in self._uc['hosts'][hw]['service_profiles']:
 168                 self._first_controller = hw
 169                 break
 170
 171         logging.info('First controller is %s', self._first_controller)
 172         self._first_controller_bmc = self._setup_bmc_for_node(self._first_controller)
 173
 174         domain = self._uc['hosts'][self._first_controller].get('network_domain')
 175         extnet = self._uc['networking']['infra_external']['network_domains'][domain]
 176
 177         first_ip = extnet['ip_range_start']
 178         self._vip = str(IPAddress(first_ip))
 179
 180         pre_allocated_ips = self._uc['hosts'][self._first_controller].get('pre_allocated_ips', None)
 181         if pre_allocated_ips:
 182             pre_allocated_infra_external_ip = pre_allocated_ips.get('infra_external', None)
 183             self._first_controller_ip = str(IPAddress(pre_allocated_infra_external_ip))
 184
 185         if not self._first_controller_ip:
 186             self._first_controller_ip = str(IPAddress(first_ip)+1)
 187
 188     def _create_cloud_config(self):
 189         logging.info('Create network config file')
 190
 191         domain = self._uc['hosts'][self._first_controller].get('network_domain')
 192         extnet = self._uc['networking']['infra_external']['network_domains'][domain]
 193
 194         vlan = extnet.get('vlan')
 195         gateway = extnet['gateway']
 196         dns = self._uc['networking']['dns'][0]
 197         cidr = extnet['cidr']
 198         prefix = IPNetwork(cidr).prefixlen
 199
 200         controller_network_profile = self._uc['hosts'][self._first_controller]['network_profiles'][0]
 201         mappings = self._uc['network_profiles'][controller_network_profile]['interface_net_mapping']
 202         for interface in mappings:
 203             if 'infra_external' in mappings[interface]:
 204                 infra_external_interface = interface
 205                 break
 206
 207         if infra_external_interface.startswith('bond'):
 208             bonds = self._uc['network_profiles'][controller_network_profile]['bonding_interfaces']
 209             device = bonds[infra_external_interface][0]
 210         else:
 211             device = infra_external_interface
 212
 213         # TODO
 214         # ROOTFS_DISK
 215
 216         logging.debug('VLAN=%s', vlan)
 217         logging.debug('DEV=%s', device)
 218         logging.debug('IP=%s/%s', self._first_controller_ip, prefix)
 219         logging.debug('DGW=%s', gateway)
 220         logging.debug('NAMESERVER=%s', dns)
 221         logging.debug('ISO_URL="%s"', self._iso_url)
 222
 223         network_config_filename = '{}/network_config'.format(self._logdir)
 224         with open(network_config_filename, 'w') as f:
 225             if vlan:
 226                 f.write('VLAN={}\n'.format(vlan))
 227             f.write('DEV={}\n'.format(device))
 228             f.write('IP={}/{}\n'.format(self._first_controller_ip, prefix))
 229             f.write('DGW={}\n'.format(gateway))
 230             f.write('NAMESERVER={}\n'.format(dns))
 231             f.write('\n')
 232             f.write('ISO_URL="{}"'.format(self._iso_url))
 233
 234         return network_config_filename
 235
 236     def _create_callback_file(self):
 237         logging.debug('CALLBACK_URL="%s"', self._callback_url)
 238
 239         callback_url_filename = '{}/callback_url'.format(self._logdir)
 240         with open(callback_url_filename, 'w') as f:
 241             f.write(self._callback_url)
 242
 243         return callback_url_filename
 244
 245     def _patch_iso(self, iso_target, file_list):
 246         logging.info('Patch boot ISO')
 247         logging.debug('Original ISO: %s', self._boot_iso_path)
 248         logging.debug('Target ISO: %s', iso_target)
 249
 250         file_list_str = ' '.join(file_list)
 251         logging.debug('Files to add: %s', file_list_str)
 252
 253         self._execute_shell('/usr/bin/patchiso.sh {} {} {}'.format(self._boot_iso_path,
 254                                                                    iso_target,
 255                                                                    file_list_str), 'patch ISO')
 256
 257     def _put_file(self, ip, user, passwd, file_name, to_file=''):
 258         self._execute_shell('sshpass -p {} scp {} {} {}@{}:{}'.format(passwd,
 259                                                                       Installer.SSH_OPTS,
 260                                                                       file_name,
 261                                                                       user,
 262                                                                       ip,
 263                                                                       to_file), 'put file')
 264
 265     def _get_file(self, ip, user, passwd, file_name, recursive=False):
 266         if recursive:
 267             self._execute_shell('sshpass -p {} scp {} -r {}@{}:{} {}'.format(passwd,
 268                                                                              Installer.SSH_OPTS,
 269                                                                              user,
 270                                                                              ip,
 271                                                                              file_name,
 272                                                                              self._logdir), 'get files')
 273         else:
 274             self._execute_shell('sshpass -p {} scp {} {}@{}:{} {}'.format(passwd,
 275                                                                           Installer.SSH_OPTS,
 276                                                                           user,
 277                                                                           ip,
 278                                                                           file_name,
 279                                                                           self._logdir), 'get file')
 280
 281     def _run_node_command(self, ip, user, passwd, command):
 282         self._execute_shell('sshpass -p {} ssh {} {}@{} {}'.format(passwd,
 283                                                                    Installer.SSH_OPTS,
 284                                                                    user,
 285                                                                    ip,
 286                                                                    command), 'run command: {}'.format(command))
 287
 288     def _get_node_logs(self, ip, user, passwd):
 289         self._get_file(ip, user, passwd, '/srv/deployment/log/cm.log')
 290         self._get_file(ip, user, passwd, '/srv/deployment/log/bootstrap.log')
 291         self._get_file(ip, user, passwd, '/var/log/ironic', recursive=True)
 292
 293     def _create_hosts_file(self, file_name):
 294         with open(file_name, 'w') as hosts_file:
 295             for host in self._uc['hosts'].keys():
 296                 hosts_file.write('{}\n'.format(host))
 297
 298     def _get_journal_logs(self, ip, user, passwd):
 299         hosts_file_name = 'host_names'
 300         hosts_file_path = '{}/{}'.format(self._logdir, hosts_file_name)
 301         self._create_hosts_file(hosts_file_name)
 302
 303         host_list = ' '.join(self._uc['hosts'].keys())
 304
 305         self._put_file(ip, user, passwd, hosts_file_name)
 306         self._put_file(ip, user, passwd, '/opt/scripts/get_journals.sh')
 307
 308         self._run_node_command(ip, user, passwd, 'sh ./get_journals.sh {}'.format(hosts_file_name))
 309
 310         self._get_file(ip, user, passwd, '/tmp/node_journals.tgz')
 311
 312     def _get_logs_from_console(self, bmc, admin_user, admin_passwd):
 313         bmc_host = bmc.get_host()
 314         bmc_user = bmc.get_user()
 315         bmc_passwd = bmc.get_passwd()
 316         bmc_priv_level = bmc.get_priv_level()
 317
 318         log_file = '{}/cat_bootstrap.log'.format(self._logdir)
 319         try:
 320             cat_file = CatFile(bmc_host, bmc_user, bmc_passwd, bmc_priv_level, admin_user, admin_passwd)
 321             cat_file.cat('/srv/deployment/log/bootstrap.log', log_file)
 322         except CatFileException as ex:
 323             logging.info('Could not cat file from console: %s', str(ex))
 324
 325             cat_file = CatFile(bmc_host, bmc_user, bmc_passwd, bmc_priv_level, 'root', 'root')
 326             cat_file.cat('/srv/deployment/log/bootstrap.log', log_file)
 327
 328     def get_logs(self, admin_passwd):
 329         admin_user = self._uc['users']['admin_user_name']
 330
 331         ssh_check_command = 'nc -w1 {} 22 </dev/null &> /dev/null'.format(self._first_controller_ip)
 332         ssh_check_fails = os.system(ssh_check_command)
 333
 334         if not ssh_check_fails:
 335             self._get_node_logs(self._first_controller_ip, admin_user, admin_passwd)
 336
 337             self._get_journal_logs(self._first_controller_ip, admin_user, admin_passwd)
 338         else:
 339             self._get_logs_from_console(self._first_controller_bmc,
 340                                         admin_user,
 341                                         admin_passwd)
 342
 343     def _setup_bmcs(self):
 344         other_bmcs = []
 345         for hw in sorted(self._uc['hosts']):
 346             logging.info('HW node name is %s', hw)
 347
 348             bmc = self._setup_bmc_for_node(hw)
 349             bmc.setup_sol()
 350
 351             if hw != self._first_controller:
 352                 other_bmcs.append(bmc)
 353                 bmc.power('off')
 354
 355         if not self._disable_bmc_initial_reset:
 356             self._first_controller_bmc.reset()
 357             time_after_reset = int(time.time())
 358
 359         if not self._disable_other_bmc_reset:
 360             for bmc in other_bmcs:
 361                 bmc.reset()
 362
 363         if not self._disable_bmc_initial_reset:
 364             # Make sure we sleep at least 6min after the first controller BMC reset
 365             sleep_time = 6*60-int(time.time())-time_after_reset
 366             if sleep_time > 0:
 367                 logging.debug('Waiting for first controller BMC to stabilize \
 368                                (%s sec) after reset', sleep_time)
 369                 time.sleep(sleep_time)
 370
 371     def get_access_info(self):
 372         access_info = {'vip': self._vip,
 373                        'installer_node_ip': self._first_controller_ip,
 374                        'admin_user': self._uc['users']['admin_user_name']}
 375
 376         return access_info
 377
 378     def _set_progress(self, description, failed=False):
 379         if failed:
 380             state = 'failed'
 381         else:
 382             state = 'ongoing'
 383
 384         self._callback_server.set_state(self._callback_uuid, state, description)
 385
 386     def install(self):
 387         try:
 388             logging.info('Start install')
 389
 390             if os.path.dirname(self._boot_iso_path) == '':
 391                 self._boot_iso_path = '{}/{}'.format(os.getcwd(), self._boot_iso_path)
 392
 393             if self._logdir:
 394                 if not os.path.exists(self._logdir):
 395                     os.makedirs(self._logdir)
 396             else:
 397                 self._logdir = '.'
 398
 399             self._set_progress('Setup BMCs')
 400             self._setup_bmcs()
 401
 402             self._set_progress('Create config files')
 403             network_config_filename = self._create_cloud_config()
 404             callback_url_filename = self._create_callback_file()
 405
 406             patch_files = [self._yaml_path,
 407                            network_config_filename,
 408                            callback_url_filename]
 409
 410             if self._client_cert:
 411                 patch_files.append(self._client_cert)
 412             if self._client_key:
 413                 patch_files.append(self._client_key)
 414             if self._ca_cert:
 415                 patch_files.append(self._ca_cert)
 416
 417             self._set_progress('Setup boot options for virtual media')
 418             self._first_controller_bmc.setup_boot_options_for_virtual_media()
 419
 420             self._set_progress('Attach iso as virtual media')
 421             self._attach_iso_as_virtual_media(patch_files)
 422
 423             self._set_progress('Boot from virtual media')
 424             self._first_controller_bmc.boot_from_virtual_media()
 425
 426             self._set_progress('Wait for bootup')
 427             self._first_controller_bmc.wait_for_bootup()
 428
 429             self._set_progress('Wait deployment start')
 430
 431             self._first_controller_bmc.close()
 432         except BMCException as ex:
 433             logging.error('Installation failed: %s', str(ex))
 434             raise InstallException(str(ex))
 435
 436 def main():
 437     parser = argparse.ArgumentParser()
 438     parser.add_argument('-y', '--yaml', required=True,
 439                         help='User config yaml file path')
 440     parser.add_argument('-b', '--boot-iso', required=True,
 441                         help='Path to boot ISO image in NFS mount')
 442     parser.add_argument('-i', '--iso', required=True,
 443                         help='URL to ISO image')
 444     parser.add_argument('-d', '--debug', action='store_true', required=False,
 445                         help='Debug level for logging')
 446     parser.add_argument('-l', '--logdir', required=True,
 447                         help='Directory path for log files')
 448     parser.add_argument('-c', '--callback-url', required=True,
 449                         help='Callback URL for progress reporting')
 450     parser.add_argument('-K', '--client-key', required=True,
 451                         help='Client key file path')
 452     parser.add_argument('-C', '--client-cert', required=True,
 453                         help='Client cert file path')
 454     parser.add_argument('-A', '--ca-cert', required=True,
 455                         help='CA cert file path')
 456     parser.add_argument('-H', '--host-ip', required=True,
 457                         help='IP for hosting HTTPD and NFS')
 458     parser.add_argument('-T', '--http-port', required=False,
 459                         help='Port for HTTPD')
 460
 461     parsed_args = parser.parse_args()
 462
 463     if parsed_args.debug:
 464         log_level = logging.DEBUG
 465     else:
 466         log_level = logging.INFO
 467
 468     logging.basicConfig(stream=sys.stdout, level=log_level)
 469
 470     logging.debug('args: %s', parsed_args)
 471     installer = Installer(parsed_args.yaml, parsed_args.logdir, parsed_args)
 472
 473     installer.install()
 474
 475 if __name__ == "__main__":
 476     sys.exit(main())