src/remoteinstaller/installer/install.py

   1 # Copyright 2019 Nokia
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import logging
  16 import sys
  17 import argparse
  18 import subprocess
  19 import os
  20 import importlib
  21 import time
  22 import distutils.util
  23
  24 from yaml import load
  25 from netaddr import IPNetwork
  26 from netaddr import IPAddress
  27
  28 import hw_detector.hw_detect_lib as hw_detect
  29 from hw_detector.hw_exception import HWException
  30 from remoteinstaller.installer.bmc_management.bmctools import BMCException
  31 from remoteinstaller.installer.catfile import CatFile
  32 from remoteinstaller.installer.catfile import CatFileException
  33
  34 class InstallException(Exception):
  35     pass
  36
  37 class Installer(object):
  38     SSH_OPTS = ('-o StrictHostKeyChecking=no '
  39                 '-o UserKnownHostsFile=/dev/null '
  40                 '-o ServerAliveInterval=60')
  41
  42     def __init__(self, callback_server, callback_uuid, yaml, logdir, args=None):
  43         self._callback_server = callback_server
  44         self._callback_uuid = callback_uuid
  45         self._yaml_path = yaml
  46         self._uc = self._read_user_config(self._yaml_path)
  47         self._logdir = logdir
  48
  49         self._boot_iso_path = None
  50         self._iso_url = None
  51         self._callback_url = None
  52         self._client_key = None
  53         self._client_cert = None
  54         self._ca_cert = None
  55         self._own_ip = None
  56         self._tag = None
  57         self._disable_bmc_initial_reset = False
  58         self._disable_other_bmc_reset = True
  59
  60         if args:
  61             self._set_arguments(args)
  62
  63         self._vip = None
  64         self._first_controller = None
  65         self._first_controller_ip = None
  66         self._first_controller_bmc = None
  67
  68         self._define_first_controller()
  69
  70     def _get_bool_arg(self, args, arg, default):
  71         if hasattr(args, arg):
  72             arg_value = vars(args)[arg]
  73             if not isinstance(arg_value, bool):
  74                 if isinstance(arg_value, basestring):
  75                     try:
  76                         arg_value = bool(distutils.util.strtobool(arg_value))
  77                         return arg_value
  78                     except ValueError:
  79                         logging.warning('Invalid value for %s: %s', arg, arg_value)
  80             else:
  81                 return arg_value
  82
  83         return default
  84
  85     def _set_arguments(self, args):
  86         self._disable_bmc_initial_reset = self._get_bool_arg(args, 'disable_bmc_initial_reset', self._disable_bmc_initial_reset)
  87         self._disable_other_bmc_reset = self._get_bool_arg(args, 'disable_other_bmc_reset', self._disable_other_bmc_reset)
  88
  89         self._boot_iso_path = args.boot_iso
  90         self._iso_url = args.iso
  91         self._callback_url = args.callback_url
  92         self._client_key = args.client_key
  93         self._client_cert = args.client_cert
  94         self._ca_cert = args.ca_cert
  95         self._own_ip = args.host_ip
  96         self._tag = args.tag
  97
  98     @staticmethod
  99     def _read_user_config(config_file_path):
 100         logging.debug('Read user config from %s', config_file_path)
 101
 102         try:
 103             with open(config_file_path, 'r') as f:
 104                 y = load(f)
 105
 106             return y
 107         except Exception as ex:
 108             raise InstallException(str(ex))
 109
 110     @staticmethod
 111     def _execute_shell(command, desc=''):
 112         logging.debug('Execute %s with command: %s', desc, command)
 113
 114         p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 115         out, _ = p.communicate()
 116         if p.returncode:
 117             logging.warning('Failed to %s: %s (rc=%s)', desc, out, p.returncode)
 118             raise InstallException('Failed to {}'.format(desc))
 119
 120         return (p.returncode, out)
 121
 122     def _attach_iso_as_virtual_media(self, file_list):
 123         logging.info('Attach ISO as virtual media')
 124
 125         nfs_mount = os.path.dirname(self._boot_iso_path)
 126         boot_iso_filename = os.path.basename(self._boot_iso_path)
 127         patched_iso_filename = '{}/{}_{}'.format(nfs_mount, self._tag, boot_iso_filename)
 128
 129         self._patch_iso(patched_iso_filename, file_list)
 130
 131         self._first_controller_bmc.attach_virtual_cd(self._own_ip, nfs_mount, os.path.basename(patched_iso_filename))
 132
 133     def _setup_bmc_for_node(self, hw):
 134         bmc_log_path = '{}/{}.log'.format(self._logdir, hw)
 135
 136         host = self._uc['hosts'][hw]['hwmgmt']['address']
 137         user = self._uc['hosts'][hw]['hwmgmt']['user']
 138         passwd = self._uc['hosts'][hw]['hwmgmt']['password']
 139
 140         try:
 141             hw_data = hw_detect.get_hw_data(host, user, passwd, False)
 142         except HWException as e:
 143             error = "Harware not detected for {}: {}".format(hw, str(e))
 144             logging.error(error)
 145             raise BMCException(error)
 146
 147         logging.debug("Hardware belongs to %s product family", (hw_data['product_family']))
 148         if 'Unknown' in hw_data['product_family']:
 149             error = "Hardware not detected for %s" % hw
 150             logging.error(error)
 151             raise BMCException(error)
 152
 153         bmc_mod_name = 'remoteinstaller.installer.bmc_management.{}'.format(hw_data['product_family'].lower())
 154         bmc_mod = importlib.import_module(bmc_mod_name)
 155         bmc_class = getattr(bmc_mod, hw_data['product_family'])
 156         bmc = bmc_class(host, user, passwd, bmc_log_path)
 157         bmc.set_host_name(hw)
 158
 159         return bmc
 160
 161     def _define_first_controller(self):
 162         for hw in sorted(self._uc['hosts']):
 163             logging.debug('HW node name is %s', hw)
 164
 165             if 'controller' in self._uc['hosts'][hw]['service_profiles'] or \
 166             'caas_master' in self._uc['hosts'][hw]['service_profiles']:
 167                 self._first_controller = hw
 168                 break
 169
 170         logging.info('First controller is %s', self._first_controller)
 171         self._first_controller_bmc = self._setup_bmc_for_node(self._first_controller)
 172
 173         domain = self._uc['hosts'][self._first_controller].get('network_domain')
 174         extnet = self._uc['networking']['infra_external']['network_domains'][domain]
 175
 176         first_ip = extnet['ip_range_start']
 177         self._vip = str(IPAddress(first_ip))
 178
 179         pre_allocated_ips = self._uc['hosts'][self._first_controller].get('pre_allocated_ips', None)
 180         if pre_allocated_ips:
 181             pre_allocated_infra_external_ip = pre_allocated_ips.get('infra_external', None)
 182             self._first_controller_ip = str(IPAddress(pre_allocated_infra_external_ip))
 183
 184         if not self._first_controller_ip:
 185             self._first_controller_ip = str(IPAddress(first_ip)+1)
 186
 187     def _create_cloud_config(self):
 188         logging.info('Create network config file')
 189
 190         domain = self._uc['hosts'][self._first_controller].get('network_domain')
 191         extnet = self._uc['networking']['infra_external']['network_domains'][domain]
 192
 193         vlan = extnet.get('vlan')
 194         gateway = extnet['gateway']
 195         dns = self._uc['networking']['dns'][0]
 196         cidr = extnet['cidr']
 197         prefix = IPNetwork(cidr).prefixlen
 198
 199         controller_network_profile = self._uc['hosts'][self._first_controller]['network_profiles'][0]
 200         mappings = self._uc['network_profiles'][controller_network_profile]['interface_net_mapping']
 201         for interface in mappings:
 202             if 'infra_external' in mappings[interface]:
 203                 infra_external_interface = interface
 204                 break
 205
 206         if infra_external_interface.startswith('bond'):
 207             bonds = self._uc['network_profiles'][controller_network_profile]['bonding_interfaces']
 208             device = bonds[infra_external_interface][0]
 209         else:
 210             device = infra_external_interface
 211
 212         # TODO
 213         # ROOTFS_DISK
 214
 215         logging.debug('VLAN=%s', vlan)
 216         logging.debug('DEV=%s', device)
 217         logging.debug('IP=%s/%s', self._first_controller_ip, prefix)
 218         logging.debug('DGW=%s', gateway)
 219         logging.debug('NAMESERVER=%s', dns)
 220         logging.debug('ISO_URL="%s"', self._iso_url)
 221
 222         network_config_filename = '{}/network_config'.format(self._logdir)
 223         with open(network_config_filename, 'w') as f:
 224             if vlan:
 225                 f.write('VLAN={}\n'.format(vlan))
 226             f.write('DEV={}\n'.format(device))
 227             f.write('IP={}/{}\n'.format(self._first_controller_ip, prefix))
 228             f.write('DGW={}\n'.format(gateway))
 229             f.write('NAMESERVER={}\n'.format(dns))
 230             f.write('\n')
 231             f.write('ISO_URL="{}"'.format(self._iso_url))
 232
 233         return network_config_filename
 234
 235     def _create_callback_file(self):
 236         logging.debug('CALLBACK_URL="%s"', self._callback_url)
 237
 238         callback_url_filename = '{}/callback_url'.format(self._logdir)
 239         with open(callback_url_filename, 'w') as f:
 240             f.write(self._callback_url)
 241
 242         return callback_url_filename
 243
 244     def _patch_iso(self, iso_target, file_list):
 245         logging.info('Patch boot ISO')
 246         logging.debug('Original ISO: %s', self._boot_iso_path)
 247         logging.debug('Target ISO: %s', iso_target)
 248
 249         file_list_str = ' '.join(file_list)
 250         logging.debug('Files to add: %s', file_list_str)
 251
 252         self._execute_shell('/usr/bin/patchiso.sh {} {} {}'.format(self._boot_iso_path,
 253                                                                    iso_target,
 254                                                                    file_list_str), 'patch ISO')
 255
 256     def _put_file(self, ip, user, passwd, file_name, to_file=''):
 257         self._execute_shell('sshpass -p {} scp {} {} {}@{}:{}'.format(passwd,
 258                                                                       Installer.SSH_OPTS,
 259                                                                       file_name,
 260                                                                       user,
 261                                                                       ip,
 262                                                                       to_file), 'put file')
 263
 264     def _get_file(self, ip, user, passwd, file_name, recursive=False):
 265         if recursive:
 266             self._execute_shell('sshpass -p {} scp {} -r {}@{}:{} {}'.format(passwd,
 267                                                                              Installer.SSH_OPTS,
 268                                                                              user,
 269                                                                              ip,
 270                                                                              file_name,
 271                                                                              self._logdir), 'get files')
 272         else:
 273             self._execute_shell('sshpass -p {} scp {} {}@{}:{} {}'.format(passwd,
 274                                                                           Installer.SSH_OPTS,
 275                                                                           user,
 276                                                                           ip,
 277                                                                           file_name,
 278                                                                           self._logdir), 'get file')
 279
 280     def _run_node_command(self, ip, user, passwd, command):
 281         self._execute_shell('sshpass -p {} ssh {} {}@{} {}'.format(passwd,
 282                                                                    Installer.SSH_OPTS,
 283                                                                    user,
 284                                                                    ip,
 285                                                                    command), 'run command: {}'.format(command))
 286
 287     def _get_node_logs(self, ip, user, passwd):
 288         self._get_file(ip, user, passwd, '/srv/deployment/log/cm.log')
 289         self._get_file(ip, user, passwd, '/srv/deployment/log/bootstrap.log')
 290         self._get_file(ip, user, passwd, '/var/log/ironic', recursive=True)
 291
 292     def _create_hosts_file(self, file_name):
 293         with open(file_name, 'w') as hosts_file:
 294             for host in self._uc['hosts'].keys():
 295                 hosts_file.write('{}\n'.format(host))
 296
 297     def _get_journal_logs(self, ip, user, passwd):
 298         hosts_file_name = 'host_names'
 299         hosts_file_path = '{}/{}'.format(self._logdir, hosts_file_name)
 300         self._create_hosts_file(hosts_file_name)
 301
 302         host_list = ' '.join(self._uc['hosts'].keys())
 303
 304         self._put_file(ip, user, passwd, hosts_file_name)
 305         self._put_file(ip, user, passwd, '/opt/scripts/get_journals.sh')
 306
 307         self._run_node_command(ip, user, passwd, 'sh ./get_journals.sh {}'.format(hosts_file_name))
 308
 309         self._get_file(ip, user, passwd, '/tmp/node_journals.tgz')
 310
 311     def _get_logs_from_console(self, bmc, admin_user, admin_passwd):
 312         bmc_host = bmc.get_host()
 313         bmc_user = bmc.get_user()
 314         bmc_passwd = bmc.get_passwd()
 315
 316         log_file = '{}/cat_bootstrap.log'.format(self._logdir)
 317         try:
 318             cat_file = CatFile(bmc_host, bmc_user, bmc_passwd, admin_user, admin_passwd)
 319             cat_file.cat('/srv/deployment/log/bootstrap.log', log_file)
 320         except CatFileException as ex:
 321             logging.info('Could not cat file from console: %s', str(ex))
 322
 323             cat_file = CatFile(bmc_host, bmc_user, bmc_passwd, 'root', 'root')
 324             cat_file.cat('/srv/deployment/log/bootstrap.log', log_file)
 325
 326     def get_logs(self, admin_passwd):
 327         admin_user = self._uc['users']['admin_user_name']
 328
 329         ssh_check_command = 'nc -w1 {} 22 </dev/null &> /dev/null'.format(self._first_controller_ip)
 330         ssh_check_fails = os.system(ssh_check_command)
 331
 332         if not ssh_check_fails:
 333             self._get_node_logs(self._first_controller_ip, admin_user, admin_passwd)
 334
 335             self._get_journal_logs(self._first_controller_ip, admin_user, admin_passwd)
 336         else:
 337             self._get_logs_from_console(self._first_controller_bmc,
 338                                         admin_user,
 339                                         admin_passwd)
 340
 341     def _setup_bmcs(self):
 342         other_bmcs = []
 343         for hw in sorted(self._uc['hosts']):
 344             logging.info('HW node name is %s', hw)
 345
 346             bmc = self._setup_bmc_for_node(hw)
 347             bmc.setup_sol()
 348
 349             if hw != self._first_controller:
 350                 other_bmcs.append(bmc)
 351                 bmc.power('off')
 352
 353         if not self._disable_bmc_initial_reset:
 354             self._first_controller_bmc.reset()
 355             time_after_reset = int(time.time())
 356
 357         if not self._disable_other_bmc_reset:
 358             for bmc in other_bmcs:
 359                 bmc.reset()
 360
 361         if not self._disable_bmc_initial_reset:
 362             # Make sure we sleep at least 6min after the first controller BMC reset
 363             sleep_time = 6*60-int(time.time())-time_after_reset
 364             if sleep_time > 0:
 365                 logging.debug('Waiting for first controller BMC to stabilize \
 366                                (%s sec) after reset', sleep_time)
 367                 time.sleep(sleep_time)
 368
 369     def get_access_info(self):
 370         access_info = {'vip': self._vip,
 371                        'installer_node_ip': self._first_controller_ip,
 372                        'admin_user': self._uc['users']['admin_user_name']}
 373
 374         return access_info
 375
 376     def _set_progress(self, description, failed=False):
 377         if failed:
 378             state = 'failed'
 379         else:
 380             state = 'ongoing'
 381
 382         self._callback_server.set_state(self._callback_uuid, state, description)
 383
 384     def install(self):
 385         try:
 386             logging.info('Start install')
 387
 388             if os.path.dirname(self._boot_iso_path) == '':
 389                 self._boot_iso_path = '{}/{}'.format(os.getcwd(), self._boot_iso_path)
 390
 391             if self._logdir:
 392                 if not os.path.exists(self._logdir):
 393                     os.makedirs(self._logdir)
 394             else:
 395                 self._logdir = '.'
 396
 397             self._set_progress('Setup BMCs')
 398             self._setup_bmcs()
 399
 400             self._set_progress('Create config files')
 401             network_config_filename = self._create_cloud_config()
 402             callback_url_filename = self._create_callback_file()
 403
 404             patch_files = [self._yaml_path,
 405                            network_config_filename,
 406                            callback_url_filename]
 407
 408             if self._client_cert:
 409                 patch_files.append(self._client_cert)
 410             if self._client_key:
 411                 patch_files.append(self._client_key)
 412             if self._ca_cert:
 413                 patch_files.append(self._ca_cert)
 414
 415             self._set_progress('Setup boot options for virtual media')
 416             self._first_controller_bmc.setup_boot_options_for_virtual_media()
 417
 418             self._set_progress('Attach iso as virtual media')
 419             self._attach_iso_as_virtual_media(patch_files)
 420
 421             self._set_progress('Boot from virtual media')
 422             self._first_controller_bmc.boot_from_virtual_media()
 423
 424             self._set_progress('Wait for bootup')
 425             self._first_controller_bmc.wait_for_bootup()
 426
 427             self._set_progress('Wait deployment start')
 428
 429             self._first_controller_bmc.close()
 430         except BMCException as ex:
 431             logging.error('Installation failed: %s', str(ex))
 432             raise InstallException(str(ex))
 433
 434 def main():
 435     parser = argparse.ArgumentParser()
 436     parser.add_argument('-y', '--yaml', required=True,
 437                         help='User config yaml file path')
 438     parser.add_argument('-b', '--boot-iso', required=True,
 439                         help='Path to boot ISO image in NFS mount')
 440     parser.add_argument('-i', '--iso', required=True,
 441                         help='URL to ISO image')
 442     parser.add_argument('-d', '--debug', action='store_true', required=False,
 443                         help='Debug level for logging')
 444     parser.add_argument('-l', '--logdir', required=True,
 445                         help='Directory path for log files')
 446     parser.add_argument('-c', '--callback-url', required=True,
 447                         help='Callback URL for progress reporting')
 448     parser.add_argument('-K', '--client-key', required=True,
 449                         help='Client key file path')
 450     parser.add_argument('-C', '--client-cert', required=True,
 451                         help='Client cert file path')
 452     parser.add_argument('-A', '--ca-cert', required=True,
 453                         help='CA cert file path')
 454     parser.add_argument('-H', '--host-ip', required=True,
 455                         help='IP for hosting HTTPD and NFS')
 456     parser.add_argument('-T', '--http-port', required=False,
 457                         help='Port for HTTPD')
 458
 459     parsed_args = parser.parse_args()
 460
 461     if parsed_args.debug:
 462         log_level = logging.DEBUG
 463     else:
 464         log_level = logging.INFO
 465
 466     logging.basicConfig(stream=sys.stdout, level=log_level)
 467
 468     logging.debug('args: %s', parsed_args)
 469     installer = Installer(parsed_args.yaml, parsed_args.logdir, parsed_args)
 470
 471     installer.install()
 472
 473 if __name__ == "__main__":
 474     sys.exit(main())