src/remoteinstaller/installer/install.py

   1 # Copyright 2019 Nokia
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import logging
  16 import sys
  17 import argparse
  18 import subprocess
  19 import os
  20 import importlib
  21 import time
  22
  23 from yaml import load
  24 from netaddr import IPNetwork
  25 from netaddr import IPAddress
  26
  27 import hw_detector.hw_detect_lib as hw_detect
  28 from hw_detector.hw_exception import HWException
  29 from remoteinstaller.installer.bmc_management.bmctools import BMCException
  30 from remoteinstaller.installer.catfile import CatFile
  31 from remoteinstaller.installer.catfile import CatFileException
  32
  33 class InstallException(Exception):
  34     pass
  35
  36 class Installer(object):
  37     SSH_OPTS = '-o StrictHostKeyChecking=no \
  38                 -o UserKnownHostsFile=/dev/null \
  39                 -o ServerAliveInterval=60'
  40
  41     def __init__(self, callback_server, callback_uuid, yaml, logdir, args=None):
  42         self._callback_server = callback_server
  43         self._callback_uuid = callback_uuid
  44         self._yaml_path = yaml
  45         self._uc = self._read_user_config(self._yaml_path)
  46         self._logdir = logdir
  47
  48         self._boot_iso_path = None
  49         self._iso_url = None
  50         self._callback_url = None
  51         self._client_key = None
  52         self._client_cert = None
  53         self._ca_cert = None
  54         self._own_ip = None
  55         self._tag = None
  56         if args:
  57             self._set_arguments(args)
  58
  59         # TODO
  60         self._disable_bmc_initial_reset = False
  61         self._disable_other_bmc_reset = True
  62
  63         self._vip = None
  64         self._first_controller = None
  65         self._first_controller_ip = None
  66         self._first_controller_bmc = None
  67
  68         self._define_first_controller()
  69
  70     def _set_arguments(self, args):
  71         self._boot_iso_path = args.boot_iso
  72         self._iso_url = args.iso
  73         self._callback_url = args.callback_url
  74         self._client_key = args.client_key
  75         self._client_cert = args.client_cert
  76         self._ca_cert = args.ca_cert
  77         self._own_ip = args.host_ip
  78         self._tag = args.tag
  79
  80     @staticmethod
  81     def _read_user_config(config_file_path):
  82         logging.debug('Read user config from %s', config_file_path)
  83
  84         try:
  85             with open(config_file_path, 'r') as f:
  86                 y = load(f)
  87
  88             return y
  89         except Exception as ex:
  90             raise InstallException(str(ex))
  91
  92     @staticmethod
  93     def _execute_shell(command, desc=''):
  94         logging.debug('Execute %s with command: %s', desc, command)
  95
  96         p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  97         out, _ = p.communicate()
  98         if p.returncode:
  99             logging.warning('Failed to %s: %s (rc=%s)', desc, out, p.returncode)
 100             raise InstallException('Failed to {}'.format(desc))
 101
 102         return (p.returncode, out)
 103
 104     def _attach_iso_as_virtual_media(self, file_list):
 105         logging.info('Attach ISO as virtual media')
 106
 107         nfs_mount = os.path.dirname(self._boot_iso_path)
 108         boot_iso_filename = os.path.basename(self._boot_iso_path)
 109         patched_iso_filename = '{}/{}_{}'.format(nfs_mount, self._tag, boot_iso_filename)
 110
 111         self._patch_iso(patched_iso_filename, file_list)
 112
 113         self._first_controller_bmc.attach_virtual_cd(self._own_ip, nfs_mount, os.path.basename(patched_iso_filename))
 114
 115     def _setup_bmc_for_node(self, hw):
 116         bmc_log_path = '{}/{}.log'.format(self._logdir, hw)
 117
 118         host = self._uc['hosts'][hw]['hwmgmt']['address']
 119         user = self._uc['hosts'][hw]['hwmgmt']['user']
 120         passwd = self._uc['hosts'][hw]['hwmgmt']['password']
 121
 122         try:
 123             hw_data = hw_detect.get_hw_data(host, user, passwd, False)
 124         except HWException as e:
 125             error = "Harware not detected for {}: {}".format(hw, str(e))
 126             logging.error(error)
 127             raise BMCException(error)
 128
 129         logging.debug("Hardware belongs to %s product family", (hw_data['product_family']))
 130         if 'Unknown' in hw_data['product_family']:
 131             error = "Hardware not detected for %s" % hw
 132             logging.error(error)
 133             raise BMCException(error)
 134
 135         bmc_mod_name = 'remoteinstaller.installer.bmc_management.{}'.format(hw_data['product_family'].lower())
 136         bmc_mod = importlib.import_module(bmc_mod_name)
 137         bmc_class = getattr(bmc_mod, hw_data['product_family'])
 138         bmc = bmc_class(host, user, passwd, bmc_log_path)
 139         bmc.set_host_name(hw)
 140
 141         return bmc
 142
 143     def _define_first_controller(self):
 144         for hw in sorted(self._uc['hosts']):
 145             logging.debug('HW node name is %s', hw)
 146
 147             if 'controller' in self._uc['hosts'][hw]['service_profiles'] or \
 148             'caas_master' in self._uc['hosts'][hw]['service_profiles']:
 149                 self._first_controller = hw
 150                 break
 151
 152         logging.info('First controller is %s', self._first_controller)
 153         self._first_controller_bmc = self._setup_bmc_for_node(self._first_controller)
 154
 155         domain = self._uc['hosts'][self._first_controller].get('network_domain')
 156         extnet = self._uc['networking']['infra_external']['network_domains'][domain]
 157
 158         first_ip = extnet['ip_range_start']
 159         self._vip = str(IPAddress(first_ip))
 160
 161         pre_allocated_ips = self._uc['hosts'][self._first_controller].get('pre_allocated_ips', None)
 162         if pre_allocated_ips:
 163             pre_allocated_infra_external_ip = pre_allocated_ips.get('infra_external', None)
 164             self._first_controller_ip = str(IPAddress(pre_allocated_infra_external_ip))
 165
 166         if not self._first_controller_ip:
 167             self._first_controller_ip = str(IPAddress(first_ip)+1)
 168
 169     def _create_cloud_config(self):
 170         logging.info('Create network config file')
 171
 172         domain = self._uc['hosts'][self._first_controller].get('network_domain')
 173         extnet = self._uc['networking']['infra_external']['network_domains'][domain]
 174
 175         vlan = extnet.get('vlan')
 176         gateway = extnet['gateway']
 177         dns = self._uc['networking']['dns'][0]
 178         cidr = extnet['cidr']
 179         prefix = IPNetwork(cidr).prefixlen
 180
 181         controller_network_profile = self._uc['hosts'][self._first_controller]['network_profiles'][0]
 182         mappings = self._uc['network_profiles'][controller_network_profile]['interface_net_mapping']
 183         for interface in mappings:
 184             if 'infra_external' in mappings[interface]:
 185                 infra_external_interface = interface
 186                 break
 187
 188         if infra_external_interface.startswith('bond'):
 189             bonds = self._uc['network_profiles'][controller_network_profile]['bonding_interfaces']
 190             device = bonds[infra_external_interface][0]
 191         else:
 192             device = infra_external_interface
 193
 194         # TODO
 195         # ROOTFS_DISK
 196
 197         logging.debug('VLAN=%s', vlan)
 198         logging.debug('DEV=%s', device)
 199         logging.debug('IP=%s/%s', self._first_controller_ip, prefix)
 200         logging.debug('DGW=%s', gateway)
 201         logging.debug('NAMESERVER=%s', dns)
 202         logging.debug('ISO_URL="%s"', self._iso_url)
 203
 204         network_config_filename = '{}/network_config'.format(self._logdir)
 205         with open(network_config_filename, 'w') as f:
 206             if vlan:
 207                 f.write('VLAN={}\n'.format(vlan))
 208             f.write('DEV={}\n'.format(device))
 209             f.write('IP={}/{}\n'.format(self._first_controller_ip, prefix))
 210             f.write('DGW={}\n'.format(gateway))
 211             f.write('NAMESERVER={}\n'.format(dns))
 212             f.write('\n')
 213             f.write('ISO_URL="{}"'.format(self._iso_url))
 214
 215         return network_config_filename
 216
 217     def _create_callback_file(self):
 218         logging.debug('CALLBACK_URL="%s"', self._callback_url)
 219
 220         callback_url_filename = '{}/callback_url'.format(self._logdir)
 221         with open(callback_url_filename, 'w') as f:
 222             f.write(self._callback_url)
 223
 224         return callback_url_filename
 225
 226     def _patch_iso(self, iso_target, file_list):
 227         logging.info('Patch boot ISO')
 228         logging.debug('Original ISO: %s', self._boot_iso_path)
 229         logging.debug('Target ISO: %s', iso_target)
 230
 231         file_list_str = ' '.join(file_list)
 232         logging.debug('Files to add: %s', file_list_str)
 233
 234         self._execute_shell('/usr/bin/patchiso.sh {} {} {}'.format(self._boot_iso_path,
 235                                                                    iso_target,
 236                                                                    file_list_str), 'patch ISO')
 237
 238     def _put_file(self, ip, user, passwd, file_name, to_file=''):
 239         self._execute_shell('sshpass -p {} scp {} {} {}@{}:{}'.format(passwd,
 240                                                                       Installer.SSH_OPTS,
 241                                                                       file_name,
 242                                                                       user,
 243                                                                       ip,
 244                                                                       to_file), 'put file')
 245
 246     def _get_file(self, ip, user, passwd, file_name, recursive=False):
 247         if recursive:
 248             self._execute_shell('sshpass -p {} scp {} -r {}@{}:{} {}'.format(passwd,
 249                                                                              Installer.SSH_OPTS,
 250                                                                              user,
 251                                                                              ip,
 252                                                                              file_name,
 253                                                                              self._logdir), 'get files')
 254         else:
 255             self._execute_shell('sshpass -p {} scp {} {}@{}:{} {}'.format(passwd,
 256                                                                           Installer.SSH_OPTS,
 257                                                                           user,
 258                                                                           ip,
 259                                                                           file_name,
 260                                                                           self._logdir), 'get file')
 261
 262     def _get_node_logs(self, ip, user, passwd):
 263         self._get_file(ip, user, passwd, '/srv/deployment/log/cm.log')
 264         self._get_file(ip, user, passwd, '/srv/deployment/log/bootstrap.log')
 265         self._get_file(ip, user, passwd, '/var/log/ironic', recursive=True)
 266
 267     def _get_journal_logs(self, ip, user, passwd):
 268         self._put_file(ip, user, passwd, '/opt/scripts/get_journals.sh')
 269         self._put_file(ip, user, passwd, '/opt/scripts/print_hosts.py')
 270
 271         self._execute_shell('sh ./get_journals.sh', 'run get_journals.sh')
 272
 273         self._get_file(ip, user, passwd, '/tmp/node_journals.tgz')
 274
 275     def _get_logs_from_console(self, bmc, admin_user, admin_passwd):
 276         bmc_host = bmc.get_host()
 277         bmc_user = bmc.get_user()
 278         bmc_passwd = bmc.get_passwd()
 279
 280         log_file = '{}/cat_bootstrap.log'.format(self._logdir)
 281         try:
 282             cat_file = CatFile(bmc_host, bmc_user, bmc_passwd, admin_user, admin_passwd)
 283             cat_file.cat('/srv/deployment/log/bootstrap.log', log_file)
 284         except CatFileException as ex:
 285             logging.info('Could not cat file from console: %s', str(ex))
 286
 287             cat_file = CatFile(bmc_host, bmc_user, bmc_passwd, 'root', 'root')
 288             cat_file.cat('/srv/deployment/log/bootstrap.log', log_file)
 289
 290     def get_logs(self, admin_passwd):
 291         admin_user = self._uc['users']['admin_user_name']
 292
 293         ssh_check_command = 'nc -w1 {} 22 </dev/null &> /dev/null'.format(self._first_controller_ip)
 294         ssh_check_fails = os.system(ssh_check_command)
 295
 296         if not ssh_check_fails:
 297             self._get_node_logs(self._first_controller_ip, admin_user, admin_passwd)
 298
 299             self._get_journal_logs(self._first_controller_ip, admin_user, admin_passwd)
 300         else:
 301             self._get_logs_from_console(self._first_controller_bmc,
 302                                         admin_user,
 303                                         admin_passwd)
 304
 305     def _setup_bmcs(self):
 306         other_bmcs = []
 307         for hw in sorted(self._uc['hosts']):
 308             logging.info('HW node name is %s', hw)
 309
 310             bmc = self._setup_bmc_for_node(hw)
 311             bmc.setup_sol()
 312
 313             if hw != self._first_controller:
 314                 other_bmcs.append(bmc)
 315                 bmc.power('off')
 316
 317         if not self._disable_bmc_initial_reset:
 318             self._first_controller_bmc.reset()
 319             time_after_reset = int(time.time())
 320
 321         if not self._disable_other_bmc_reset:
 322             for bmc in other_bmcs:
 323                 bmc.reset()
 324
 325         if not self._disable_bmc_initial_reset:
 326             # Make sure we sleep at least 6min after the first controller BMC reset
 327             sleep_time = 6*60-int(time.time())-time_after_reset
 328             if sleep_time > 0:
 329                 logging.debug('Waiting for first controller BMC to stabilize \
 330                                (%s sec) after reset', sleep_time)
 331                 time.sleep(sleep_time)
 332
 333     def get_access_info(self):
 334         access_info = {'vip': self._vip,
 335                        'installer_node_ip': self._first_controller_ip,
 336                        'admin_user': self._uc['users']['admin_user_name']}
 337
 338         return access_info
 339
 340     def _set_progress(self, description, failed=False):
 341         if failed:
 342             state = 'failed'
 343         else:
 344             state = 'ongoing'
 345
 346         self._callback_server.set_state(self._callback_uuid, state, description)
 347
 348     def install(self):
 349         try:
 350             logging.info('Start install')
 351
 352             if os.path.dirname(self._boot_iso_path) == '':
 353                 self._boot_iso_path = '{}/{}'.format(os.getcwd(), self._boot_iso_path)
 354
 355             if self._logdir:
 356                 if not os.path.exists(self._logdir):
 357                     os.makedirs(self._logdir)
 358             else:
 359                 self._logdir = '.'
 360
 361             self._set_progress('Setup BMCs')
 362             self._setup_bmcs()
 363
 364             self._set_progress('Create config files')
 365             network_config_filename = self._create_cloud_config()
 366             callback_url_filename = self._create_callback_file()
 367
 368             patch_files = [self._yaml_path,
 369                            network_config_filename,
 370                            callback_url_filename]
 371
 372             if self._client_cert:
 373                 patch_files.append(self._client_cert)
 374             if self._client_key:
 375                 patch_files.append(self._client_key)
 376             if self._ca_cert:
 377                 patch_files.append(self._ca_cert)
 378
 379             self._set_progress('Setup boot options for virtual media')
 380             self._first_controller_bmc.setup_boot_options_for_virtual_media()
 381
 382             self._set_progress('Attach iso as virtual media')
 383             self._attach_iso_as_virtual_media(patch_files)
 384
 385             self._set_progress('Boot from virtual media')
 386             self._first_controller_bmc.boot_from_virtual_media()
 387
 388             self._set_progress('Wait for bootup')
 389             self._first_controller_bmc.wait_for_bootup()
 390
 391             self._first_controller_bmc.close()
 392         except BMCException as ex:
 393             logging.error('Installation failed: %s', str(ex))
 394             raise InstallException(str(ex))
 395
 396 def main():
 397     parser = argparse.ArgumentParser()
 398     parser.add_argument('-y', '--yaml', required=True,
 399                         help='User config yaml file path')
 400     parser.add_argument('-b', '--boot-iso', required=True,
 401                         help='Path to boot ISO image in NFS mount')
 402     parser.add_argument('-i', '--iso', required=True,
 403                         help='URL to ISO image')
 404     parser.add_argument('-d', '--debug', action='store_true', required=False,
 405                         help='Debug level for logging')
 406     parser.add_argument('-l', '--logdir', required=True,
 407                         help='Directory path for log files')
 408     parser.add_argument('-c', '--callback-url', required=True,
 409                         help='Callback URL for progress reporting')
 410     parser.add_argument('-K', '--client-key', required=True,
 411                         help='Client key file path')
 412     parser.add_argument('-C', '--client-cert', required=True,
 413                         help='Client cert file path')
 414     parser.add_argument('-A', '--ca-cert', required=True,
 415                         help='CA cert file path')
 416     parser.add_argument('-H', '--host-ip', required=True,
 417                         help='IP for hosting HTTPD and NFS')
 418     parser.add_argument('-T', '--http-port', required=False,
 419                         help='Port for HTTPD')
 420
 421     parsed_args = parser.parse_args()
 422
 423     if parsed_args.debug:
 424         log_level = logging.DEBUG
 425     else:
 426         log_level = logging.INFO
 427
 428     logging.basicConfig(stream=sys.stdout, level=log_level)
 429
 430     logging.debug('args: %s', parsed_args)
 431     installer = Installer(parsed_args.yaml, parsed_args.logdir, parsed_args)
 432
 433     installer.install()
 434
 435 if __name__ == "__main__":
 436     sys.exit(main())