src/remoteinstaller/installer/install.py

   1 # Copyright 2019 Nokia
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import logging
  16 import sys
  17 import argparse
  18 import subprocess
  19 import os
  20 import importlib
  21 import time
  22
  23 from yaml import load
  24 from netaddr import IPNetwork
  25 from netaddr import IPAddress
  26
  27 import hw_detector.hw_detect_lib as hw_detect
  28 from hw_detector.hw_exception import HWException
  29 from remoteinstaller.installer.bmc_management.bmctools import BMCException
  30 from remoteinstaller.installer.catfile import CatFile
  31 from remoteinstaller.installer.catfile import CatFileException
  32
  33 class InstallException(Exception):
  34     pass
  35
  36 class Installer(object):
  37     SSH_OPTS = '-o StrictHostKeyChecking=no \
  38                 -o UserKnownHostsFile=/dev/null \
  39                 -o ServerAliveInterval=60'
  40
  41     def __init__(self, args):
  42         self._yaml_path = args.yaml
  43         self._boot_iso_path = args.boot_iso
  44         self._iso_url = args.iso
  45         self._logdir = args.logdir
  46         self._callback_url = args.callback_url
  47         self._client_key = args.client_key
  48         self._client_cert = args.client_cert
  49         self._ca_cert = args.ca_cert
  50         self._own_ip = args.host_ip
  51         self._tag = args.tag
  52         self._http_port = args.http_port
  53
  54         # TODO
  55         self._disable_bmc_initial_reset = True
  56         self._disable_other_bmc_reset = True
  57
  58         self._uc = self._read_user_config(self._yaml_path)
  59         self._vip = None
  60         self._first_controller_ip = None
  61         self._first_controller_bmc = None
  62
  63     @staticmethod
  64     def _read_user_config(config_file_path):
  65         logging.debug('Read user config from %s', config_file_path)
  66
  67         try:
  68             with open(config_file_path, 'r') as f:
  69                 y = load(f)
  70
  71             return y
  72         except Exception as ex:
  73             raise InstallException(str(ex))
  74
  75     @staticmethod
  76     def _execute_shell(command, desc=''):
  77         logging.debug('Execute %s with command: %s', desc, command)
  78
  79         p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  80         out, _ = p.communicate()
  81         if p.returncode:
  82             logging.warning('Failed to %s: %s (rc=%s)', desc, out, p.returncode)
  83             raise InstallException('Failed to {}'.format(desc))
  84
  85         return (p.returncode, out)
  86
  87     def _attach_iso_as_virtual_media(self, file_list):
  88         logging.info('Attach ISO as virtual media')
  89
  90         nfs_mount = os.path.dirname(self._boot_iso_path)
  91         boot_iso_filename = os.path.basename(self._boot_iso_path)
  92         patched_iso_filename = '{}/{}_{}'.format(nfs_mount, self._tag, boot_iso_filename)
  93
  94         self._patch_iso(patched_iso_filename, file_list)
  95
  96         self._first_controller_bmc.attach_virtual_cd(self._own_ip, nfs_mount, os.path.basename(patched_iso_filename))
  97
  98     def _create_cloud_config(self, first_controller):
  99         logging.info('Create network config file')
 100
 101         domain = self._uc['hosts'][first_controller].get('network_domain')
 102         extnet = self._uc['networking']['infra_external']['network_domains'][domain]
 103
 104         vlan = extnet.get('vlan')
 105         first_ip = extnet['ip_range_start']
 106         gateway = extnet['gateway']
 107         dns = self._uc['networking']['dns'][0]
 108         cidr = extnet['cidr']
 109         prefix = IPNetwork(cidr).prefixlen
 110
 111         self._vip = str(IPAddress(first_ip))
 112
 113         pre_allocated_ips = self._uc['hosts'][first_controller].get('pre_allocated_ips', None)
 114         if pre_allocated_ips:
 115             pre_allocated_infra_external_ip = pre_allocated_ips.get('infra_external', None)
 116             self._first_controller_ip = str(IPAddress(pre_allocated_infra_external_ip))
 117
 118         if not self._first_controller_ip:
 119             self._first_controller_ip = str(IPAddress(first_ip)+1)
 120
 121         controller_network_profile = self._uc['hosts'][first_controller]['network_profiles'][0]
 122         mappings = self._uc['network_profiles'][controller_network_profile]['interface_net_mapping']
 123         for interface in mappings:
 124             if 'infra_external' in mappings[interface]:
 125                 infra_external_interface = interface
 126                 break
 127
 128         if infra_external_interface.startswith('bond'):
 129             bonds = self._uc['network_profiles'][controller_network_profile]['bonding_interfaces']
 130             device = bonds[infra_external_interface][0]
 131         else:
 132             device = infra_external_interface
 133
 134         # TODO
 135         # ROOTFS_DISK
 136
 137         logging.debug('VLAN=%s', vlan)
 138         logging.debug('DEV=%s', device)
 139         logging.debug('IP=%s/%s', self._first_controller_ip, prefix)
 140         logging.debug('DGW=%s', gateway)
 141         logging.debug('NAMESERVER=%s', dns)
 142         logging.debug('ISO_URL="%s"', self._iso_url)
 143
 144         network_config_filename = '{}/network_config'.format(os.getcwd())
 145         with open(network_config_filename, 'w') as f:
 146             if vlan:
 147                 f.write('VLAN={}\n'.format(vlan))
 148             f.write('DEV={}\n'.format(device))
 149             f.write('IP={}/{}\n'.format(self._first_controller_ip, prefix))
 150             f.write('DGW={}\n'.format(gateway))
 151             f.write('NAMESERVER={}\n'.format(dns))
 152             f.write('\n')
 153             f.write('ISO_URL="{}"'.format(self._iso_url))
 154
 155         logging.debug('CALLBACK_URL="%s"', self._callback_url)
 156
 157         callback_url_filename = '{}/callback_url'.format(os.getcwd())
 158         with open(callback_url_filename, 'w') as f:
 159             f.write(self._callback_url)
 160
 161         if self._client_cert:
 162             return [self._yaml_path,
 163                     network_config_filename,
 164                     callback_url_filename,
 165                     self._client_key,
 166                     self._client_cert,
 167                     self._ca_cert]
 168
 169         return [self._yaml_path,
 170                 network_config_filename,
 171                 callback_url_filename]
 172
 173     def _patch_iso(self, iso_target, file_list):
 174         logging.info('Patch boot ISO')
 175         logging.debug('Original ISO: %s', self._boot_iso_path)
 176         logging.debug('Target ISO: %s', iso_target)
 177
 178         file_list_str = ' '.join(file_list)
 179         logging.debug('Files to add: %s', file_list_str)
 180
 181         self._execute_shell('/usr/bin/patchiso.sh {} {} {}'.format(self._boot_iso_path,
 182                                                                    iso_target,
 183                                                                    file_list_str), 'patch ISO')
 184
 185     def _put_file(self, ip, user, passwd, file_name, to_file=''):
 186         self._execute_shell('sshpass -p {} scp {} {} {}@{}:{}'.format(passwd,
 187                                                                       Installer.SSH_OPTS,
 188                                                                       file_name,
 189                                                                       user,
 190                                                                       ip,
 191                                                                       to_file))
 192
 193     def _get_file(self, log_dir, ip, user, passwd, file_name, recursive=False):
 194         if recursive:
 195             self._execute_shell('sshpass -p {} scp {} -r {}@{}:{} {}'.format(passwd,
 196                                                                              Installer.SSH_OPTS,
 197                                                                              user,
 198                                                                              ip,
 199                                                                              file_name,
 200                                                                              log_dir))
 201         else:
 202             self._execute_shell('sshpass -p {} scp {} {}@{}:{} {}'.format(passwd,
 203                                                                           Installer.SSH_OPTS,
 204                                                                           user,
 205                                                                           ip,
 206                                                                           file_name,
 207                                                                           log_dir))
 208
 209     def _get_node_logs(self, log_dir, ip, user, passwd):
 210         self._get_file(log_dir, ip, user, passwd, '/srv/deployment/log/cm.log')
 211         self._get_file(log_dir, ip, user, passwd, '/srv/deployment/log/bootstrap.log')
 212         self._get_file(log_dir, ip, user, passwd, '/var/log/ironic', recursive=True)
 213
 214     def _get_journal_logs(self, log_dir, ip, user, passwd):
 215         self._put_file(ip, user, passwd, '/opt/remoteinstaller/get_journals.sh')
 216         self._put_file(ip, user, passwd, '/opt/remoteinstaller/print_hosts.py')
 217
 218         self._execute_shell('sh ./get_journals.sh')
 219
 220         self._get_file(log_dir, ip, user, passwd, '/tmp/node_journals.tgz')
 221
 222     def _get_logs_from_console(self, log_dir, bmc, admin_user, admin_passwd):
 223         bmc_host = bmc.get_host()
 224         bmc_user = bmc.get_user()
 225         bmc_passwd = bmc.get_passwd()
 226
 227         log_file = '{}/cat_bootstrap.log'.format(log_dir)
 228         try:
 229             cat_file = CatFile(bmc_host, bmc_user, bmc_passwd, admin_user, admin_passwd)
 230             cat_file.cat('/srv/deployment/log/bootstrap.log', log_file)
 231         except CatFileException as ex:
 232             logging.info('Could not cat file from console: %s', str(ex))
 233
 234             cat_file = CatFile(bmc_host, bmc_user, bmc_passwd, 'root', 'root')
 235             cat_file.cat('/srv/deployment/log/bootstrap.log', log_file)
 236
 237     def get_logs(self, log_dir, admin_passwd):
 238         admin_user = self._uc['users']['admin_user_name']
 239
 240         ssh_command = 'nc -w1 {} 22 </dev/null &> /dev/null'.format(self._first_controller_ip)
 241         ssh_responds = self._execute_shell(ssh_command)
 242
 243         if ssh_responds:
 244             self._get_node_logs(log_dir, self._first_controller_ip, admin_user, admin_passwd)
 245
 246             self._get_journal_logs(log_dir, self._first_controller_ip, admin_user, admin_passwd)
 247         else:
 248             self._get_logs_from_console(log_dir,
 249                                         self._first_controller_bmc,
 250                                         admin_user,
 251                                         admin_passwd)
 252
 253     def install(self):
 254         try:
 255             logging.info('Start install')
 256
 257             if os.path.dirname(self._boot_iso_path) == '':
 258                 self._boot_iso_path = '{}/{}'.format(os.getcwd(), self._boot_iso_path)
 259
 260             if self._logdir:
 261                 if not os.path.exists(self._logdir):
 262                     os.makedirs(self._logdir)
 263             else:
 264                 self._logdir = '.'
 265
 266             other_bmcs = []
 267             first_controller = None
 268             for hw in sorted(self._uc['hosts']):
 269                 logging.info('HW node name is %s', hw)
 270
 271                 if not first_controller:
 272                     if 'controller' in self._uc['hosts'][hw]['service_profiles'] or \
 273                     'caas_master' in self._uc['hosts'][hw]['service_profiles']:
 274                         first_controller = hw
 275                         logging.info('HW is first controller')
 276
 277                 host = self._uc['hosts'][hw]['hwmgmt']['address']
 278                 user = self._uc['hosts'][hw]['hwmgmt']['user']
 279                 passwd = self._uc['hosts'][hw]['hwmgmt']['password']
 280
 281                 bmc_log_path = '{}/{}.log'.format(self._logdir, hw)
 282
 283                 try:
 284                     hw_data = hw_detect.get_hw_data(host, user, passwd, False)
 285                 except HWException as e:
 286                     error = "Harware not detected for {}: {}".format(hw, str(e))
 287                     logging.error(error)
 288                     raise BMCException(error)
 289
 290                 logging.info("Hardware belongs to %s product family", (hw_data['product_family']))
 291                 if 'Unknown' in hw_data['product_family']:
 292                     error = "Hardware not detected for %s" % hw
 293                     logging.error(error)
 294                     raise BMCException(error)
 295
 296                 bmc_mod_name = 'remoteinstaller.installer.bmc_management.{}'.format(hw_data['product_family'].lower())
 297                 bmc_mod = importlib.import_module(bmc_mod_name)
 298                 bmc_class = getattr(bmc_mod, hw_data['product_family'])
 299                 bmc = bmc_class(host, user, passwd, bmc_log_path)
 300                 bmc.set_host_name(hw)
 301
 302                 bmc.setup_sol()
 303
 304                 if hw != first_controller:
 305                     other_bmcs.append(bmc)
 306                     bmc.power('off')
 307                 else:
 308                     self._first_controller_bmc = bmc
 309
 310             logging.debug('First controller: %s', first_controller)
 311
 312             if not self._disable_bmc_initial_reset:
 313                 self._first_controller_bmc.reset()
 314                 time_after_reset = int(time.time())
 315
 316             if not self._disable_other_bmc_reset:
 317                 for bmc in other_bmcs:
 318                     bmc.reset()
 319
 320             if not self._disable_bmc_initial_reset:
 321                 # Make sure we sleep at least 6min after the first controller BMC reset
 322                 sleep_time = 6*60-int(time.time())-time_after_reset
 323                 if sleep_time > 0:
 324                     logging.debug('Waiting for first controller BMC to stabilize \
 325                                    (%s sec) after reset', sleep_time)
 326                     time.sleep(sleep_time)
 327
 328             config_file_names = self._create_cloud_config(first_controller)
 329
 330             self._first_controller_bmc.setup_boot_options_for_virtual_media()
 331
 332             self._attach_iso_as_virtual_media(config_file_names)
 333
 334             self._first_controller_bmc.boot_from_virtual_media()
 335
 336             self._first_controller_bmc.wait_for_bootup()
 337
 338             self._first_controller_bmc.close()
 339
 340             access_info = {'vip': self._vip,
 341                            'installer_node_ip': self._first_controller_ip,
 342                            'admin_user': self._uc['users']['admin_user_name']}
 343
 344             return access_info
 345         except BMCException as ex:
 346             logging.error('Installation failed: %s', str(ex))
 347             raise InstallException(str(ex))
 348
 349 def main():
 350     parser = argparse.ArgumentParser()
 351     parser.add_argument('-y', '--yaml', required=True,
 352                         help='User config yaml file path')
 353     parser.add_argument('-b', '--boot-iso', required=True,
 354                         help='Path to boot ISO image in NFS mount')
 355     parser.add_argument('-i', '--iso', required=True,
 356                         help='URL to ISO image')
 357     parser.add_argument('-d', '--debug', action='store_true', required=False,
 358                         help='Debug level for logging')
 359     parser.add_argument('-l', '--logdir', required=False,
 360                         help='Directory path for log files')
 361     parser.add_argument('-c', '--callback-url', required=True,
 362                         help='Callback URL for progress reporting')
 363     parser.add_argument('-K', '--client-key', required=True,
 364                         help='Client key file path')
 365     parser.add_argument('-C', '--client-cert', required=True,
 366                         help='Client cert file path')
 367     parser.add_argument('-A', '--ca-cert', required=True,
 368                         help='CA cert file path')
 369     parser.add_argument('-H', '--host-ip', required=True,
 370                         help='IP for hosting HTTPD and NFS')
 371     parser.add_argument('-T', '--http-port', required=False,
 372                         help='Port for HTTPD')
 373
 374     parsed_args = parser.parse_args()
 375
 376     if parsed_args.debug:
 377         log_level = logging.DEBUG
 378     else:
 379         log_level = logging.INFO
 380
 381     logging.basicConfig(stream=sys.stdout, level=log_level)
 382
 383     logging.debug('args: %s', parsed_args)
 384     installer = Installer(parsed_args)
 385
 386     installer.install()
 387
 388 if __name__ == "__main__":
 389     sys.exit(main())