src/remoteinstaller/installer/bmc_management/bmctools.py

   1 # Copyright 2019 Nokia
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import subprocess
  16 import time
  17 import logging
  18 import ipaddress
  19 import pexpect
  20
  21 class BMCException(Exception):
  22     pass
  23
  24 class BMC(object):
  25     def __init__(self, host, user, passwd, log_path=None):
  26         self._host = host
  27         self._user = user
  28         self._passwd = passwd
  29         if log_path:
  30             self._log_path = log_path
  31         else:
  32             self._log_path = 'console.log'
  33         self._sol = None
  34         self._host_name = None
  35
  36     def set_host_name(self, host_name):
  37         if not self._host_name:
  38             self._host_name = host_name
  39
  40     def get_host_name(self):
  41         if self._host_name:
  42             return self._host_name
  43
  44         return '<NONAME>'
  45
  46     def get_host(self):
  47         return self._host
  48
  49     def get_user(self):
  50         return self._user
  51
  52     def get_passwd(self):
  53         return self._passwd
  54
  55     def reset(self):
  56         logging.info('Reset BMC of %s: %s', self.get_host_name(), self.get_host())
  57
  58         self._run_ipmitool_command('bmc reset cold')
  59
  60         success = self._wait_for_bmc_reset(180)
  61         if not success:
  62             raise BMCException('BMC reset failed, BMC did not come up')
  63
  64     def _set_boot_from_virtual_media(self):
  65         raise NotImplementedError
  66
  67     def _detach_virtual_media(self):
  68         raise NotImplementedError
  69
  70     def _get_bmc_nfs_service_status(self):
  71         raise NotImplementedError
  72
  73     def _wait_for_bmc_responding(self, timeout, expected_to_respond=True):
  74         if expected_to_respond:
  75             logging.debug('Wait for BMC to start responding')
  76         else:
  77             logging.debug('Wait for BMC to stop responding')
  78
  79         start_time = int(time.time()*1000)
  80
  81         response = (not expected_to_respond)
  82         while response != expected_to_respond:
  83             rc, _ = self._run_ipmitool_command('bmc info', can_fail=True)
  84             response = (rc == 0)
  85
  86             if response == expected_to_respond:
  87                 break
  88
  89             time_now = int(time.time()*1000)
  90             if time_now-start_time > timeout*1000:
  91                 logging.debug('Wait timed out')
  92                 break
  93
  94             logging.debug('Still waiting for BMC')
  95             time.sleep(10)
  96
  97         return response == expected_to_respond
  98
  99     def _wait_for_bmc_webpage(self, timeout):
 100         host = ipaddress.ip_address(unicode(self._host))
 101         if host.version == 6:
 102             host = "[%s]" %host
 103
 104         command = 'curl -g --insecure -o /dev/null https://{}/index.html'.format(host)
 105
 106         start_time = int(time.time()*1000)
 107         rc = 1
 108         while rc != 0:
 109             p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 110             _, _ = p.communicate()
 111             rc = p.returncode
 112
 113             if rc == 0:
 114                 break
 115
 116             time_now = int(time.time()*1000)
 117             if time_now-start_time > timeout*1000:
 118                 logging.debug('Wait timed out')
 119                 break
 120
 121             logging.debug('Still waiting for BMC webpage')
 122             time.sleep(10)
 123
 124         return rc == 0
 125
 126     def _wait_for_bmc_not_responding(self, timeout):
 127         return self._wait_for_bmc_responding(timeout, False)
 128
 129     def _wait_for_bmc_reset(self, timeout):
 130         logging.debug('Wait for BMC to reset')
 131
 132         success = True
 133         if not self._wait_for_bmc_not_responding(timeout):
 134             success = False
 135             msg = 'BMC did not go down as expected'
 136             logging.warning(msg)
 137         else:
 138             logging.debug('As expected, BMC is not responding')
 139
 140         if not self._wait_for_bmc_responding(timeout):
 141             success = False
 142             msg = 'BMC did not come up as expected'
 143             logging.warning(msg)
 144         else:
 145             logging.debug('As expected, BMC is responding')
 146
 147         if not self._wait_for_bmc_webpage(timeout):
 148             success = False
 149             msg = 'BMC webpage did not start to respond'
 150             logging.warning(msg)
 151         else:
 152             logging.debug('As expected, BMC webpage is responding')
 153
 154         return success
 155
 156     def setup_boot_options_for_virtual_media(self):
 157         logging.debug('Setup boot options')
 158
 159         self._disable_boot_flag_timeout()
 160         self._set_boot_from_virtual_media()
 161
 162     def power(self, power_command):
 163         logging.debug('Run host power command (%s) %s', self._host, power_command)
 164
 165         return self._run_ipmitool_command('power {}'.format(power_command)).strip()
 166
 167     def wait_for_bootup(self):
 168         logging.debug('Wait for prompt after booting from hd')
 169
 170         try:
 171             self._expect_flag_in_console('localhost login:', timeout=1200)
 172         except BMCException as ex:
 173             self._send_to_console('\n')
 174             self._expect_flag_in_console('localhost login:', timeout=30)
 175
 176     def setup_sol(self):
 177         logging.debug('Setup SOL for %s', self._host)
 178
 179         self._run_ipmitool_command('sol set non-volatile-bit-rate 115.2')
 180         self._run_ipmitool_command('sol set volatile-bit-rate 115.2')
 181
 182     def boot_from_virtual_media(self):
 183         logging.info('Boot from virtual media')
 184
 185         self._trigger_boot()
 186         self._wait_for_virtual_media_detach_phase()
 187
 188         self._detach_virtual_media()
 189         self._set_boot_from_hd_no_boot()
 190
 191         logging.info('Boot should continue from disk now')
 192
 193     def close(self):
 194         if self._sol:
 195             self._sol.terminate()
 196         self._sol = None
 197
 198     @staticmethod
 199     def _convert_to_hex(ascii_string, padding=False, length=0):
 200         hex_value = ''.join('0x{} '.format(c.encode('hex')) for c in ascii_string).strip()
 201         if padding and (len(ascii_string) < length):
 202             hex_value += ''.join(' 0x00' for _ in range(len(ascii_string), length))
 203
 204         return hex_value
 205
 206     @staticmethod
 207     def _convert_to_ascii(hex_string):
 208         return ''.join('{}'.format(c.decode('hex')) for c in hex_string)
 209
 210     def _execute_ipmitool_command(self, ipmi_command):
 211         command = 'ipmitool -I lanplus -H {} -U {} -P {} {}'.format(self._host, self._user, self._passwd, ipmi_command)
 212
 213         p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 214         out, _ = p.communicate()
 215         rc = p.returncode
 216
 217         return (rc, out)
 218
 219     def _run_ipmitool_command(self, ipmi_command, can_fail=False, retries=5):
 220         logging.debug('Run ipmitool command: %s', ipmi_command)
 221
 222         if can_fail:
 223             return self._execute_ipmitool_command(ipmi_command)
 224         else:
 225             while retries > 0:
 226                 rc, out = self._execute_ipmitool_command(ipmi_command)
 227                 if not rc:
 228                     break
 229
 230                 if retries > 0:
 231                     logging.debug('Retry command')
 232                     time.sleep(5)
 233
 234                 retries -= 1
 235
 236             if rc:
 237                 logging.warning('ipmitool failed: %s', out)
 238                 raise BMCException('ipmitool call failed with rc: {}'.format(rc))
 239
 240         return out
 241
 242     def _run_ipmitool_raw_command(self, ipmi_raw_command):
 243         logging.debug('Run ipmitool raw command')
 244
 245         out = self._run_ipmitool_command('raw {}'.format(ipmi_raw_command))
 246
 247         out_bytes = out.replace('\n', '').strip().split(' ')
 248         return out_bytes
 249
 250     def _disable_boot_flag_timeout(self):
 251         logging.debug('Disable boot flag timeout (%s)', self._host)
 252
 253         status_code = self._run_ipmitool_raw_command('0x00 0x08 0x03 0x1f')
 254         if status_code[0] != '':
 255             raise BMCException('Could not disable boot flag timeout (rc={})'.format(status_code[0]))
 256
 257     def _open_console(self):
 258         logging.debug('Open SOL console (log in %s)', self._log_path)
 259
 260         expect_session = pexpect.spawn('ipmitool -I lanplus -H {} -U {} -P {} sol deactivate'.format(self._host, self._user, self._passwd))
 261         expect_session.expect(pexpect.EOF)
 262
 263         logfile = open(self._log_path, 'ab')
 264
 265         expect_session = pexpect.spawn('ipmitool -I lanplus -H {} -U {} -P {} sol activate'.format(self._host, self._user, self._passwd), timeout=None, logfile=logfile)
 266
 267         return expect_session
 268
 269     def _send_to_console(self, chars):
 270         logging.debug('Sending %s to console', chars.replace('\n', '\\n'))
 271
 272         if not self._sol:
 273             self._sol = self._open_console()
 274
 275         self._sol.send(chars)
 276
 277     def _expect_flag_in_console(self, flags, timeout=600):
 278         logging.debug('Expect a flag in console output within %s seconds ("%s")', timeout, flags)
 279
 280         time_begin = time.time()
 281
 282         remaining_time = timeout
 283
 284         while remaining_time > 0:
 285             if not self._sol:
 286                 try:
 287                     self._sol = self._open_console()
 288                 except pexpect.TIMEOUT as e:
 289                     logging.debug(e)
 290                     raise BMCException('Could not open console: {}'.format(str(e)))
 291
 292             try:
 293                 self._sol.expect(flags, timeout=remaining_time)
 294                 logging.debug('Flag found in log')
 295                 return
 296             except pexpect.TIMEOUT as e:
 297                 logging.debug(e)
 298                 raise BMCException('Expected message in console did not occur in time ({})'.format(flags))
 299             except pexpect.EOF as e:
 300                 logging.warning('Got EOF from console')
 301                 if 'SOL session closed by BMC' in self._sol.before:
 302                     logging.debug('Found: "SOL session closed by BMC" in console')
 303                 elapsed_time = time.time()-time_begin
 304                 remaining_time = timeout-elapsed_time
 305                 if remaining_time > 0:
 306                     logging.info('Retry to expect a flag in console, %s seconds remaining', remaining_time)
 307                     self.close()
 308
 309         raise BMCException('Expected message in console did not occur in time ({})'.format(flags))
 310
 311     def _wait_for_bios_settings_done(self):
 312         logging.debug('Wait until BIOS settings are updated')
 313
 314         self._expect_flag_in_console('Booting...', timeout=300)
 315
 316     def _set_boot_from_hd_no_boot(self):
 317         logging.debug('Set boot from hd (%s), no boot', self._host)
 318
 319         self._run_ipmitool_command('chassis bootdev disk options=persistent')
 320
 321     def _wait_for_virtual_media_detach_phase(self):
 322         logging.debug('Wait until virtual media can be detached')
 323
 324         self._expect_flag_in_console(['Copying cloud guest image',
 325                                       'Installing OS to HDD',
 326                                       'Extending partition and filesystem size'],
 327                                      timeout=1200)
 328
 329     def _wait_for_bmc_nfs_service(self, timeout, expected_status):
 330         logging.debug('Wait for BMC NFS service')
 331
 332         start_time = int(time.time()*1000)
 333
 334         status = ''
 335         while status != expected_status:
 336             status = self._get_bmc_nfs_service_status()
 337
 338             if status == expected_status or status == 'nfserror':
 339                 logging.debug('Breaking from wait loop. status = %s', status)
 340                 break
 341
 342             time_now = int(time.time()*1000)
 343             if time_now-start_time > timeout*1000:
 344                 logging.debug('Wait timed out')
 345                 break
 346             time.sleep(10)
 347
 348         return status == expected_status
 349
 350     def _trigger_boot(self):
 351         logging.debug('Trigger boot')
 352
 353         power_state = self.power('status')
 354         logging.debug('State is: %s', power_state)
 355         if power_state == 'Chassis Power is off':
 356             self.power('on')
 357         else:
 358             self.power('reset')