src/remoteinstaller/installer/bmc_management/bmctools.py

   1 # Copyright 2019 Nokia
   2
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import subprocess
  16 import time
  17 import logging
  18 import ipaddress
  19 import pexpect
  20
  21 class BMCException(Exception):
  22     pass
  23
  24 class BMC(object):
  25     def __init__(self, host, user, passwd, priv_level='ADMINISTRATOR', log_path=None):
  26         self._host = host
  27         self._user = user
  28         self._passwd = passwd
  29         self._priv_level = priv_level
  30         if log_path:
  31             self._log_path = log_path
  32         else:
  33             self._log_path = 'console.log'
  34         self._sol = None
  35         self._host_name = None
  36
  37     def set_host_name(self, host_name):
  38         if not self._host_name:
  39             self._host_name = host_name
  40
  41     def get_host_name(self):
  42         if self._host_name:
  43             return self._host_name
  44
  45         return '<NONAME>'
  46
  47     def get_host(self):
  48         return self._host
  49
  50     def get_user(self):
  51         return self._user
  52
  53     def get_passwd(self):
  54         return self._passwd
  55
  56     def get_priv_level(self):
  57         return self._priv_level
  58
  59     def reset(self):
  60         logging.info('Reset BMC of %s: %s', self.get_host_name(), self.get_host())
  61
  62         self._run_ipmitool_command('bmc reset cold')
  63
  64         success = self._wait_for_bmc_reset(180)
  65         if not success:
  66             raise BMCException('BMC reset failed, BMC did not come up')
  67
  68     def _set_boot_from_virtual_media(self):
  69         raise NotImplementedError
  70
  71     def _detach_virtual_media(self):
  72         raise NotImplementedError
  73
  74     def _get_bmc_nfs_service_status(self):
  75         raise NotImplementedError
  76
  77     def _wait_for_bmc_responding(self, timeout, expected_to_respond=True):
  78         if expected_to_respond:
  79             logging.debug('Wait for BMC to start responding')
  80         else:
  81             logging.debug('Wait for BMC to stop responding')
  82
  83         start_time = int(time.time()*1000)
  84
  85         response = (not expected_to_respond)
  86         while response != expected_to_respond:
  87             rc, _ = self._run_ipmitool_command('bmc info', can_fail=True)
  88             response = (rc == 0)
  89
  90             if response == expected_to_respond:
  91                 break
  92
  93             time_now = int(time.time()*1000)
  94             if time_now-start_time > timeout*1000:
  95                 logging.debug('Wait timed out')
  96                 break
  97
  98             logging.debug('Still waiting for BMC')
  99             time.sleep(10)
 100
 101         return response == expected_to_respond
 102
 103     def _wait_for_bmc_webpage(self, timeout):
 104         host = ipaddress.ip_address(unicode(self._host))
 105         if host.version == 6:
 106             host = "[%s]" %host
 107
 108         command = 'curl -g --insecure -o /dev/null https://{}/index.html'.format(host)
 109
 110         start_time = int(time.time()*1000)
 111         rc = 1
 112         while rc != 0:
 113             p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 114             _, _ = p.communicate()
 115             rc = p.returncode
 116
 117             if rc == 0:
 118                 break
 119
 120             time_now = int(time.time()*1000)
 121             if time_now-start_time > timeout*1000:
 122                 logging.debug('Wait timed out')
 123                 break
 124
 125             logging.debug('Still waiting for BMC webpage')
 126             time.sleep(10)
 127
 128         return rc == 0
 129
 130     def _wait_for_bmc_not_responding(self, timeout):
 131         return self._wait_for_bmc_responding(timeout, False)
 132
 133     def _wait_for_bmc_reset(self, timeout):
 134         logging.debug('Wait for BMC to reset')
 135
 136         success = True
 137         if not self._wait_for_bmc_not_responding(timeout):
 138             success = False
 139             msg = 'BMC did not go down as expected'
 140             logging.warning(msg)
 141         else:
 142             logging.debug('As expected, BMC is not responding')
 143
 144         if not self._wait_for_bmc_responding(timeout):
 145             success = False
 146             msg = 'BMC did not come up as expected'
 147             logging.warning(msg)
 148         else:
 149             logging.debug('As expected, BMC is responding')
 150
 151         if not self._wait_for_bmc_webpage(timeout):
 152             success = False
 153             msg = 'BMC webpage did not start to respond'
 154             logging.warning(msg)
 155         else:
 156             logging.debug('As expected, BMC webpage is responding')
 157
 158         return success
 159
 160     def setup_boot_options_for_virtual_media(self):
 161         logging.debug('Setup boot options')
 162
 163         self._disable_boot_flag_timeout()
 164         self._set_boot_from_virtual_media()
 165
 166     def power(self, power_command):
 167         logging.debug('Run host power command (%s) %s', self._host, power_command)
 168
 169         return self._run_ipmitool_command('power {}'.format(power_command)).strip()
 170
 171     def wait_for_bootup(self):
 172         logging.debug('Wait for prompt after booting from hd')
 173
 174         try:
 175             self._expect_flag_in_console('localhost login:', timeout=1200)
 176         except BMCException as ex:
 177             self._send_to_console('\n')
 178             self._expect_flag_in_console('localhost login:', timeout=30)
 179
 180     def setup_sol(self):
 181         logging.debug('Setup SOL for %s', self._host)
 182
 183         self._run_ipmitool_command('sol set non-volatile-bit-rate 115.2')
 184         self._run_ipmitool_command('sol set volatile-bit-rate 115.2')
 185
 186     def boot_from_virtual_media(self):
 187         logging.info('Boot from virtual media')
 188
 189         self._trigger_boot()
 190         self._wait_for_virtual_media_detach_phase()
 191
 192         self._detach_virtual_media()
 193         self._set_boot_from_hd_no_boot()
 194
 195         logging.info('Boot should continue from disk now')
 196
 197     def close(self):
 198         if self._sol:
 199             self._sol.terminate()
 200         self._sol = None
 201
 202     @staticmethod
 203     def _convert_to_hex(ascii_string, padding=False, length=0):
 204         hex_value = ''.join('0x{} '.format(c.encode('hex')) for c in ascii_string).strip()
 205         if padding and (len(ascii_string) < length):
 206             hex_value += ''.join(' 0x00' for _ in range(len(ascii_string), length))
 207
 208         return hex_value
 209
 210     @staticmethod
 211     def _convert_to_ascii(hex_string):
 212         return ''.join('{}'.format(c.decode('hex')) for c in hex_string)
 213
 214     def _execute_ipmitool_command(self, ipmi_command):
 215         command = 'ipmitool -I lanplus -H {} -U {} -P {} -L {} {}'.format(self._host, self._user, self._passwd, self._priv_level, ipmi_command)
 216
 217         p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 218         out, _ = p.communicate()
 219         rc = p.returncode
 220
 221         return (rc, out)
 222
 223     def _run_ipmitool_command(self, ipmi_command, can_fail=False, retries=5):
 224         logging.debug('Run ipmitool command: %s', ipmi_command)
 225
 226         if can_fail:
 227             return self._execute_ipmitool_command(ipmi_command)
 228         else:
 229             while retries > 0:
 230                 rc, out = self._execute_ipmitool_command(ipmi_command)
 231                 if not rc:
 232                     break
 233
 234                 if retries > 0:
 235                     logging.debug('Retry command')
 236                     time.sleep(5)
 237
 238                 retries -= 1
 239
 240             if rc:
 241                 logging.warning('ipmitool failed: %s', out)
 242                 raise BMCException('ipmitool call failed with rc: {}'.format(rc))
 243
 244         return out
 245
 246     def _run_ipmitool_raw_command(self, ipmi_raw_command):
 247         logging.debug('Run ipmitool raw command')
 248
 249         out = self._run_ipmitool_command('raw {}'.format(ipmi_raw_command))
 250
 251         out_bytes = out.replace('\n', '').strip().split(' ')
 252         return out_bytes
 253
 254     def _disable_boot_flag_timeout(self):
 255         logging.debug('Disable boot flag timeout (%s)', self._host)
 256
 257         status_code = self._run_ipmitool_raw_command('0x00 0x08 0x03 0x1f')
 258         if status_code[0] != '':
 259             raise BMCException('Could not disable boot flag timeout (rc={})'.format(status_code[0]))
 260
 261     def _open_console(self):
 262         logging.debug('Open SOL console (log in %s)', self._log_path)
 263
 264         expect_session = pexpect.spawn('ipmitool -I lanplus -H {} -U {} -P {} -L {} sol deactivate'.format(self._host, self._user, self._passwd, self._priv_level))
 265         expect_session.expect(pexpect.EOF)
 266
 267         logfile = open(self._log_path, 'ab')
 268
 269         expect_session = pexpect.spawn('ipmitool -I lanplus -H {} -U {} -P {} -L {} sol activate'.format(self._host, self._user, self._passwd, self._priv_level), timeout=None, logfile=logfile)
 270
 271         return expect_session
 272
 273     def _send_to_console(self, chars):
 274         logging.debug('Sending %s to console', chars.replace('\n', '\\n'))
 275
 276         if not self._sol:
 277             self._sol = self._open_console()
 278
 279         self._sol.send(chars)
 280
 281     def _expect_flag_in_console(self, flags, timeout=600):
 282         logging.debug('Expect a flag in console output within %s seconds ("%s")', timeout, flags)
 283
 284         time_begin = time.time()
 285
 286         remaining_time = timeout
 287
 288         while remaining_time > 0:
 289             if not self._sol:
 290                 try:
 291                     self._sol = self._open_console()
 292                 except pexpect.TIMEOUT as e:
 293                     logging.debug(e)
 294                     raise BMCException('Could not open console: {}'.format(str(e)))
 295
 296             try:
 297                 self._sol.expect(flags, timeout=remaining_time)
 298                 logging.debug('Flag found in log')
 299                 return
 300             except pexpect.TIMEOUT as e:
 301                 logging.debug(e)
 302                 raise BMCException('Expected message in console did not occur in time ({})'.format(flags))
 303             except pexpect.EOF as e:
 304                 logging.warning('Got EOF from console')
 305                 if 'SOL session closed by BMC' in self._sol.before:
 306                     logging.debug('Found: "SOL session closed by BMC" in console')
 307                 elapsed_time = time.time()-time_begin
 308                 remaining_time = timeout-elapsed_time
 309                 if remaining_time > 0:
 310                     logging.info('Retry to expect a flag in console, %s seconds remaining', remaining_time)
 311                     self.close()
 312
 313         raise BMCException('Expected message in console did not occur in time ({})'.format(flags))
 314
 315     def _wait_for_bios_settings_done(self):
 316         logging.debug('Wait until BIOS settings are updated')
 317
 318         self._expect_flag_in_console('Booting...', timeout=300)
 319
 320     def _set_boot_from_hd_no_boot(self):
 321         logging.debug('Set boot from hd (%s), no boot', self._host)
 322
 323         self._run_ipmitool_command('chassis bootdev disk options=persistent')
 324
 325     def _wait_for_virtual_media_detach_phase(self):
 326         logging.debug('Wait until virtual media can be detached')
 327
 328         self._expect_flag_in_console(['Copying cloud guest image',
 329                                       'Installing OS to HDD',
 330                                       'Extending partition and filesystem size'],
 331                                      timeout=1200)
 332
 333     def _wait_for_bmc_nfs_service(self, timeout, expected_status):
 334         logging.debug('Wait for BMC NFS service')
 335
 336         start_time = int(time.time()*1000)
 337
 338         status = ''
 339         while status != expected_status:
 340             status = self._get_bmc_nfs_service_status()
 341
 342             if status == expected_status or status == 'nfserror':
 343                 logging.debug('Breaking from wait loop. status = %s', status)
 344                 break
 345
 346             time_now = int(time.time()*1000)
 347             if time_now-start_time > timeout*1000:
 348                 logging.debug('Wait timed out')
 349                 break
 350             time.sleep(10)
 351
 352         return status == expected_status
 353
 354     def _trigger_boot(self):
 355         logging.debug('Trigger boot')
 356
 357         power_state = self.power('status')
 358         logging.debug('State is: %s', power_state)
 359         if power_state == 'Chassis Power is off':
 360             self.power('on')
 361         else:
 362             self.power('reset')