src/remoteinstaller/installer/bmc_management/bmctools.py

   1 # Copyright 2019 Nokia
   2 # Copyright 2020 ENEA
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 import subprocess
  17 import time
  18 import logging
  19 import ipaddress
  20 import pexpect
  21
  22 class BMCException(Exception):
  23     pass
  24
  25 class BMC(object):
  26     def __init__(self, host, user, passwd, priv_level='ADMINISTRATOR', log_path=None):
  27         self._host = host
  28         self._user = user
  29         self._passwd = passwd
  30         self._priv_level = priv_level
  31         if log_path:
  32             self._log_path = log_path
  33         else:
  34             self._log_path = 'console.log'
  35         self._sol = None
  36         self._host_name = None
  37
  38     def set_host_name(self, host_name):
  39         if not self._host_name:
  40             self._host_name = host_name
  41
  42     def get_host_name(self):
  43         if self._host_name:
  44             return self._host_name
  45
  46         return '<NONAME>'
  47
  48     def get_host(self):
  49         return self._host
  50
  51     def get_user(self):
  52         return self._user
  53
  54     def get_passwd(self):
  55         return self._passwd
  56
  57     def get_priv_level(self):
  58         return self._priv_level
  59
  60     def reset(self):
  61         logging.info('Reset BMC of %s: %s', self.get_host_name(), self.get_host())
  62
  63         self._run_ipmitool_command('bmc reset cold')
  64
  65         success = self._wait_for_bmc_reset(180)
  66         if not success:
  67             raise BMCException('BMC reset failed, BMC did not come up')
  68
  69     def _set_boot_from_virtual_media(self):
  70         raise NotImplementedError
  71
  72     def _detach_virtual_media(self):
  73         raise NotImplementedError
  74
  75     def _get_bmc_nfs_service_status(self):
  76         raise NotImplementedError
  77
  78     def _wait_for_bmc_responding(self, timeout, expected_to_respond=True):
  79         if expected_to_respond:
  80             logging.debug('Wait for BMC to start responding')
  81         else:
  82             logging.debug('Wait for BMC to stop responding')
  83
  84         start_time = int(time.time()*1000)
  85
  86         response = (not expected_to_respond)
  87         while response != expected_to_respond:
  88             rc, _ = self._run_ipmitool_command('bmc info', can_fail=True)
  89             response = (rc == 0)
  90
  91             if response == expected_to_respond:
  92                 break
  93
  94             time_now = int(time.time()*1000)
  95             if time_now-start_time > timeout*1000:
  96                 logging.debug('Wait timed out')
  97                 break
  98
  99             logging.debug('Still waiting for BMC')
 100             time.sleep(10)
 101
 102         return response == expected_to_respond
 103
 104     def _wait_for_bmc_webpage(self, timeout):
 105         host = ipaddress.ip_address(unicode(self._host))
 106         if host.version == 6:
 107             host = "[%s]" %host
 108
 109         command = 'curl -g --insecure -o /dev/null https://{}/index.html'.format(host)
 110
 111         start_time = int(time.time()*1000)
 112         rc = 1
 113         while rc != 0:
 114             p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 115             _, _ = p.communicate()
 116             rc = p.returncode
 117
 118             if rc == 0:
 119                 break
 120
 121             time_now = int(time.time()*1000)
 122             if time_now-start_time > timeout*1000:
 123                 logging.debug('Wait timed out')
 124                 break
 125
 126             logging.debug('Still waiting for BMC webpage')
 127             time.sleep(10)
 128
 129         return rc == 0
 130
 131     def _wait_for_bmc_not_responding(self, timeout):
 132         return self._wait_for_bmc_responding(timeout, False)
 133
 134     def _wait_for_bmc_reset(self, timeout):
 135         logging.debug('Wait for BMC to reset')
 136
 137         success = True
 138         if not self._wait_for_bmc_not_responding(timeout):
 139             success = False
 140             msg = 'BMC did not go down as expected'
 141             logging.warning(msg)
 142         else:
 143             logging.debug('As expected, BMC is not responding')
 144
 145         if not self._wait_for_bmc_responding(timeout):
 146             success = False
 147             msg = 'BMC did not come up as expected'
 148             logging.warning(msg)
 149         else:
 150             logging.debug('As expected, BMC is responding')
 151
 152         if not self._wait_for_bmc_webpage(timeout):
 153             success = False
 154             msg = 'BMC webpage did not start to respond'
 155             logging.warning(msg)
 156         else:
 157             logging.debug('As expected, BMC webpage is responding')
 158
 159         return success
 160
 161     def setup_boot_options_for_virtual_media(self):
 162         logging.debug('Setup boot options')
 163
 164         self._disable_boot_flag_timeout()
 165         self._set_boot_from_virtual_media()
 166
 167     def power(self, power_command):
 168         logging.debug('Run host power command (%s) %s', self._host, power_command)
 169
 170         return self._run_ipmitool_command('power {}'.format(power_command)).strip()
 171
 172     def wait_for_bootup(self):
 173         logging.debug('Wait for prompt after booting from hd')
 174
 175         try:
 176             self._expect_flag_in_console('localhost login:', timeout=1200)
 177         except BMCException as ex:
 178             self._send_to_console('\n')
 179             self._expect_flag_in_console('localhost login:', timeout=30)
 180
 181     def setup_sol(self):
 182         logging.debug('Setup SOL for %s', self._host)
 183
 184         self._run_ipmitool_command('sol set non-volatile-bit-rate 115.2')
 185         self._run_ipmitool_command('sol set volatile-bit-rate 115.2')
 186
 187     def boot_from_virtual_media(self):
 188         logging.info('Boot from virtual media')
 189
 190         self._trigger_boot()
 191         self._wait_for_virtual_media_detach_phase()
 192
 193         self._detach_virtual_media()
 194         self._set_boot_from_hd_no_boot()
 195
 196         logging.info('Boot should continue from disk now')
 197
 198     def close(self):
 199         if self._sol:
 200             self._sol.terminate()
 201         self._sol = None
 202
 203     @staticmethod
 204     def _convert_to_hex(ascii_string, padding=False, length=0):
 205         if padding:
 206             ascii_string = ascii_string.ljust(length, '\0')
 207         return ' '.join('0x{}'.format(c.encode('hex')) for c in ascii_string)
 208
 209     @staticmethod
 210     def _convert_to_ascii(hex_string):
 211         return hex_string.replace('0x','').replace(' ','').decode('hex')
 212
 213     def _execute_ipmitool_command(self, ipmi_command):
 214         command = 'ipmitool -I lanplus -H {} -U {} -P {} -L {} {}'.format(self._host, self._user, self._passwd, self._priv_level, ipmi_command)
 215
 216         p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 217         out, _ = p.communicate()
 218         rc = p.returncode
 219
 220         return (rc, out)
 221
 222     def _run_ipmitool_command(self, ipmi_command, can_fail=False, retries=5):
 223         logging.debug('Run ipmitool command: %s', ipmi_command)
 224
 225         if can_fail:
 226             return self._execute_ipmitool_command(ipmi_command)
 227         else:
 228             while retries > 0:
 229                 rc, out = self._execute_ipmitool_command(ipmi_command)
 230                 if not rc:
 231                     break
 232
 233                 if retries > 0:
 234                     logging.debug('Retry command')
 235                     time.sleep(5)
 236
 237                 retries -= 1
 238
 239             if rc:
 240                 logging.warning('ipmitool failed: %s', out)
 241                 raise BMCException('ipmitool call failed with rc: {}'.format(rc))
 242
 243         return out
 244
 245     def _run_ipmitool_raw_command(self, ipmi_raw_command):
 246         logging.debug('Run ipmitool raw command')
 247
 248         out = self._run_ipmitool_command('raw {}'.format(ipmi_raw_command))
 249
 250         out_bytes = out.replace('\n', '').strip().split(' ')
 251         return out_bytes
 252
 253     def _disable_boot_flag_timeout(self):
 254         logging.debug('Disable boot flag timeout (%s)', self._host)
 255
 256         status_code = self._run_ipmitool_raw_command('0x00 0x08 0x03 0x1f')
 257         if status_code[0] != '':
 258             raise BMCException('Could not disable boot flag timeout (rc={})'.format(status_code[0]))
 259
 260     def _open_console(self):
 261         logging.debug('Open SOL console (log in %s)', self._log_path)
 262
 263         expect_session = pexpect.spawn('ipmitool -I lanplus -H {} -U {} -P {} -L {} sol deactivate'.format(self._host, self._user, self._passwd, self._priv_level))
 264         expect_session.expect(pexpect.EOF)
 265
 266         logfile = open(self._log_path, 'ab')
 267
 268         expect_session = pexpect.spawn('ipmitool -I lanplus -H {} -U {} -P {} -L {} sol activate'.format(self._host, self._user, self._passwd, self._priv_level), timeout=None, logfile=logfile)
 269
 270         return expect_session
 271
 272     def _send_to_console(self, chars):
 273         logging.debug('Sending %s to console', chars.replace('\n', '\\n'))
 274
 275         if not self._sol:
 276             self._sol = self._open_console()
 277
 278         self._sol.send(chars)
 279
 280     def _expect_flag_in_console(self, flags, timeout=600):
 281         logging.debug('Expect a flag in console output within %s seconds ("%s")', timeout, flags)
 282
 283         time_begin = time.time()
 284
 285         remaining_time = timeout
 286
 287         while remaining_time > 0:
 288             if not self._sol:
 289                 try:
 290                     self._sol = self._open_console()
 291                 except pexpect.TIMEOUT as e:
 292                     logging.debug(e)
 293                     raise BMCException('Could not open console: {}'.format(str(e)))
 294
 295             try:
 296                 self._sol.expect(flags, timeout=remaining_time)
 297                 logging.debug('Flag found in log')
 298                 return
 299             except pexpect.TIMEOUT as e:
 300                 logging.debug(e)
 301                 raise BMCException('Expected message in console did not occur in time ({})'.format(flags))
 302             except pexpect.EOF as e:
 303                 logging.warning('Got EOF from console')
 304                 if 'SOL session closed by BMC' in self._sol.before:
 305                     logging.debug('Found: "SOL session closed by BMC" in console')
 306                 elapsed_time = time.time()-time_begin
 307                 remaining_time = timeout-elapsed_time
 308                 if remaining_time > 0:
 309                     logging.info('Retry to expect a flag in console, %s seconds remaining', remaining_time)
 310                     self.close()
 311
 312         raise BMCException('Expected message in console did not occur in time ({})'.format(flags))
 313
 314     def _wait_for_bios_settings_done(self):
 315         logging.debug('Wait until BIOS settings are updated')
 316
 317         self._expect_flag_in_console('Booting...', timeout=300)
 318
 319     def _set_boot_from_hd_no_boot(self):
 320         logging.debug('Set boot from hd (%s), no boot', self._host)
 321
 322         self._run_ipmitool_command('chassis bootdev disk options=persistent')
 323
 324     def _wait_for_virtual_media_detach_phase(self):
 325         logging.debug('Wait until virtual media can be detached')
 326
 327         self._expect_flag_in_console(['Copying cloud guest image',
 328                                       'Installing OS to HDD',
 329                                       'Extending partition and filesystem size'],
 330                                      timeout=1200)
 331
 332     def _wait_for_bmc_nfs_service(self, timeout, expected_status):
 333         logging.debug('Wait for BMC NFS service')
 334
 335         start_time = int(time.time()*1000)
 336
 337         status = ''
 338         while status != expected_status:
 339             status = self._get_bmc_nfs_service_status()
 340
 341             if status == expected_status or status == 'nfserror':
 342                 logging.debug('Breaking from wait loop. status = %s', status)
 343                 break
 344
 345             time_now = int(time.time()*1000)
 346             if time_now-start_time > timeout*1000:
 347                 logging.debug('Wait timed out')
 348                 break
 349             time.sleep(10)
 350
 351         return status == expected_status
 352
 353     def _trigger_boot(self):
 354         logging.debug('Trigger boot')
 355
 356         power_state = self.power('status')
 357         logging.debug('State is: %s', power_state)
 358         if power_state == 'Chassis Power is off':
 359             self.power('on')
 360         else:
 361             self.power('reset')