X-Git-Url: https://gerrit.akraino.org/r/gitweb?p=ta%2Fremote-installer.git;a=blobdiff_plain;f=src%2Fremoteinstaller%2Finstaller%2Fbmc_management%2Fbmctools.py;fp=src%2Fremoteinstaller%2Finstaller%2Fbmc_management%2Fbmctools.py;h=dfdeaea5e97abbe84a9ee89b7d9447a9ca11f84a;hp=0000000000000000000000000000000000000000;hb=f9adb9143ef94b16ae16941652e75deccad506ef;hpb=3a2c5cc0fe9265242032882d68129b7faf47235c

diff --git a/src/remoteinstaller/installer/bmc_management/bmctools.py b/src/remoteinstaller/installer/bmc_management/bmctools.py
new file mode 100644
index 0000000..dfdeaea
--- /dev/null
+++ b/src/remoteinstaller/installer/bmc_management/bmctools.py
@@ -0,0 +1,356 @@
+# Copyright 2019 Nokia
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+import time
+import logging
+import ipaddress
+import pexpect
+
+class BMCException(Exception):
+    pass
+
+class BMC(object):
+    def __init__(self, host, user, passwd, log_path=None):
+        self._host = host
+        self._user = user
+        self._passwd = passwd
+        if log_path:
+            self._log_path = log_path
+        else:
+            self._log_path = 'console.log'
+        self._sol = None
+        self._host_name = None
+
+    def set_host_name(self, host_name):
+        if not self._host_name:
+            self._host_name = host_name
+
+    def get_host_name(self):
+        if self._host_name:
+            return self._host_name
+
+        return '<NONAME>'
+
+    def get_host(self):
+        return self._host
+
+    def get_user(self):
+        return self._user
+
+    def get_passwd(self):
+        return self._passwd
+
+    def reset(self):
+        logging.info('Reset BMC of %s: %s', self.get_host_name(), self.get_host())
+
+        self._run_ipmitool_command('bmc reset cold')
+
+        success = self._wait_for_bmc_reset(180)
+        if not success:
+            raise BMCException('BMC reset failed, BMC did not come up')
+
+    def _set_boot_from_virtual_media(self):
+        raise NotImplementedError
+
+    def _detach_virtual_media(self):
+        raise NotImplementedError
+
+    def _get_bmc_nfs_service_status(self):
+        raise NotImplementedError
+
+    def _wait_for_bmc_responding(self, timeout, expected_to_respond=True):
+        if expected_to_respond:
+            logging.debug('Wait for BMC to start responding')
+        else:
+            logging.debug('Wait for BMC to stop responding')
+
+        start_time = int(time.time()*1000)
+
+        response = (not expected_to_respond)
+        while response != expected_to_respond:
+            rc, _ = self._run_ipmitool_command('bmc info', can_fail=True)
+            response = (rc == 0)
+
+            if response == expected_to_respond:
+                break
+
+            time_now = int(time.time()*1000)
+            if time_now-start_time > timeout*1000:
+                logging.debug('Wait timed out')
+                break
+
+            logging.debug('Still waiting for BMC')
+            time.sleep(10)
+
+        return response == expected_to_respond
+
+    def _wait_for_bmc_webpage(self, timeout):
+        host = ipaddress.ip_address(unicode(self._host))
+        if host.version == 6:
+            host = "[%s]" %host
+
+        command = 'curl -g --insecure -o /dev/null https://{}/index.html'.format(host)
+
+        start_time = int(time.time()*1000)
+        rc = 1
+        while rc != 0:
+            p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+            _, _ = p.communicate()
+            rc = p.returncode
+
+            if rc == 0:
+                break
+
+            time_now = int(time.time()*1000)
+            if time_now-start_time > timeout*1000:
+                logging.debug('Wait timed out')
+                break
+
+            logging.debug('Still waiting for BMC webpage')
+            time.sleep(10)
+
+        return rc == 0
+
+    def _wait_for_bmc_not_responding(self, timeout):
+        return self._wait_for_bmc_responding(timeout, False)
+
+    def _wait_for_bmc_reset(self, timeout):
+        logging.debug('Wait for BMC to reset')
+
+        success = True
+        if not self._wait_for_bmc_not_responding(timeout):
+            success = False
+            msg = 'BMC did not go down as expected'
+            logging.warning(msg)
+        else:
+            logging.debug('As expected, BMC is not responding')
+
+        if not self._wait_for_bmc_responding(timeout):
+            success = False
+            msg = 'BMC did not come up as expected'
+            logging.warning(msg)
+        else:
+            logging.debug('As expected, BMC is responding')
+
+        if not self._wait_for_bmc_webpage(timeout):
+            success = False
+            msg = 'BMC webpage did not start to respond'
+            logging.warning(msg)
+        else:
+            logging.debug('As expected, BMC webpage is responding')
+
+        return success
+
+    def setup_boot_options_for_virtual_media(self):
+        logging.debug('Setup boot options')
+
+        self._disable_boot_flag_timeout()
+        self._set_boot_from_virtual_media()
+
+    def power(self, power_command):
+        logging.debug('Run host power command (%s) %s', self._host, power_command)
+
+        return self._run_ipmitool_command('power {}'.format(power_command)).strip()
+
+    def wait_for_bootup(self):
+        logging.debug('Wait for prompt after booting from hd')
+
+        try:
+            self._expect_flag_in_console('localhost login:', timeout=1200)
+        except BMCException as ex:
+            self._send_to_console('\n')
+            self._expect_flag_in_console('localhost login:', timeout=30)
+
+    def setup_sol(self):
+        logging.debug('Setup SOL for %s', self._host)
+
+        self._run_ipmitool_command('sol set non-volatile-bit-rate 115.2')
+        self._run_ipmitool_command('sol set volatile-bit-rate 115.2')
+
+    def boot_from_virtual_media(self):
+        logging.info('Boot from virtual media')
+
+        self._trigger_boot()
+        self._wait_for_virtual_media_detach_phase()
+
+        self._detach_virtual_media()
+        self._set_boot_from_hd_no_boot()
+
+        logging.info('Boot should continue from disk now')
+
+    def close(self):
+        if self._sol:
+            self._sol.terminate()
+        self._sol = None
+
+    @staticmethod
+    def _convert_to_hex(ascii_string, padding=False, length=0):
+        hex_value = ''.join('0x{} '.format(c.encode('hex')) for c in ascii_string).strip()
+        if padding and (len(ascii_string) < length):
+            hex_value += ''.join(' 0x00' for _ in range(len(ascii_string), length))
+
+        return hex_value
+
+    @staticmethod
+    def _convert_to_ascii(hex_string):
+        return ''.join('{}'.format(c.decode('hex')) for c in hex_string)
+
+    def _execute_ipmitool_command(self, ipmi_command):
+        command = 'ipmitool -I lanplus -H {} -U {} -P {} {}'.format(self._host, self._user, self._passwd, ipmi_command)
+
+        p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        out, _ = p.communicate()
+        rc = p.returncode
+
+        return (rc, out)
+
+    def _run_ipmitool_command(self, ipmi_command, can_fail=False, retries=5):
+        logging.debug('Run ipmitool command: %s', ipmi_command)
+
+        if can_fail:
+            return self._execute_ipmitool_command(ipmi_command)
+        else:
+            while retries > 0:
+                rc, out = self._execute_ipmitool_command(ipmi_command)
+                if not rc:
+                    break
+
+                if retries > 0:
+                    logging.debug('Retry command')
+                    time.sleep(5)
+
+                retries -= 1
+
+            if rc:
+                logging.warning('ipmitool failed: %s', out)
+                raise BMCException('ipmitool call failed with rc: {}'.format(rc))
+
+        return out
+
+    def _run_ipmitool_raw_command(self, ipmi_raw_command):
+        logging.debug('Run ipmitool raw command')
+
+        out = self._run_ipmitool_command('raw {}'.format(ipmi_raw_command))
+
+        out_bytes = out.replace('\n', '').strip().split(' ')
+        return out_bytes
+
+    def _disable_boot_flag_timeout(self):
+        logging.debug('Disable boot flag timeout (%s)', self._host)
+
+        status_code = self._run_ipmitool_raw_command('0x00 0x08 0x03 0x1f')
+        if status_code[0] != '':
+            raise BMCException('Could not disable boot flag timeout (rc={})'.format(status_code[0]))
+
+    def _open_console(self):
+        logging.debug('Open SOL console (log in %s)', self._log_path)
+
+        expect_session = pexpect.spawn('ipmitool -I lanplus -H {} -U {} -P {} sol deactivate'.format(self._host, self._user, self._passwd))
+        expect_session.expect(pexpect.EOF)
+
+        logfile = open(self._log_path, 'ab')
+
+        expect_session = pexpect.spawn('ipmitool -I lanplus -H {} -U {} -P {} sol activate'.format(self._host, self._user, self._passwd), timeout=None, logfile=logfile)
+
+        return expect_session
+
+    def _send_to_console(self, chars):
+        logging.debug('Sending %s to console', chars.replace('\n', '\\n'))
+
+        if not self._sol:
+            self._sol = self._open_console()
+
+        self._sol.send(chars)
+
+    def _expect_flag_in_console(self, flags, timeout=600):
+        logging.debug('Expect a flag in console output within %s seconds ("%s")', timeout, flags)
+
+        time_begin = time.time()
+
+        remaining_time = timeout
+
+        while remaining_time > 0:
+            if not self._sol:
+                try:
+                    self._sol = self._open_console()
+                except pexpect.TIMEOUT as e:
+                    logging.debug(e)
+                    raise BMCException('Could not open console: {}'.format(str(e)))
+
+            try:
+                self._sol.expect(flags, timeout=remaining_time)
+                logging.debug('Flag found in log')
+                return
+            except pexpect.TIMEOUT as e:
+                logging.debug(e)
+                raise BMCException('Expected message in console did not occur in time ({})'.format(flags))
+            except pexpect.EOF as e:
+                logging.warning('Got EOF from console')
+                if 'SOL session closed by BMC' in self._sol.before:
+                    logging.debug('Found: "SOL session closed by BMC" in console')
+                elapsed_time = time.time()-time_begin
+                remaining_time = timeout-elapsed_time
+                if remaining_time > 0:
+                    logging.info('Retry to expect a flag in console, %s seconds remaining', remaining_time)
+                    self.close()
+
+    def _wait_for_bios_settings_done(self):
+        logging.debug('Wait until BIOS settings are updated')
+
+        self._expect_flag_in_console('Booting...', timeout=300)
+
+    def _set_boot_from_hd_no_boot(self):
+        logging.debug('Set boot from hd (%s), no boot', self._host)
+
+        self._run_ipmitool_command('chassis bootdev disk options=persistent')
+
+    def _wait_for_virtual_media_detach_phase(self):
+        logging.debug('Wait until virtual media can be detached')
+
+        self._expect_flag_in_console(['Copying cloud guest image',
+                                      'Installing OS to HDD',
+                                      'Extending partition and filesystem size'],
+                                     timeout=1200)
+
+    def _wait_for_bmc_nfs_service(self, timeout, expected_status):
+        logging.debug('Wait for BMC NFS service')
+
+        start_time = int(time.time()*1000)
+
+        status = ''
+        while status != expected_status:
+            status = self._get_bmc_nfs_service_status()
+
+            if status == expected_status or status == 'nfserror':
+                logging.debug('Breaking from wait loop. status = %s', status)
+                break
+
+            time_now = int(time.time()*1000)
+            if time_now-start_time > timeout*1000:
+                logging.debug('Wait timed out')
+                break
+            time.sleep(10)
+
+        return status == expected_status
+
+    def _trigger_boot(self):
+        logging.debug('Trigger boot')
+
+        power_state = self.power('status')
+        logging.debug('State is: %s', power_state)
+        if power_state == 'Chassis Power is off':
+            self.power('on')
+        else:
+            self.power('reset')