b5be98d3b9298fb6c3ae72504f9f0006848bebcd
[ta/remote-installer.git] / src / remoteinstaller / installer / bmc_management / bmctools.py
1 # Copyright 2019 Nokia
2 # Copyright 2020 ENEA
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 #     http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15
16 import subprocess
17 import time
18 import logging
19 import ipaddress
20 import pexpect
21
22 class BMCException(Exception):
23     pass
24
25 class BMC(object):
26     def __init__(self, host, user, passwd, priv_level='ADMINISTRATOR', log_path=None):
27         self._host = host
28         self._user = user
29         self._passwd = passwd
30         self._priv_level = priv_level
31         if log_path:
32             self._log_path = log_path
33         else:
34             self._log_path = 'console.log'
35         self._sol = None
36         self._host_name = None
37
38     def set_host_name(self, host_name):
39         if not self._host_name:
40             self._host_name = host_name
41
42     def get_host_name(self):
43         if self._host_name:
44             return self._host_name
45
46         return '<NONAME>'
47
48     def get_host(self):
49         return self._host
50
51     def get_user(self):
52         return self._user
53
54     def get_passwd(self):
55         return self._passwd
56
57     def get_priv_level(self):
58         return self._priv_level
59     
60     def reset(self):
61         logging.info('Reset BMC of %s: %s', self.get_host_name(), self.get_host())
62
63         self._run_ipmitool_command('bmc reset cold')
64
65         success = self._wait_for_bmc_reset(180)
66         if not success:
67             raise BMCException('BMC reset failed, BMC did not come up')
68
69     def _set_boot_from_virtual_media(self):
70         raise NotImplementedError
71
72     def _detach_virtual_media(self):
73         raise NotImplementedError
74
75     def _get_bmc_nfs_service_status(self):
76         raise NotImplementedError
77
78     def _wait_for_bmc_responding(self, timeout, expected_to_respond=True):
79         if expected_to_respond:
80             logging.debug('Wait for BMC to start responding')
81         else:
82             logging.debug('Wait for BMC to stop responding')
83
84         start_time = int(time.time()*1000)
85
86         response = (not expected_to_respond)
87         while response != expected_to_respond:
88             rc, _ = self._run_ipmitool_command('bmc info', can_fail=True)
89             response = (rc == 0)
90
91             if response == expected_to_respond:
92                 break
93
94             time_now = int(time.time()*1000)
95             if time_now-start_time > timeout*1000:
96                 logging.debug('Wait timed out')
97                 break
98
99             logging.debug('Still waiting for BMC')
100             time.sleep(10)
101
102         return response == expected_to_respond
103
104     def _wait_for_bmc_webpage(self, timeout):
105         host = ipaddress.ip_address(unicode(self._host))
106         if host.version == 6:
107             host = "[%s]" %host
108
109         command = 'curl -g --insecure -o /dev/null https://{}/index.html'.format(host)
110
111         start_time = int(time.time()*1000)
112         rc = 1
113         while rc != 0:
114             p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
115             _, _ = p.communicate()
116             rc = p.returncode
117
118             if rc == 0:
119                 break
120
121             time_now = int(time.time()*1000)
122             if time_now-start_time > timeout*1000:
123                 logging.debug('Wait timed out')
124                 break
125
126             logging.debug('Still waiting for BMC webpage')
127             time.sleep(10)
128
129         return rc == 0
130
131     def _wait_for_bmc_not_responding(self, timeout):
132         return self._wait_for_bmc_responding(timeout, False)
133
134     def _wait_for_bmc_reset(self, timeout):
135         logging.debug('Wait for BMC to reset')
136
137         success = True
138         if not self._wait_for_bmc_not_responding(timeout):
139             success = False
140             msg = 'BMC did not go down as expected'
141             logging.warning(msg)
142         else:
143             logging.debug('As expected, BMC is not responding')
144
145         if not self._wait_for_bmc_responding(timeout):
146             success = False
147             msg = 'BMC did not come up as expected'
148             logging.warning(msg)
149         else:
150             logging.debug('As expected, BMC is responding')
151
152         if not self._wait_for_bmc_webpage(timeout):
153             success = False
154             msg = 'BMC webpage did not start to respond'
155             logging.warning(msg)
156         else:
157             logging.debug('As expected, BMC webpage is responding')
158
159         return success
160
161     def setup_boot_options_for_virtual_media(self):
162         logging.debug('Setup boot options')
163
164         self._disable_boot_flag_timeout()
165         self._set_boot_from_virtual_media()
166
167     def power(self, power_command):
168         logging.debug('Run host power command (%s) %s', self._host, power_command)
169
170         return self._run_ipmitool_command('power {}'.format(power_command)).strip()
171
172     def wait_for_bootup(self):
173         logging.debug('Wait for prompt after booting from hd')
174
175         try:
176             self._expect_flag_in_console('localhost login:', timeout=1200)
177         except BMCException as ex:
178             self._send_to_console('\n')
179             self._expect_flag_in_console('localhost login:', timeout=30)
180
181     def setup_sol(self):
182         logging.debug('Setup SOL for %s', self._host)
183
184         self._run_ipmitool_command('sol set non-volatile-bit-rate 115.2')
185         self._run_ipmitool_command('sol set volatile-bit-rate 115.2')
186
187     def boot_from_virtual_media(self):
188         logging.info('Boot from virtual media')
189
190         self._trigger_boot()
191         self._wait_for_virtual_media_detach_phase()
192
193         self._detach_virtual_media()
194         self._set_boot_from_hd_no_boot()
195
196         logging.info('Boot should continue from disk now')
197
198     def close(self):
199         if self._sol:
200             self._sol.terminate()
201         self._sol = None
202
203     @staticmethod
204     def _convert_to_hex(ascii_string, padding=False, length=0):
205         if padding:
206             ascii_string = ascii_string.ljust(length, '\0')
207         return ' '.join('0x{}'.format(c.encode('hex')) for c in ascii_string)
208
209     @staticmethod
210     def _convert_to_ascii(hex_string):
211         return hex_string.replace('0x','').replace(' ','').decode('hex')
212
213     def _execute_ipmitool_command(self, ipmi_command):
214         command = 'ipmitool -I lanplus -H {} -U {} -P {} -L {} {}'.format(self._host, self._user, self._passwd, self._priv_level, ipmi_command)
215
216         p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
217         out, _ = p.communicate()
218         rc = p.returncode
219
220         return (rc, out)
221
222     def _run_ipmitool_command(self, ipmi_command, can_fail=False, retries=5):
223         logging.debug('Run ipmitool command: %s', ipmi_command)
224
225         if can_fail:
226             return self._execute_ipmitool_command(ipmi_command)
227         else:
228             while retries > 0:
229                 rc, out = self._execute_ipmitool_command(ipmi_command)
230                 if not rc:
231                     break
232
233                 if retries > 0:
234                     logging.debug('Retry command')
235                     time.sleep(5)
236
237                 retries -= 1
238
239             if rc:
240                 logging.warning('ipmitool failed: %s', out)
241                 raise BMCException('ipmitool call failed with rc: {}'.format(rc))
242
243         return out
244
245     def _run_ipmitool_raw_command(self, ipmi_raw_command):
246         logging.debug('Run ipmitool raw command')
247
248         out = self._run_ipmitool_command('raw {}'.format(ipmi_raw_command))
249
250         out_bytes = out.replace('\n', '').strip().split(' ')
251         return out_bytes
252
253     def _disable_boot_flag_timeout(self):
254         logging.debug('Disable boot flag timeout (%s)', self._host)
255
256         status_code = self._run_ipmitool_raw_command('0x00 0x08 0x03 0x1f')
257         if status_code[0] != '':
258             raise BMCException('Could not disable boot flag timeout (rc={})'.format(status_code[0]))
259
260     def _open_console(self):
261         logging.debug('Open SOL console (log in %s)', self._log_path)
262
263         expect_session = pexpect.spawn('ipmitool -I lanplus -H {} -U {} -P {} -L {} sol deactivate'.format(self._host, self._user, self._passwd, self._priv_level))
264         expect_session.expect(pexpect.EOF)
265
266         logfile = open(self._log_path, 'ab')
267
268         expect_session = pexpect.spawn('ipmitool -I lanplus -H {} -U {} -P {} -L {} sol activate'.format(self._host, self._user, self._passwd, self._priv_level), timeout=None, logfile=logfile)
269
270         return expect_session
271
272     def _send_to_console(self, chars):
273         logging.debug('Sending %s to console', chars.replace('\n', '\\n'))
274
275         if not self._sol:
276             self._sol = self._open_console()
277
278         self._sol.send(chars)
279
280     def _expect_flag_in_console(self, flags, timeout=600):
281         logging.debug('Expect a flag in console output within %s seconds ("%s")', timeout, flags)
282
283         time_begin = time.time()
284
285         remaining_time = timeout
286
287         while remaining_time > 0:
288             if not self._sol:
289                 try:
290                     self._sol = self._open_console()
291                 except pexpect.TIMEOUT as e:
292                     logging.debug(e)
293                     raise BMCException('Could not open console: {}'.format(str(e)))
294
295             try:
296                 self._sol.expect(flags, timeout=remaining_time)
297                 logging.debug('Flag found in log')
298                 return
299             except pexpect.TIMEOUT as e:
300                 logging.debug(e)
301                 raise BMCException('Expected message in console did not occur in time ({})'.format(flags))
302             except pexpect.EOF as e:
303                 logging.warning('Got EOF from console')
304                 if 'SOL session closed by BMC' in self._sol.before:
305                     logging.debug('Found: "SOL session closed by BMC" in console')
306                 elapsed_time = time.time()-time_begin
307                 remaining_time = timeout-elapsed_time
308                 if remaining_time > 0:
309                     logging.info('Retry to expect a flag in console, %s seconds remaining', remaining_time)
310                     self.close()
311
312         raise BMCException('Expected message in console did not occur in time ({})'.format(flags))
313
314     def _wait_for_bios_settings_done(self):
315         logging.debug('Wait until BIOS settings are updated')
316
317         self._expect_flag_in_console('Booting...', timeout=300)
318
319     def _set_boot_from_hd_no_boot(self):
320         logging.debug('Set boot from hd (%s), no boot', self._host)
321
322         self._run_ipmitool_command('chassis bootdev disk options=persistent')
323
324     def _wait_for_virtual_media_detach_phase(self):
325         logging.debug('Wait until virtual media can be detached')
326
327         self._expect_flag_in_console(['Copying cloud guest image',
328                                       'Installing OS to HDD',
329                                       'Extending partition and filesystem size'],
330                                      timeout=1200)
331
332     def _wait_for_bmc_nfs_service(self, timeout, expected_status):
333         logging.debug('Wait for BMC NFS service')
334
335         start_time = int(time.time()*1000)
336
337         status = ''
338         while status != expected_status:
339             status = self._get_bmc_nfs_service_status()
340
341             if status == expected_status or status == 'nfserror':
342                 logging.debug('Breaking from wait loop. status = %s', status)
343                 break
344
345             time_now = int(time.time()*1000)
346             if time_now-start_time > timeout*1000:
347                 logging.debug('Wait timed out')
348                 break
349             time.sleep(10)
350
351         return status == expected_status
352
353     def _trigger_boot(self):
354         logging.debug('Trigger boot')
355
356         power_state = self.power('status')
357         logging.debug('State is: %s', power_state)
358         if power_state == 'Chassis Power is off':
359             self.power('on')
360         else:
361             self.power('reset')