FIX: Pass IPMI priv_level to hardware detector
[ta/remote-installer.git] / src / remoteinstaller / installer / install.py
1 # Copyright 2019 Nokia
2
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14
15 import logging
16 import sys
17 import argparse
18 import subprocess
19 import os
20 import importlib
21 import time
22 import distutils.util
23
24 from yaml import load
25 from netaddr import IPNetwork
26 from netaddr import IPAddress
27
28 import hw_detector.hw_detect_lib as hw_detect
29 from hw_detector.hw_exception import HWException
30 from remoteinstaller.installer.bmc_management.bmctools import BMCException
31 from remoteinstaller.installer.catfile import CatFile
32 from remoteinstaller.installer.catfile import CatFileException
33
34 class InstallException(Exception):
35     pass
36
37 class Installer(object):
38     SSH_OPTS = ('-o StrictHostKeyChecking=no '
39                 '-o UserKnownHostsFile=/dev/null '
40                 '-o ServerAliveInterval=60')
41
42     def __init__(self, callback_server, callback_uuid, yaml, logdir, args=None):
43         self._callback_server = callback_server
44         self._callback_uuid = callback_uuid
45         self._yaml_path = yaml
46         self._uc = self._read_user_config(self._yaml_path)
47         self._logdir = logdir
48
49         self._boot_iso_path = None
50         self._iso_url = None
51         self._callback_url = None
52         self._client_key = None
53         self._client_cert = None
54         self._ca_cert = None
55         self._own_ip = None
56         self._tag = None
57         self._disable_bmc_initial_reset = False
58         self._disable_other_bmc_reset = True
59
60         if args:
61             self._set_arguments(args)
62
63         self._vip = None
64         self._first_controller = None
65         self._first_controller_ip = None
66         self._first_controller_bmc = None
67
68         self._define_first_controller()
69
70     def _get_bool_arg(self, args, arg, default):
71         if hasattr(args, arg):
72             arg_value = vars(args)[arg]
73             if not isinstance(arg_value, bool):
74                 if isinstance(arg_value, basestring):
75                     try:
76                         arg_value = bool(distutils.util.strtobool(arg_value))
77                         return arg_value
78                     except ValueError:
79                         logging.warning('Invalid value for %s: %s', arg, arg_value)
80             else:
81                 return arg_value
82
83         return default
84
85     def _set_arguments(self, args):
86         self._disable_bmc_initial_reset = self._get_bool_arg(args, 'disable_bmc_initial_reset', self._disable_bmc_initial_reset)
87         self._disable_other_bmc_reset = self._get_bool_arg(args, 'disable_other_bmc_reset', self._disable_other_bmc_reset)
88
89         self._boot_iso_path = args.boot_iso
90         self._iso_url = args.iso
91         self._callback_url = args.callback_url
92         self._client_key = args.client_key
93         self._client_cert = args.client_cert
94         self._ca_cert = args.ca_cert
95         self._own_ip = args.host_ip
96         self._tag = args.tag
97
98     @staticmethod
99     def _read_user_config(config_file_path):
100         logging.debug('Read user config from %s', config_file_path)
101
102         try:
103             with open(config_file_path, 'r') as f:
104                 y = load(f)
105
106             return y
107         except Exception as ex:
108             raise InstallException(str(ex))
109
110     @staticmethod
111     def _execute_shell(command, desc=''):
112         logging.debug('Execute %s with command: %s', desc, command)
113
114         p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
115         out, _ = p.communicate()
116         if p.returncode:
117             logging.warning('Failed to %s: %s (rc=%s)', desc, out, p.returncode)
118             raise InstallException('Failed to {}'.format(desc))
119
120         return (p.returncode, out)
121
122     def _attach_iso_as_virtual_media(self, file_list):
123         logging.info('Attach ISO as virtual media')
124
125         nfs_mount = os.path.dirname(self._boot_iso_path)
126         boot_iso_filename = os.path.basename(self._boot_iso_path)
127         patched_iso_filename = '{}/{}_{}'.format(nfs_mount, self._tag, boot_iso_filename)
128
129         self._patch_iso(patched_iso_filename, file_list)
130
131         self._first_controller_bmc.attach_virtual_cd(self._own_ip, nfs_mount, os.path.basename(patched_iso_filename))
132
133     def _setup_bmc_for_node(self, hw):
134         bmc_log_path = '{}/{}.log'.format(self._logdir, hw)
135
136         host = self._uc['hosts'][hw]['hwmgmt']['address']
137         user = self._uc['hosts'][hw]['hwmgmt']['user']
138         passwd = self._uc['hosts'][hw]['hwmgmt']['password']
139         priv_level = self._uc['hosts'][hw]['hwmgmt'].get('priv_level', 'ADMINISTRATOR')
140
141         try:
142             hw_data = hw_detect.get_hw_data(host, user, passwd, priv_level, False)
143         except HWException as e:
144             error = "Hardware not detected for {}: {}".format(hw, str(e))
145             logging.error(error)
146             raise BMCException(error)
147
148         logging.debug("Hardware belongs to %s product family", (hw_data['product_family']))
149         if 'Unknown' in hw_data['product_family']:
150             error = "Hardware not detected for %s" % hw
151             logging.error(error)
152             raise BMCException(error)
153
154         bmc_mod_name = 'remoteinstaller.installer.bmc_management.{}'.format(hw_data['product_family'].lower())
155         bmc_mod = importlib.import_module(bmc_mod_name)
156         bmc_class = getattr(bmc_mod, hw_data['product_family'])
157         bmc = bmc_class(host, user, passwd, bmc_log_path)
158         bmc.set_host_name(hw)
159
160         return bmc
161
162     def _define_first_controller(self):
163         for hw in sorted(self._uc['hosts']):
164             logging.debug('HW node name is %s', hw)
165
166             if 'controller' in self._uc['hosts'][hw]['service_profiles'] or \
167             'caas_master' in self._uc['hosts'][hw]['service_profiles']:
168                 self._first_controller = hw
169                 break
170
171         logging.info('First controller is %s', self._first_controller)
172         self._first_controller_bmc = self._setup_bmc_for_node(self._first_controller)
173
174         domain = self._uc['hosts'][self._first_controller].get('network_domain')
175         extnet = self._uc['networking']['infra_external']['network_domains'][domain]
176
177         first_ip = extnet['ip_range_start']
178         self._vip = str(IPAddress(first_ip))
179
180         pre_allocated_ips = self._uc['hosts'][self._first_controller].get('pre_allocated_ips', None)
181         if pre_allocated_ips:
182             pre_allocated_infra_external_ip = pre_allocated_ips.get('infra_external', None)
183             self._first_controller_ip = str(IPAddress(pre_allocated_infra_external_ip))
184
185         if not self._first_controller_ip:
186             self._first_controller_ip = str(IPAddress(first_ip)+1)
187
188     def _create_cloud_config(self):
189         logging.info('Create network config file')
190
191         domain = self._uc['hosts'][self._first_controller].get('network_domain')
192         extnet = self._uc['networking']['infra_external']['network_domains'][domain]
193
194         vlan = extnet.get('vlan')
195         gateway = extnet['gateway']
196         dns = self._uc['networking']['dns'][0]
197         cidr = extnet['cidr']
198         prefix = IPNetwork(cidr).prefixlen
199
200         controller_network_profile = self._uc['hosts'][self._first_controller]['network_profiles'][0]
201         mappings = self._uc['network_profiles'][controller_network_profile]['interface_net_mapping']
202         for interface in mappings:
203             if 'infra_external' in mappings[interface]:
204                 infra_external_interface = interface
205                 break
206
207         if infra_external_interface.startswith('bond'):
208             bonds = self._uc['network_profiles'][controller_network_profile]['bonding_interfaces']
209             device = bonds[infra_external_interface][0]
210         else:
211             device = infra_external_interface
212
213         # TODO
214         # ROOTFS_DISK
215
216         logging.debug('VLAN=%s', vlan)
217         logging.debug('DEV=%s', device)
218         logging.debug('IP=%s/%s', self._first_controller_ip, prefix)
219         logging.debug('DGW=%s', gateway)
220         logging.debug('NAMESERVER=%s', dns)
221         logging.debug('ISO_URL="%s"', self._iso_url)
222
223         network_config_filename = '{}/network_config'.format(self._logdir)
224         with open(network_config_filename, 'w') as f:
225             if vlan:
226                 f.write('VLAN={}\n'.format(vlan))
227             f.write('DEV={}\n'.format(device))
228             f.write('IP={}/{}\n'.format(self._first_controller_ip, prefix))
229             f.write('DGW={}\n'.format(gateway))
230             f.write('NAMESERVER={}\n'.format(dns))
231             f.write('\n')
232             f.write('ISO_URL="{}"'.format(self._iso_url))
233
234         return network_config_filename
235
236     def _create_callback_file(self):
237         logging.debug('CALLBACK_URL="%s"', self._callback_url)
238
239         callback_url_filename = '{}/callback_url'.format(self._logdir)
240         with open(callback_url_filename, 'w') as f:
241             f.write(self._callback_url)
242
243         return callback_url_filename
244
245     def _patch_iso(self, iso_target, file_list):
246         logging.info('Patch boot ISO')
247         logging.debug('Original ISO: %s', self._boot_iso_path)
248         logging.debug('Target ISO: %s', iso_target)
249
250         file_list_str = ' '.join(file_list)
251         logging.debug('Files to add: %s', file_list_str)
252
253         self._execute_shell('/usr/bin/patchiso.sh {} {} {}'.format(self._boot_iso_path,
254                                                                    iso_target,
255                                                                    file_list_str), 'patch ISO')
256
257     def _put_file(self, ip, user, passwd, file_name, to_file=''):
258         self._execute_shell('sshpass -p {} scp {} {} {}@{}:{}'.format(passwd,
259                                                                       Installer.SSH_OPTS,
260                                                                       file_name,
261                                                                       user,
262                                                                       ip,
263                                                                       to_file), 'put file')
264
265     def _get_file(self, ip, user, passwd, file_name, recursive=False):
266         if recursive:
267             self._execute_shell('sshpass -p {} scp {} -r {}@{}:{} {}'.format(passwd,
268                                                                              Installer.SSH_OPTS,
269                                                                              user,
270                                                                              ip,
271                                                                              file_name,
272                                                                              self._logdir), 'get files')
273         else:
274             self._execute_shell('sshpass -p {} scp {} {}@{}:{} {}'.format(passwd,
275                                                                           Installer.SSH_OPTS,
276                                                                           user,
277                                                                           ip,
278                                                                           file_name,
279                                                                           self._logdir), 'get file')
280
281     def _run_node_command(self, ip, user, passwd, command):
282         self._execute_shell('sshpass -p {} ssh {} {}@{} {}'.format(passwd,
283                                                                    Installer.SSH_OPTS,
284                                                                    user,
285                                                                    ip,
286                                                                    command), 'run command: {}'.format(command))
287
288     def _get_node_logs(self, ip, user, passwd):
289         self._get_file(ip, user, passwd, '/srv/deployment/log/cm.log')
290         self._get_file(ip, user, passwd, '/srv/deployment/log/bootstrap.log')
291         self._get_file(ip, user, passwd, '/var/log/ironic', recursive=True)
292
293     def _create_hosts_file(self, file_name):
294         with open(file_name, 'w') as hosts_file:
295             for host in self._uc['hosts'].keys():
296                 hosts_file.write('{}\n'.format(host))
297
298     def _get_journal_logs(self, ip, user, passwd):
299         hosts_file_name = 'host_names'
300         hosts_file_path = '{}/{}'.format(self._logdir, hosts_file_name)
301         self._create_hosts_file(hosts_file_name)
302
303         host_list = ' '.join(self._uc['hosts'].keys())
304
305         self._put_file(ip, user, passwd, hosts_file_name)
306         self._put_file(ip, user, passwd, '/opt/scripts/get_journals.sh')
307
308         self._run_node_command(ip, user, passwd, 'sh ./get_journals.sh {}'.format(hosts_file_name))
309
310         self._get_file(ip, user, passwd, '/tmp/node_journals.tgz')
311
312     def _get_logs_from_console(self, bmc, admin_user, admin_passwd):
313         bmc_host = bmc.get_host()
314         bmc_user = bmc.get_user()
315         bmc_passwd = bmc.get_passwd()
316
317         log_file = '{}/cat_bootstrap.log'.format(self._logdir)
318         try:
319             cat_file = CatFile(bmc_host, bmc_user, bmc_passwd, admin_user, admin_passwd)
320             cat_file.cat('/srv/deployment/log/bootstrap.log', log_file)
321         except CatFileException as ex:
322             logging.info('Could not cat file from console: %s', str(ex))
323
324             cat_file = CatFile(bmc_host, bmc_user, bmc_passwd, 'root', 'root')
325             cat_file.cat('/srv/deployment/log/bootstrap.log', log_file)
326
327     def get_logs(self, admin_passwd):
328         admin_user = self._uc['users']['admin_user_name']
329
330         ssh_check_command = 'nc -w1 {} 22 </dev/null &> /dev/null'.format(self._first_controller_ip)
331         ssh_check_fails = os.system(ssh_check_command)
332
333         if not ssh_check_fails:
334             self._get_node_logs(self._first_controller_ip, admin_user, admin_passwd)
335
336             self._get_journal_logs(self._first_controller_ip, admin_user, admin_passwd)
337         else:
338             self._get_logs_from_console(self._first_controller_bmc,
339                                         admin_user,
340                                         admin_passwd)
341
342     def _setup_bmcs(self):
343         other_bmcs = []
344         for hw in sorted(self._uc['hosts']):
345             logging.info('HW node name is %s', hw)
346
347             bmc = self._setup_bmc_for_node(hw)
348             bmc.setup_sol()
349
350             if hw != self._first_controller:
351                 other_bmcs.append(bmc)
352                 bmc.power('off')
353
354         if not self._disable_bmc_initial_reset:
355             self._first_controller_bmc.reset()
356             time_after_reset = int(time.time())
357
358         if not self._disable_other_bmc_reset:
359             for bmc in other_bmcs:
360                 bmc.reset()
361
362         if not self._disable_bmc_initial_reset:
363             # Make sure we sleep at least 6min after the first controller BMC reset
364             sleep_time = 6*60-int(time.time())-time_after_reset
365             if sleep_time > 0:
366                 logging.debug('Waiting for first controller BMC to stabilize \
367                                (%s sec) after reset', sleep_time)
368                 time.sleep(sleep_time)
369
370     def get_access_info(self):
371         access_info = {'vip': self._vip,
372                        'installer_node_ip': self._first_controller_ip,
373                        'admin_user': self._uc['users']['admin_user_name']}
374
375         return access_info
376
377     def _set_progress(self, description, failed=False):
378         if failed:
379             state = 'failed'
380         else:
381             state = 'ongoing'
382
383         self._callback_server.set_state(self._callback_uuid, state, description)
384
385     def install(self):
386         try:
387             logging.info('Start install')
388
389             if os.path.dirname(self._boot_iso_path) == '':
390                 self._boot_iso_path = '{}/{}'.format(os.getcwd(), self._boot_iso_path)
391
392             if self._logdir:
393                 if not os.path.exists(self._logdir):
394                     os.makedirs(self._logdir)
395             else:
396                 self._logdir = '.'
397
398             self._set_progress('Setup BMCs')
399             self._setup_bmcs()
400
401             self._set_progress('Create config files')
402             network_config_filename = self._create_cloud_config()
403             callback_url_filename = self._create_callback_file()
404
405             patch_files = [self._yaml_path,
406                            network_config_filename,
407                            callback_url_filename]
408
409             if self._client_cert:
410                 patch_files.append(self._client_cert)
411             if self._client_key:
412                 patch_files.append(self._client_key)
413             if self._ca_cert:
414                 patch_files.append(self._ca_cert)
415
416             self._set_progress('Setup boot options for virtual media')
417             self._first_controller_bmc.setup_boot_options_for_virtual_media()
418
419             self._set_progress('Attach iso as virtual media')
420             self._attach_iso_as_virtual_media(patch_files)
421
422             self._set_progress('Boot from virtual media')
423             self._first_controller_bmc.boot_from_virtual_media()
424
425             self._set_progress('Wait for bootup')
426             self._first_controller_bmc.wait_for_bootup()
427
428             self._set_progress('Wait deployment start')
429
430             self._first_controller_bmc.close()
431         except BMCException as ex:
432             logging.error('Installation failed: %s', str(ex))
433             raise InstallException(str(ex))
434
435 def main():
436     parser = argparse.ArgumentParser()
437     parser.add_argument('-y', '--yaml', required=True,
438                         help='User config yaml file path')
439     parser.add_argument('-b', '--boot-iso', required=True,
440                         help='Path to boot ISO image in NFS mount')
441     parser.add_argument('-i', '--iso', required=True,
442                         help='URL to ISO image')
443     parser.add_argument('-d', '--debug', action='store_true', required=False,
444                         help='Debug level for logging')
445     parser.add_argument('-l', '--logdir', required=True,
446                         help='Directory path for log files')
447     parser.add_argument('-c', '--callback-url', required=True,
448                         help='Callback URL for progress reporting')
449     parser.add_argument('-K', '--client-key', required=True,
450                         help='Client key file path')
451     parser.add_argument('-C', '--client-cert', required=True,
452                         help='Client cert file path')
453     parser.add_argument('-A', '--ca-cert', required=True,
454                         help='CA cert file path')
455     parser.add_argument('-H', '--host-ip', required=True,
456                         help='IP for hosting HTTPD and NFS')
457     parser.add_argument('-T', '--http-port', required=False,
458                         help='Port for HTTPD')
459
460     parsed_args = parser.parse_args()
461
462     if parsed_args.debug:
463         log_level = logging.DEBUG
464     else:
465         log_level = logging.INFO
466
467     logging.basicConfig(stream=sys.stdout, level=log_level)
468
469     logging.debug('args: %s', parsed_args)
470     installer = Installer(parsed_args.yaml, parsed_args.logdir, parsed_args)
471
472     installer.install()
473
474 if __name__ == "__main__":
475     sys.exit(main())