Handle missing infra_external
[ta/remote-installer.git] / src / remoteinstaller / installer / install.py
1 # Copyright 2019 Nokia
2
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14
15 import logging
16 import sys
17 import argparse
18 import subprocess
19 import os
20 import importlib
21 import time
22 import distutils.util
23
24 from yaml import load
25 from netaddr import IPNetwork
26 from netaddr import IPAddress
27
28 import hw_detector.hw_detect_lib as hw_detect
29 from hw_detector.hw_exception import HWException
30 from remoteinstaller.installer.bmc_management.bmctools import BMCException
31 from remoteinstaller.installer.catfile import CatFile
32 from remoteinstaller.installer.catfile import CatFileException
33
34 class InstallException(Exception):
35     pass
36
37 class Installer(object):
38     SSH_OPTS = ('-o StrictHostKeyChecking=no '
39                 '-o UserKnownHostsFile=/dev/null '
40                 '-o ServerAliveInterval=60')
41
42     def __init__(self, callback_server, callback_uuid, yaml, logdir, args=None):
43         self._callback_server = callback_server
44         self._callback_uuid = callback_uuid
45         self._yaml_path = yaml
46         self._uc = self._read_user_config(self._yaml_path)
47         self._logdir = logdir
48
49         self._boot_iso_path = None
50         self._iso_url = None
51         self._callback_url = None
52         self._client_key = None
53         self._client_cert = None
54         self._ca_cert = None
55         self._own_ip = None
56         self._tag = None
57         self._disable_bmc_initial_reset = False
58         self._disable_other_bmc_reset = True
59
60         if args:
61             self._set_arguments(args)
62
63         self._vip = None
64         self._first_controller = None
65         self._first_controller_ip = None
66         self._first_controller_bmc = None
67
68         self._define_first_controller()
69
70     def _get_bool_arg(self, args, arg, default):
71         if hasattr(args, arg):
72             arg_value = vars(args)[arg]
73             if not isinstance(arg_value, bool):
74                 if isinstance(arg_value, basestring):
75                     try:
76                         arg_value = bool(distutils.util.strtobool(arg_value))
77                         return arg_value
78                     except ValueError:
79                         logging.warning('Invalid value for %s: %s', arg, arg_value)
80             else:
81                 return arg_value
82
83         return default
84
85     def _set_arguments(self, args):
86         self._disable_bmc_initial_reset = self._get_bool_arg(args, 'disable_bmc_initial_reset', self._disable_bmc_initial_reset)
87         self._disable_other_bmc_reset = self._get_bool_arg(args, 'disable_other_bmc_reset', self._disable_other_bmc_reset)
88
89         self._boot_iso_path = args.boot_iso
90         self._iso_url = args.iso
91         self._callback_url = args.callback_url
92         self._client_key = args.client_key
93         self._client_cert = args.client_cert
94         self._ca_cert = args.ca_cert
95         self._own_ip = args.host_ip
96         self._tag = args.tag
97
98     @staticmethod
99     def _read_user_config(config_file_path):
100         logging.debug('Read user config from %s', config_file_path)
101
102         try:
103             with open(config_file_path, 'r') as f:
104                 y = load(f)
105
106             return y
107         except Exception as ex:
108             raise InstallException(str(ex))
109
110     @staticmethod
111     def _execute_shell(command, desc=''):
112         logging.debug('Execute %s with command: %s', desc, command)
113
114         p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
115         out, _ = p.communicate()
116         if p.returncode:
117             logging.warning('Failed to %s: %s (rc=%s)', desc, out, p.returncode)
118             raise InstallException('Failed to {}'.format(desc))
119
120         return (p.returncode, out)
121
122     def _attach_iso_as_virtual_media(self, file_list):
123         logging.info('Attach ISO as virtual media')
124
125         nfs_mount = os.path.dirname(self._boot_iso_path)
126         boot_iso_filename = os.path.basename(self._boot_iso_path)
127         patched_iso_filename = '{}/{}_{}'.format(nfs_mount, self._tag, boot_iso_filename)
128
129         self._patch_iso(patched_iso_filename, file_list)
130
131         self._first_controller_bmc.attach_virtual_cd(self._own_ip, nfs_mount, os.path.basename(patched_iso_filename))
132
133     def _setup_bmc_for_node(self, hw):
134         bmc_log_path = '{}/{}.log'.format(self._logdir, hw)
135
136         host = self._uc['hosts'][hw]['hwmgmt']['address']
137         user = self._uc['hosts'][hw]['hwmgmt']['user']
138         passwd = self._uc['hosts'][hw]['hwmgmt']['password']
139         priv_level = self._uc['hosts'][hw]['hwmgmt'].get('priv_level', 'ADMINISTRATOR')
140
141         try:
142             hw_data = hw_detect.get_hw_data(host, user, passwd, priv_level, False)
143         except HWException as e:
144             error = "Hardware not detected for {}: {}".format(hw, str(e))
145             logging.error(error)
146             raise BMCException(error)
147
148         logging.debug("Hardware belongs to %s product family", (hw_data['product_family']))
149         if 'Unknown' in hw_data['product_family']:
150             error = "Hardware not detected for %s" % hw
151             logging.error(error)
152             raise BMCException(error)
153
154         bmc_mod_name = 'remoteinstaller.installer.bmc_management.{}'.format(hw_data['product_family'].lower())
155         bmc_mod = importlib.import_module(bmc_mod_name)
156         bmc_class = getattr(bmc_mod, hw_data['product_family'])
157         bmc = bmc_class(host, user, passwd, priv_level, bmc_log_path)
158         bmc.set_host_name(hw)
159
160         return bmc
161
162     def _define_first_controller(self):
163         for hw in sorted(self._uc['hosts']):
164             logging.debug('HW node name is %s', hw)
165
166             if 'controller' in self._uc['hosts'][hw]['service_profiles'] or \
167             'caas_master' in self._uc['hosts'][hw]['service_profiles']:
168                 self._first_controller = hw
169                 break
170
171         logging.info('First controller is %s', self._first_controller)
172         self._first_controller_bmc = self._setup_bmc_for_node(self._first_controller)
173
174         domain = self._uc['hosts'][self._first_controller].get('network_domain')
175         extnet = self._uc['networking']['infra_external']['network_domains'][domain]
176
177         first_ip = extnet['ip_range_start']
178         self._vip = str(IPAddress(first_ip))
179
180         pre_allocated_ips = self._uc['hosts'][self._first_controller].get('pre_allocated_ips', None)
181         if pre_allocated_ips:
182             pre_allocated_infra_external_ip = pre_allocated_ips.get('infra_external', None)
183             if pre_allocated_infra_external_ip:
184                 self._first_controller_ip = str(IPAddress(pre_allocated_infra_external_ip))
185
186         if not self._first_controller_ip:
187             self._first_controller_ip = str(IPAddress(first_ip)+1)
188
189     def _create_cloud_config(self):
190         logging.info('Create network config file')
191
192         domain = self._uc['hosts'][self._first_controller].get('network_domain')
193         extnet = self._uc['networking']['infra_external']['network_domains'][domain]
194
195         vlan = extnet.get('vlan')
196         gateway = extnet['gateway']
197         dns = self._uc['networking']['dns'][0]
198         cidr = extnet['cidr']
199         prefix = IPNetwork(cidr).prefixlen
200
201         controller_network_profile = self._uc['hosts'][self._first_controller]['network_profiles'][0]
202         mappings = self._uc['network_profiles'][controller_network_profile]['interface_net_mapping']
203         for interface in mappings:
204             if 'infra_external' in mappings[interface]:
205                 infra_external_interface = interface
206                 break
207
208         if infra_external_interface.startswith('bond'):
209             bonds = self._uc['network_profiles'][controller_network_profile]['bonding_interfaces']
210             device = bonds[infra_external_interface][0]
211         else:
212             device = infra_external_interface
213
214         # TODO
215         # ROOTFS_DISK
216
217         logging.debug('VLAN=%s', vlan)
218         logging.debug('DEV=%s', device)
219         logging.debug('IP=%s/%s', self._first_controller_ip, prefix)
220         logging.debug('DGW=%s', gateway)
221         logging.debug('NAMESERVER=%s', dns)
222         logging.debug('ISO_URL="%s"', self._iso_url)
223
224         network_config_filename = '{}/network_config'.format(self._logdir)
225         with open(network_config_filename, 'w') as f:
226             if vlan:
227                 f.write('VLAN={}\n'.format(vlan))
228             f.write('DEV={}\n'.format(device))
229             f.write('IP={}/{}\n'.format(self._first_controller_ip, prefix))
230             f.write('DGW={}\n'.format(gateway))
231             f.write('NAMESERVER={}\n'.format(dns))
232             f.write('\n')
233             f.write('ISO_URL="{}"'.format(self._iso_url))
234
235         return network_config_filename
236
237     def _create_callback_file(self):
238         logging.debug('CALLBACK_URL="%s"', self._callback_url)
239
240         callback_url_filename = '{}/callback_url'.format(self._logdir)
241         with open(callback_url_filename, 'w') as f:
242             f.write(self._callback_url)
243
244         return callback_url_filename
245
246     def _patch_iso(self, iso_target, file_list):
247         logging.info('Patch boot ISO')
248         logging.debug('Original ISO: %s', self._boot_iso_path)
249         logging.debug('Target ISO: %s', iso_target)
250
251         file_list_str = ' '.join(file_list)
252         logging.debug('Files to add: %s', file_list_str)
253
254         self._execute_shell('/usr/bin/patchiso.sh {} {} {}'.format(self._boot_iso_path,
255                                                                    iso_target,
256                                                                    file_list_str), 'patch ISO')
257
258     def _put_file(self, ip, user, passwd, file_name, to_file=''):
259         self._execute_shell('sshpass -p {} scp {} {} {}@{}:{}'.format(passwd,
260                                                                       Installer.SSH_OPTS,
261                                                                       file_name,
262                                                                       user,
263                                                                       ip,
264                                                                       to_file), 'put file')
265
266     def _get_file(self, ip, user, passwd, file_name, recursive=False):
267         if recursive:
268             self._execute_shell('sshpass -p {} scp {} -r {}@{}:{} {}'.format(passwd,
269                                                                              Installer.SSH_OPTS,
270                                                                              user,
271                                                                              ip,
272                                                                              file_name,
273                                                                              self._logdir), 'get files')
274         else:
275             self._execute_shell('sshpass -p {} scp {} {}@{}:{} {}'.format(passwd,
276                                                                           Installer.SSH_OPTS,
277                                                                           user,
278                                                                           ip,
279                                                                           file_name,
280                                                                           self._logdir), 'get file')
281
282     def _run_node_command(self, ip, user, passwd, command):
283         self._execute_shell('sshpass -p {} ssh {} {}@{} {}'.format(passwd,
284                                                                    Installer.SSH_OPTS,
285                                                                    user,
286                                                                    ip,
287                                                                    command), 'run command: {}'.format(command))
288
289     def _get_node_logs(self, ip, user, passwd):
290         self._get_file(ip, user, passwd, '/srv/deployment/log/cm.log')
291         self._get_file(ip, user, passwd, '/srv/deployment/log/bootstrap.log')
292         self._get_file(ip, user, passwd, '/var/log/ironic', recursive=True)
293
294     def _create_hosts_file(self, file_name):
295         with open(file_name, 'w') as hosts_file:
296             for host in self._uc['hosts'].keys():
297                 hosts_file.write('{}\n'.format(host))
298
299     def _get_journal_logs(self, ip, user, passwd):
300         hosts_file_name = 'host_names'
301         hosts_file_path = '{}/{}'.format(self._logdir, hosts_file_name)
302         self._create_hosts_file(hosts_file_name)
303
304         host_list = ' '.join(self._uc['hosts'].keys())
305
306         self._put_file(ip, user, passwd, hosts_file_name)
307         self._put_file(ip, user, passwd, '/opt/scripts/get_journals.sh')
308
309         self._run_node_command(ip, user, passwd, 'sh ./get_journals.sh {}'.format(hosts_file_name))
310
311         self._get_file(ip, user, passwd, '/tmp/node_journals.tgz')
312
313     def _get_logs_from_console(self, bmc, admin_user, admin_passwd):
314         bmc_host = bmc.get_host()
315         bmc_user = bmc.get_user()
316         bmc_passwd = bmc.get_passwd()
317         bmc_priv_level = bmc.get_priv_level()
318         
319         log_file = '{}/cat_bootstrap.log'.format(self._logdir)
320         try:
321             cat_file = CatFile(bmc_host, bmc_user, bmc_passwd, bmc_priv_level, admin_user, admin_passwd)
322             cat_file.cat('/srv/deployment/log/bootstrap.log', log_file)
323         except CatFileException as ex:
324             logging.info('Could not cat file from console: %s', str(ex))
325
326             cat_file = CatFile(bmc_host, bmc_user, bmc_passwd, bmc_priv_level, 'root', 'root')
327             cat_file.cat('/srv/deployment/log/bootstrap.log', log_file)
328
329     def get_logs(self, admin_passwd):
330         admin_user = self._uc['users']['admin_user_name']
331
332         ssh_check_command = 'nc -w1 {} 22 </dev/null &> /dev/null'.format(self._first_controller_ip)
333         ssh_check_fails = os.system(ssh_check_command)
334
335         if not ssh_check_fails:
336             self._get_node_logs(self._first_controller_ip, admin_user, admin_passwd)
337
338             self._get_journal_logs(self._first_controller_ip, admin_user, admin_passwd)
339         else:
340             self._get_logs_from_console(self._first_controller_bmc,
341                                         admin_user,
342                                         admin_passwd)
343
344     def _setup_bmcs(self):
345         other_bmcs = []
346         for hw in sorted(self._uc['hosts']):
347             logging.info('HW node name is %s', hw)
348
349             bmc = self._setup_bmc_for_node(hw)
350             bmc.setup_sol()
351
352             if hw != self._first_controller:
353                 other_bmcs.append(bmc)
354                 bmc.power('off')
355
356         if not self._disable_bmc_initial_reset:
357             self._first_controller_bmc.reset()
358             time_after_reset = int(time.time())
359
360         if not self._disable_other_bmc_reset:
361             for bmc in other_bmcs:
362                 bmc.reset()
363
364         if not self._disable_bmc_initial_reset:
365             # Make sure we sleep at least 6min after the first controller BMC reset
366             sleep_time = 6*60-int(time.time())-time_after_reset
367             if sleep_time > 0:
368                 logging.debug('Waiting for first controller BMC to stabilize \
369                                (%s sec) after reset', sleep_time)
370                 time.sleep(sleep_time)
371
372     def get_access_info(self):
373         access_info = {'vip': self._vip,
374                        'installer_node_ip': self._first_controller_ip,
375                        'admin_user': self._uc['users']['admin_user_name']}
376
377         return access_info
378
379     def _set_progress(self, description, failed=False):
380         if failed:
381             state = 'failed'
382         else:
383             state = 'ongoing'
384
385         self._callback_server.set_state(self._callback_uuid, state, description)
386
387     def install(self):
388         try:
389             logging.info('Start install')
390
391             if os.path.dirname(self._boot_iso_path) == '':
392                 self._boot_iso_path = '{}/{}'.format(os.getcwd(), self._boot_iso_path)
393
394             if self._logdir:
395                 if not os.path.exists(self._logdir):
396                     os.makedirs(self._logdir)
397             else:
398                 self._logdir = '.'
399
400             self._set_progress('Setup BMCs')
401             self._setup_bmcs()
402
403             self._set_progress('Create config files')
404             network_config_filename = self._create_cloud_config()
405             callback_url_filename = self._create_callback_file()
406
407             patch_files = [self._yaml_path,
408                            network_config_filename,
409                            callback_url_filename]
410
411             if self._client_cert:
412                 patch_files.append(self._client_cert)
413             if self._client_key:
414                 patch_files.append(self._client_key)
415             if self._ca_cert:
416                 patch_files.append(self._ca_cert)
417
418             self._set_progress('Setup boot options for virtual media')
419             self._first_controller_bmc.setup_boot_options_for_virtual_media()
420
421             self._set_progress('Attach iso as virtual media')
422             self._attach_iso_as_virtual_media(patch_files)
423
424             self._set_progress('Boot from virtual media')
425             self._first_controller_bmc.boot_from_virtual_media()
426
427             self._set_progress('Wait for bootup')
428             self._first_controller_bmc.wait_for_bootup()
429
430             self._set_progress('Wait deployment start')
431
432             self._first_controller_bmc.close()
433         except BMCException as ex:
434             logging.error('Installation failed: %s', str(ex))
435             raise InstallException(str(ex))
436
437 def main():
438     parser = argparse.ArgumentParser()
439     parser.add_argument('-y', '--yaml', required=True,
440                         help='User config yaml file path')
441     parser.add_argument('-b', '--boot-iso', required=True,
442                         help='Path to boot ISO image in NFS mount')
443     parser.add_argument('-i', '--iso', required=True,
444                         help='URL to ISO image')
445     parser.add_argument('-d', '--debug', action='store_true', required=False,
446                         help='Debug level for logging')
447     parser.add_argument('-l', '--logdir', required=True,
448                         help='Directory path for log files')
449     parser.add_argument('-c', '--callback-url', required=True,
450                         help='Callback URL for progress reporting')
451     parser.add_argument('-K', '--client-key', required=True,
452                         help='Client key file path')
453     parser.add_argument('-C', '--client-cert', required=True,
454                         help='Client cert file path')
455     parser.add_argument('-A', '--ca-cert', required=True,
456                         help='CA cert file path')
457     parser.add_argument('-H', '--host-ip', required=True,
458                         help='IP for hosting HTTPD and NFS')
459     parser.add_argument('-T', '--http-port', required=False,
460                         help='Port for HTTPD')
461
462     parsed_args = parser.parse_args()
463
464     if parsed_args.debug:
465         log_level = logging.DEBUG
466     else:
467         log_level = logging.INFO
468
469     logging.basicConfig(stream=sys.stdout, level=log_level)
470
471     logging.debug('args: %s', parsed_args)
472     installer = Installer(parsed_args.yaml, parsed_args.logdir, parsed_args)
473
474     installer.install()
475
476 if __name__ == "__main__":
477     sys.exit(main())