3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
25 from netaddr import IPNetwork
26 from netaddr import IPAddress
28 import hw_detector.hw_detect_lib as hw_detect
29 from hw_detector.hw_exception import HWException
30 from remoteinstaller.installer.bmc_management.bmctools import BMCException
31 from remoteinstaller.installer.catfile import CatFile
32 from remoteinstaller.installer.catfile import CatFileException
34 class InstallException(Exception):
37 class Installer(object):
38 SSH_OPTS = ('-o StrictHostKeyChecking=no '
39 '-o UserKnownHostsFile=/dev/null '
40 '-o ServerAliveInterval=60')
42 def __init__(self, callback_server, callback_uuid, yaml, logdir, args=None):
43 self._callback_server = callback_server
44 self._callback_uuid = callback_uuid
45 self._yaml_path = yaml
46 self._uc = self._read_user_config(self._yaml_path)
49 self._boot_iso_path = None
51 self._callback_url = None
52 self._client_key = None
53 self._client_cert = None
57 self._disable_bmc_initial_reset = False
58 self._disable_other_bmc_reset = True
61 self._set_arguments(args)
64 self._first_controller = None
65 self._first_controller_ip = None
66 self._first_controller_bmc = None
68 self._define_first_controller()
70 def _get_bool_arg(self, args, arg, default):
71 if hasattr(args, arg):
72 arg_value = vars(args)[arg]
73 if not isinstance(arg_value, bool):
74 if isinstance(arg_value, basestring):
76 arg_value = bool(distutils.util.strtobool(arg_value))
79 logging.warning('Invalid value for %s: %s', arg, arg_value)
85 def _set_arguments(self, args):
86 self._disable_bmc_initial_reset = self._get_bool_arg(args, 'disable_bmc_initial_reset', self._disable_bmc_initial_reset)
87 self._disable_other_bmc_reset = self._get_bool_arg(args, 'disable_other_bmc_reset', self._disable_other_bmc_reset)
89 self._boot_iso_path = args.boot_iso
90 self._iso_url = args.iso
91 self._callback_url = args.callback_url
92 self._client_key = args.client_key
93 self._client_cert = args.client_cert
94 self._ca_cert = args.ca_cert
95 self._own_ip = args.host_ip
99 def _read_user_config(config_file_path):
100 logging.debug('Read user config from %s', config_file_path)
103 with open(config_file_path, 'r') as f:
107 except Exception as ex:
108 raise InstallException(str(ex))
111 def _execute_shell(command, desc=''):
112 logging.debug('Execute %s with command: %s', desc, command)
114 p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
115 out, _ = p.communicate()
117 logging.warning('Failed to %s: %s (rc=%s)', desc, out, p.returncode)
118 raise InstallException('Failed to {}'.format(desc))
120 return (p.returncode, out)
122 def _attach_iso_as_virtual_media(self, file_list):
123 logging.info('Attach ISO as virtual media')
125 nfs_mount = os.path.dirname(self._boot_iso_path)
126 boot_iso_filename = os.path.basename(self._boot_iso_path)
127 patched_iso_filename = '{}/{}_{}'.format(nfs_mount, self._tag, boot_iso_filename)
129 self._patch_iso(patched_iso_filename, file_list)
131 self._first_controller_bmc.attach_virtual_cd(self._own_ip, nfs_mount, os.path.basename(patched_iso_filename))
133 def _setup_bmc_for_node(self, hw):
134 bmc_log_path = '{}/{}.log'.format(self._logdir, hw)
136 host = self._uc['hosts'][hw]['hwmgmt']['address']
137 user = self._uc['hosts'][hw]['hwmgmt']['user']
138 passwd = self._uc['hosts'][hw]['hwmgmt']['password']
139 priv_level = self._uc['hosts'][hw]['hwmgmt'].get('priv_level', 'ADMINISTRATOR')
142 hw_data = hw_detect.get_hw_data(host, user, passwd, priv_level, False)
143 except HWException as e:
144 error = "Hardware not detected for {}: {}".format(hw, str(e))
146 raise BMCException(error)
148 logging.debug("Hardware belongs to %s product family", (hw_data['product_family']))
149 if 'Unknown' in hw_data['product_family']:
150 error = "Hardware not detected for %s" % hw
152 raise BMCException(error)
154 bmc_mod_name = 'remoteinstaller.installer.bmc_management.{}'.format(hw_data['product_family'].lower())
155 bmc_mod = importlib.import_module(bmc_mod_name)
156 bmc_class = getattr(bmc_mod, hw_data['product_family'])
157 bmc = bmc_class(host, user, passwd, bmc_log_path)
158 bmc.set_host_name(hw)
162 def _define_first_controller(self):
163 for hw in sorted(self._uc['hosts']):
164 logging.debug('HW node name is %s', hw)
166 if 'controller' in self._uc['hosts'][hw]['service_profiles'] or \
167 'caas_master' in self._uc['hosts'][hw]['service_profiles']:
168 self._first_controller = hw
171 logging.info('First controller is %s', self._first_controller)
172 self._first_controller_bmc = self._setup_bmc_for_node(self._first_controller)
174 domain = self._uc['hosts'][self._first_controller].get('network_domain')
175 extnet = self._uc['networking']['infra_external']['network_domains'][domain]
177 first_ip = extnet['ip_range_start']
178 self._vip = str(IPAddress(first_ip))
180 pre_allocated_ips = self._uc['hosts'][self._first_controller].get('pre_allocated_ips', None)
181 if pre_allocated_ips:
182 pre_allocated_infra_external_ip = pre_allocated_ips.get('infra_external', None)
183 self._first_controller_ip = str(IPAddress(pre_allocated_infra_external_ip))
185 if not self._first_controller_ip:
186 self._first_controller_ip = str(IPAddress(first_ip)+1)
188 def _create_cloud_config(self):
189 logging.info('Create network config file')
191 domain = self._uc['hosts'][self._first_controller].get('network_domain')
192 extnet = self._uc['networking']['infra_external']['network_domains'][domain]
194 vlan = extnet.get('vlan')
195 gateway = extnet['gateway']
196 dns = self._uc['networking']['dns'][0]
197 cidr = extnet['cidr']
198 prefix = IPNetwork(cidr).prefixlen
200 controller_network_profile = self._uc['hosts'][self._first_controller]['network_profiles'][0]
201 mappings = self._uc['network_profiles'][controller_network_profile]['interface_net_mapping']
202 for interface in mappings:
203 if 'infra_external' in mappings[interface]:
204 infra_external_interface = interface
207 if infra_external_interface.startswith('bond'):
208 bonds = self._uc['network_profiles'][controller_network_profile]['bonding_interfaces']
209 device = bonds[infra_external_interface][0]
211 device = infra_external_interface
216 logging.debug('VLAN=%s', vlan)
217 logging.debug('DEV=%s', device)
218 logging.debug('IP=%s/%s', self._first_controller_ip, prefix)
219 logging.debug('DGW=%s', gateway)
220 logging.debug('NAMESERVER=%s', dns)
221 logging.debug('ISO_URL="%s"', self._iso_url)
223 network_config_filename = '{}/network_config'.format(self._logdir)
224 with open(network_config_filename, 'w') as f:
226 f.write('VLAN={}\n'.format(vlan))
227 f.write('DEV={}\n'.format(device))
228 f.write('IP={}/{}\n'.format(self._first_controller_ip, prefix))
229 f.write('DGW={}\n'.format(gateway))
230 f.write('NAMESERVER={}\n'.format(dns))
232 f.write('ISO_URL="{}"'.format(self._iso_url))
234 return network_config_filename
236 def _create_callback_file(self):
237 logging.debug('CALLBACK_URL="%s"', self._callback_url)
239 callback_url_filename = '{}/callback_url'.format(self._logdir)
240 with open(callback_url_filename, 'w') as f:
241 f.write(self._callback_url)
243 return callback_url_filename
245 def _patch_iso(self, iso_target, file_list):
246 logging.info('Patch boot ISO')
247 logging.debug('Original ISO: %s', self._boot_iso_path)
248 logging.debug('Target ISO: %s', iso_target)
250 file_list_str = ' '.join(file_list)
251 logging.debug('Files to add: %s', file_list_str)
253 self._execute_shell('/usr/bin/patchiso.sh {} {} {}'.format(self._boot_iso_path,
255 file_list_str), 'patch ISO')
257 def _put_file(self, ip, user, passwd, file_name, to_file=''):
258 self._execute_shell('sshpass -p {} scp {} {} {}@{}:{}'.format(passwd,
263 to_file), 'put file')
265 def _get_file(self, ip, user, passwd, file_name, recursive=False):
267 self._execute_shell('sshpass -p {} scp {} -r {}@{}:{} {}'.format(passwd,
272 self._logdir), 'get files')
274 self._execute_shell('sshpass -p {} scp {} {}@{}:{} {}'.format(passwd,
279 self._logdir), 'get file')
281 def _run_node_command(self, ip, user, passwd, command):
282 self._execute_shell('sshpass -p {} ssh {} {}@{} {}'.format(passwd,
286 command), 'run command: {}'.format(command))
288 def _get_node_logs(self, ip, user, passwd):
289 self._get_file(ip, user, passwd, '/srv/deployment/log/cm.log')
290 self._get_file(ip, user, passwd, '/srv/deployment/log/bootstrap.log')
291 self._get_file(ip, user, passwd, '/var/log/ironic', recursive=True)
293 def _create_hosts_file(self, file_name):
294 with open(file_name, 'w') as hosts_file:
295 for host in self._uc['hosts'].keys():
296 hosts_file.write('{}\n'.format(host))
298 def _get_journal_logs(self, ip, user, passwd):
299 hosts_file_name = 'host_names'
300 hosts_file_path = '{}/{}'.format(self._logdir, hosts_file_name)
301 self._create_hosts_file(hosts_file_name)
303 host_list = ' '.join(self._uc['hosts'].keys())
305 self._put_file(ip, user, passwd, hosts_file_name)
306 self._put_file(ip, user, passwd, '/opt/scripts/get_journals.sh')
308 self._run_node_command(ip, user, passwd, 'sh ./get_journals.sh {}'.format(hosts_file_name))
310 self._get_file(ip, user, passwd, '/tmp/node_journals.tgz')
312 def _get_logs_from_console(self, bmc, admin_user, admin_passwd):
313 bmc_host = bmc.get_host()
314 bmc_user = bmc.get_user()
315 bmc_passwd = bmc.get_passwd()
317 log_file = '{}/cat_bootstrap.log'.format(self._logdir)
319 cat_file = CatFile(bmc_host, bmc_user, bmc_passwd, admin_user, admin_passwd)
320 cat_file.cat('/srv/deployment/log/bootstrap.log', log_file)
321 except CatFileException as ex:
322 logging.info('Could not cat file from console: %s', str(ex))
324 cat_file = CatFile(bmc_host, bmc_user, bmc_passwd, 'root', 'root')
325 cat_file.cat('/srv/deployment/log/bootstrap.log', log_file)
327 def get_logs(self, admin_passwd):
328 admin_user = self._uc['users']['admin_user_name']
330 ssh_check_command = 'nc -w1 {} 22 </dev/null &> /dev/null'.format(self._first_controller_ip)
331 ssh_check_fails = os.system(ssh_check_command)
333 if not ssh_check_fails:
334 self._get_node_logs(self._first_controller_ip, admin_user, admin_passwd)
336 self._get_journal_logs(self._first_controller_ip, admin_user, admin_passwd)
338 self._get_logs_from_console(self._first_controller_bmc,
342 def _setup_bmcs(self):
344 for hw in sorted(self._uc['hosts']):
345 logging.info('HW node name is %s', hw)
347 bmc = self._setup_bmc_for_node(hw)
350 if hw != self._first_controller:
351 other_bmcs.append(bmc)
354 if not self._disable_bmc_initial_reset:
355 self._first_controller_bmc.reset()
356 time_after_reset = int(time.time())
358 if not self._disable_other_bmc_reset:
359 for bmc in other_bmcs:
362 if not self._disable_bmc_initial_reset:
363 # Make sure we sleep at least 6min after the first controller BMC reset
364 sleep_time = 6*60-int(time.time())-time_after_reset
366 logging.debug('Waiting for first controller BMC to stabilize \
367 (%s sec) after reset', sleep_time)
368 time.sleep(sleep_time)
370 def get_access_info(self):
371 access_info = {'vip': self._vip,
372 'installer_node_ip': self._first_controller_ip,
373 'admin_user': self._uc['users']['admin_user_name']}
377 def _set_progress(self, description, failed=False):
383 self._callback_server.set_state(self._callback_uuid, state, description)
387 logging.info('Start install')
389 if os.path.dirname(self._boot_iso_path) == '':
390 self._boot_iso_path = '{}/{}'.format(os.getcwd(), self._boot_iso_path)
393 if not os.path.exists(self._logdir):
394 os.makedirs(self._logdir)
398 self._set_progress('Setup BMCs')
401 self._set_progress('Create config files')
402 network_config_filename = self._create_cloud_config()
403 callback_url_filename = self._create_callback_file()
405 patch_files = [self._yaml_path,
406 network_config_filename,
407 callback_url_filename]
409 if self._client_cert:
410 patch_files.append(self._client_cert)
412 patch_files.append(self._client_key)
414 patch_files.append(self._ca_cert)
416 self._set_progress('Setup boot options for virtual media')
417 self._first_controller_bmc.setup_boot_options_for_virtual_media()
419 self._set_progress('Attach iso as virtual media')
420 self._attach_iso_as_virtual_media(patch_files)
422 self._set_progress('Boot from virtual media')
423 self._first_controller_bmc.boot_from_virtual_media()
425 self._set_progress('Wait for bootup')
426 self._first_controller_bmc.wait_for_bootup()
428 self._set_progress('Wait deployment start')
430 self._first_controller_bmc.close()
431 except BMCException as ex:
432 logging.error('Installation failed: %s', str(ex))
433 raise InstallException(str(ex))
436 parser = argparse.ArgumentParser()
437 parser.add_argument('-y', '--yaml', required=True,
438 help='User config yaml file path')
439 parser.add_argument('-b', '--boot-iso', required=True,
440 help='Path to boot ISO image in NFS mount')
441 parser.add_argument('-i', '--iso', required=True,
442 help='URL to ISO image')
443 parser.add_argument('-d', '--debug', action='store_true', required=False,
444 help='Debug level for logging')
445 parser.add_argument('-l', '--logdir', required=True,
446 help='Directory path for log files')
447 parser.add_argument('-c', '--callback-url', required=True,
448 help='Callback URL for progress reporting')
449 parser.add_argument('-K', '--client-key', required=True,
450 help='Client key file path')
451 parser.add_argument('-C', '--client-cert', required=True,
452 help='Client cert file path')
453 parser.add_argument('-A', '--ca-cert', required=True,
454 help='CA cert file path')
455 parser.add_argument('-H', '--host-ip', required=True,
456 help='IP for hosting HTTPD and NFS')
457 parser.add_argument('-T', '--http-port', required=False,
458 help='Port for HTTPD')
460 parsed_args = parser.parse_args()
462 if parsed_args.debug:
463 log_level = logging.DEBUG
465 log_level = logging.INFO
467 logging.basicConfig(stream=sys.stdout, level=log_level)
469 logging.debug('args: %s', parsed_args)
470 installer = Installer(parsed_args.yaml, parsed_args.logdir, parsed_args)
474 if __name__ == "__main__":