3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
25 from netaddr import IPNetwork
26 from netaddr import IPAddress
28 import hw_detector.hw_detect_lib as hw_detect
29 from hw_detector.hw_exception import HWException
30 from remoteinstaller.installer.bmc_management.bmctools import BMCException
31 from remoteinstaller.installer.catfile import CatFile
32 from remoteinstaller.installer.catfile import CatFileException
34 class InstallException(Exception):
37 class Installer(object):
38 SSH_OPTS = ('-o StrictHostKeyChecking=no '
39 '-o UserKnownHostsFile=/dev/null '
40 '-o ServerAliveInterval=60')
42 def __init__(self, callback_server, callback_uuid, yaml, logdir, args=None):
43 self._callback_server = callback_server
44 self._callback_uuid = callback_uuid
45 self._yaml_path = yaml
46 self._uc = self._read_user_config(self._yaml_path)
49 self._boot_iso_path = None
51 self._callback_url = None
52 self._client_key = None
53 self._client_cert = None
57 self._disable_bmc_initial_reset = False
58 self._disable_other_bmc_reset = True
61 self._set_arguments(args)
64 self._first_controller = None
65 self._first_controller_ip = None
66 self._first_controller_bmc = None
68 self._define_first_controller()
70 def _get_bool_arg(self, args, arg, default):
71 if hasattr(args, arg):
72 arg_value = vars(args)[arg]
73 if not isinstance(arg_value, bool):
74 if isinstance(arg_value, basestring):
76 arg_value = bool(distutils.util.strtobool(arg_value))
79 logging.warning('Invalid value for %s: %s', arg, arg_value)
85 def _set_arguments(self, args):
86 self._disable_bmc_initial_reset = self._get_bool_arg(args, 'disable_bmc_initial_reset', self._disable_bmc_initial_reset)
87 self._disable_other_bmc_reset = self._get_bool_arg(args, 'disable_other_bmc_reset', self._disable_other_bmc_reset)
89 self._boot_iso_path = args.boot_iso
90 self._iso_url = args.iso
91 self._callback_url = args.callback_url
92 self._client_key = args.client_key
93 self._client_cert = args.client_cert
94 self._ca_cert = args.ca_cert
95 self._own_ip = args.host_ip
99 def _read_user_config(config_file_path):
100 logging.debug('Read user config from %s', config_file_path)
103 with open(config_file_path, 'r') as f:
107 except Exception as ex:
108 raise InstallException(str(ex))
111 def _execute_shell(command, desc=''):
112 logging.debug('Execute %s with command: %s', desc, command)
114 p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
115 out, _ = p.communicate()
117 logging.warning('Failed to %s: %s (rc=%s)', desc, out, p.returncode)
118 raise InstallException('Failed to {}'.format(desc))
120 return (p.returncode, out)
122 def _attach_iso_as_virtual_media(self, file_list):
123 logging.info('Attach ISO as virtual media')
125 nfs_mount = os.path.dirname(self._boot_iso_path)
126 boot_iso_filename = os.path.basename(self._boot_iso_path)
127 patched_iso_filename = '{}/{}_{}'.format(nfs_mount, self._tag, boot_iso_filename)
129 self._patch_iso(patched_iso_filename, file_list)
131 self._first_controller_bmc.attach_virtual_cd(self._own_ip, nfs_mount, os.path.basename(patched_iso_filename))
133 def _setup_bmc_for_node(self, hw):
134 bmc_log_path = '{}/{}.log'.format(self._logdir, hw)
136 host = self._uc['hosts'][hw]['hwmgmt']['address']
137 user = self._uc['hosts'][hw]['hwmgmt']['user']
138 passwd = self._uc['hosts'][hw]['hwmgmt']['password']
139 priv_level = self._uc['hosts'][hw]['hwmgmt'].get('priv_level', 'ADMINISTRATOR')
142 hw_data = hw_detect.get_hw_data(host, user, passwd, priv_level, False)
143 except HWException as e:
144 error = "Hardware not detected for {}: {}".format(hw, str(e))
146 raise BMCException(error)
148 logging.debug("Hardware belongs to %s product family", (hw_data['product_family']))
149 if 'Unknown' in hw_data['product_family']:
150 error = "Hardware not detected for %s" % hw
152 raise BMCException(error)
154 bmc_mod_name = 'remoteinstaller.installer.bmc_management.{}'.format(hw_data['product_family'].lower())
155 bmc_mod = importlib.import_module(bmc_mod_name)
156 bmc_class = getattr(bmc_mod, hw_data['product_family'])
157 bmc = bmc_class(host, user, passwd, priv_level, bmc_log_path)
158 bmc.set_host_name(hw)
162 def _define_first_controller(self):
163 for hw in sorted(self._uc['hosts']):
164 logging.debug('HW node name is %s', hw)
166 if 'controller' in self._uc['hosts'][hw]['service_profiles'] or \
167 'caas_master' in self._uc['hosts'][hw]['service_profiles']:
168 self._first_controller = hw
171 logging.info('First controller is %s', self._first_controller)
172 self._first_controller_bmc = self._setup_bmc_for_node(self._first_controller)
174 domain = self._uc['hosts'][self._first_controller].get('network_domain')
175 extnet = self._uc['networking']['infra_external']['network_domains'][domain]
177 first_ip = extnet['ip_range_start']
178 self._vip = str(IPAddress(first_ip))
180 pre_allocated_ips = self._uc['hosts'][self._first_controller].get('pre_allocated_ips', None)
181 if pre_allocated_ips:
182 pre_allocated_infra_external_ip = pre_allocated_ips.get('infra_external', None)
183 if pre_allocated_infra_external_ip:
184 self._first_controller_ip = str(IPAddress(pre_allocated_infra_external_ip))
186 if not self._first_controller_ip:
187 self._first_controller_ip = str(IPAddress(first_ip)+1)
189 def _create_cloud_config(self):
190 logging.info('Create network config file')
192 domain = self._uc['hosts'][self._first_controller].get('network_domain')
193 extnet = self._uc['networking']['infra_external']['network_domains'][domain]
195 vlan = extnet.get('vlan')
196 gateway = extnet['gateway']
197 dns = self._uc['networking']['dns'][0]
198 cidr = extnet['cidr']
199 prefix = IPNetwork(cidr).prefixlen
201 controller_network_profile = self._uc['hosts'][self._first_controller]['network_profiles'][0]
202 mappings = self._uc['network_profiles'][controller_network_profile]['interface_net_mapping']
203 for interface in mappings:
204 if 'infra_external' in mappings[interface]:
205 infra_external_interface = interface
208 if infra_external_interface.startswith('bond'):
209 bonds = self._uc['network_profiles'][controller_network_profile]['bonding_interfaces']
210 device = bonds[infra_external_interface][0]
212 device = infra_external_interface
217 logging.debug('VLAN=%s', vlan)
218 logging.debug('DEV=%s', device)
219 logging.debug('IP=%s/%s', self._first_controller_ip, prefix)
220 logging.debug('DGW=%s', gateway)
221 logging.debug('NAMESERVER=%s', dns)
222 logging.debug('ISO_URL="%s"', self._iso_url)
224 network_config_filename = '{}/network_config'.format(self._logdir)
225 with open(network_config_filename, 'w') as f:
227 f.write('VLAN={}\n'.format(vlan))
228 f.write('DEV={}\n'.format(device))
229 f.write('IP={}/{}\n'.format(self._first_controller_ip, prefix))
230 f.write('DGW={}\n'.format(gateway))
231 f.write('NAMESERVER={}\n'.format(dns))
233 f.write('ISO_URL="{}"'.format(self._iso_url))
235 return network_config_filename
237 def _create_callback_file(self):
238 logging.debug('CALLBACK_URL="%s"', self._callback_url)
240 callback_url_filename = '{}/callback_url'.format(self._logdir)
241 with open(callback_url_filename, 'w') as f:
242 f.write(self._callback_url)
244 return callback_url_filename
246 def _patch_iso(self, iso_target, file_list):
247 logging.info('Patch boot ISO')
248 logging.debug('Original ISO: %s', self._boot_iso_path)
249 logging.debug('Target ISO: %s', iso_target)
251 file_list_str = ' '.join(file_list)
252 logging.debug('Files to add: %s', file_list_str)
254 self._execute_shell('/usr/bin/patchiso.sh {} {} {}'.format(self._boot_iso_path,
256 file_list_str), 'patch ISO')
258 def _put_file(self, ip, user, passwd, file_name, to_file=''):
259 self._execute_shell('sshpass -p {} scp {} {} {}@{}:{}'.format(passwd,
264 to_file), 'put file')
266 def _get_file(self, ip, user, passwd, file_name, recursive=False):
268 self._execute_shell('sshpass -p {} scp {} -r {}@{}:{} {}'.format(passwd,
273 self._logdir), 'get files')
275 self._execute_shell('sshpass -p {} scp {} {}@{}:{} {}'.format(passwd,
280 self._logdir), 'get file')
282 def _run_node_command(self, ip, user, passwd, command):
283 self._execute_shell('sshpass -p {} ssh {} {}@{} {}'.format(passwd,
287 command), 'run command: {}'.format(command))
289 def _get_node_logs(self, ip, user, passwd):
290 self._get_file(ip, user, passwd, '/srv/deployment/log/cm.log')
291 self._get_file(ip, user, passwd, '/srv/deployment/log/bootstrap.log')
292 self._get_file(ip, user, passwd, '/var/log/ironic', recursive=True)
294 def _create_hosts_file(self, file_name):
295 with open(file_name, 'w') as hosts_file:
296 for host in self._uc['hosts'].keys():
297 hosts_file.write('{}\n'.format(host))
299 def _get_journal_logs(self, ip, user, passwd):
300 hosts_file_name = 'host_names'
301 hosts_file_path = '{}/{}'.format(self._logdir, hosts_file_name)
302 self._create_hosts_file(hosts_file_name)
304 host_list = ' '.join(self._uc['hosts'].keys())
306 self._put_file(ip, user, passwd, hosts_file_name)
307 self._put_file(ip, user, passwd, '/opt/scripts/get_journals.sh')
309 self._run_node_command(ip, user, passwd, 'sh ./get_journals.sh {}'.format(hosts_file_name))
311 self._get_file(ip, user, passwd, '/tmp/node_journals.tgz')
313 def _get_logs_from_console(self, bmc, admin_user, admin_passwd):
314 bmc_host = bmc.get_host()
315 bmc_user = bmc.get_user()
316 bmc_passwd = bmc.get_passwd()
317 bmc_priv_level = bmc.get_priv_level()
319 log_file = '{}/cat_bootstrap.log'.format(self._logdir)
321 cat_file = CatFile(bmc_host, bmc_user, bmc_passwd, bmc_priv_level, admin_user, admin_passwd)
322 cat_file.cat('/srv/deployment/log/bootstrap.log', log_file)
323 except CatFileException as ex:
324 logging.info('Could not cat file from console: %s', str(ex))
326 cat_file = CatFile(bmc_host, bmc_user, bmc_passwd, bmc_priv_level, 'root', 'root')
327 cat_file.cat('/srv/deployment/log/bootstrap.log', log_file)
329 def get_logs(self, admin_passwd):
330 admin_user = self._uc['users']['admin_user_name']
332 ssh_check_command = 'nc -w1 {} 22 </dev/null &> /dev/null'.format(self._first_controller_ip)
333 ssh_check_fails = os.system(ssh_check_command)
335 if not ssh_check_fails:
336 self._get_node_logs(self._first_controller_ip, admin_user, admin_passwd)
338 self._get_journal_logs(self._first_controller_ip, admin_user, admin_passwd)
340 self._get_logs_from_console(self._first_controller_bmc,
344 def _setup_bmcs(self):
346 for hw in sorted(self._uc['hosts']):
347 logging.info('HW node name is %s', hw)
349 bmc = self._setup_bmc_for_node(hw)
352 if hw != self._first_controller:
353 other_bmcs.append(bmc)
356 if not self._disable_bmc_initial_reset:
357 self._first_controller_bmc.reset()
358 time_after_reset = int(time.time())
360 if not self._disable_other_bmc_reset:
361 for bmc in other_bmcs:
364 if not self._disable_bmc_initial_reset:
365 # Make sure we sleep at least 6min after the first controller BMC reset
366 sleep_time = 6*60-int(time.time())-time_after_reset
368 logging.debug('Waiting for first controller BMC to stabilize \
369 (%s sec) after reset', sleep_time)
370 time.sleep(sleep_time)
372 def get_access_info(self):
373 access_info = {'vip': self._vip,
374 'installer_node_ip': self._first_controller_ip,
375 'admin_user': self._uc['users']['admin_user_name']}
379 def _set_progress(self, description, failed=False):
385 self._callback_server.set_state(self._callback_uuid, state, description)
389 logging.info('Start install')
391 if os.path.dirname(self._boot_iso_path) == '':
392 self._boot_iso_path = '{}/{}'.format(os.getcwd(), self._boot_iso_path)
395 if not os.path.exists(self._logdir):
396 os.makedirs(self._logdir)
400 self._set_progress('Setup BMCs')
403 self._set_progress('Create config files')
404 network_config_filename = self._create_cloud_config()
405 callback_url_filename = self._create_callback_file()
407 patch_files = [self._yaml_path,
408 network_config_filename,
409 callback_url_filename]
411 if self._client_cert:
412 patch_files.append(self._client_cert)
414 patch_files.append(self._client_key)
416 patch_files.append(self._ca_cert)
418 self._set_progress('Setup boot options for virtual media')
419 self._first_controller_bmc.setup_boot_options_for_virtual_media()
421 self._set_progress('Attach iso as virtual media')
422 self._attach_iso_as_virtual_media(patch_files)
424 self._set_progress('Boot from virtual media')
425 self._first_controller_bmc.boot_from_virtual_media()
427 self._set_progress('Wait for bootup')
428 self._first_controller_bmc.wait_for_bootup()
430 self._set_progress('Wait deployment start')
432 self._first_controller_bmc.close()
433 except BMCException as ex:
434 logging.error('Installation failed: %s', str(ex))
435 raise InstallException(str(ex))
438 parser = argparse.ArgumentParser()
439 parser.add_argument('-y', '--yaml', required=True,
440 help='User config yaml file path')
441 parser.add_argument('-b', '--boot-iso', required=True,
442 help='Path to boot ISO image in NFS mount')
443 parser.add_argument('-i', '--iso', required=True,
444 help='URL to ISO image')
445 parser.add_argument('-d', '--debug', action='store_true', required=False,
446 help='Debug level for logging')
447 parser.add_argument('-l', '--logdir', required=True,
448 help='Directory path for log files')
449 parser.add_argument('-c', '--callback-url', required=True,
450 help='Callback URL for progress reporting')
451 parser.add_argument('-K', '--client-key', required=True,
452 help='Client key file path')
453 parser.add_argument('-C', '--client-cert', required=True,
454 help='Client cert file path')
455 parser.add_argument('-A', '--ca-cert', required=True,
456 help='CA cert file path')
457 parser.add_argument('-H', '--host-ip', required=True,
458 help='IP for hosting HTTPD and NFS')
459 parser.add_argument('-T', '--http-port', required=False,
460 help='Port for HTTPD')
462 parsed_args = parser.parse_args()
464 if parsed_args.debug:
465 log_level = logging.DEBUG
467 log_level = logging.INFO
469 logging.basicConfig(stream=sys.stdout, level=log_level)
471 logging.debug('args: %s', parsed_args)
472 installer = Installer(parsed_args.yaml, parsed_args.logdir, parsed_args)
476 if __name__ == "__main__":