From a772a381a84dd9b906a06c314b113b997bce71f4 Mon Sep 17 00:00:00 2001 From: Jyrki Aaltonen Date: Thu, 16 May 2019 13:26:02 +0300 Subject: [PATCH] Restructure server Server and installer restructured to enable real installation. Change-Id: Icf357dbc112c4359996ad8cfec53557a260b85ad Signed-off-by: Jyrki Aaltonen --- docker-build/remote-installer/Dockerfile | 8 +- scripts/build.sh | 1 - scripts/start.sh | 23 +- src/remoteinstaller/client/remote-installer | 173 ++++++++++++ .../installer/bmc_management/bmctools.py | 2 + src/remoteinstaller/installer/install.py | 311 ++++++++++++--------- src/remoteinstaller/server/server.py | 138 +++++---- src/scripts/get_journals.sh | 31 ++ src/scripts/print_hosts.py | 20 ++ 9 files changed, 505 insertions(+), 202 deletions(-) create mode 100755 src/remoteinstaller/client/remote-installer create mode 100644 src/scripts/get_journals.sh create mode 100644 src/scripts/print_hosts.py diff --git a/docker-build/remote-installer/Dockerfile b/docker-build/remote-installer/Dockerfile index 0c06de4..38626eb 100644 --- a/docker-build/remote-installer/Dockerfile +++ b/docker-build/remote-installer/Dockerfile @@ -17,6 +17,7 @@ MAINTAINER Ralf Mueller ENV \ ETC_REMOTE_INST="/etc/remoteinstaller" \ +SCRIPTS_DIR="/opt/scripts" \ PW="root" \ API_PORT="15101" \ API_LISTEN_ADDR="0.0.0.0" \ @@ -50,7 +51,7 @@ RUN yum -y install systemd epel-release; yum clean all \ && yum install -y iproute wget openssh-server lighttpd nfs-utils \ python-setuptools python2-eventlet python-routes PyYAML \ python-netaddr pexpect net-tools tcpdump \ -ipmitool \ +ipmitool openssh-clients sshpass nmap-ncat \ # mod_ssl \ && systemctl enable sshd \ && systemctl enable lighttpd \ @@ -90,7 +91,10 @@ RUN pushd "$INSTALLER_MOUNT" \ && rm -rf * \ && popd -RUN mkdir -p "$ETC_REMOTE_INST" +RUN mkdir -p "$SCRIPTS_DIR" \ +&& mkdir -p "$ETC_REMOTE_INST" + +COPY src/scripts/get_journals.sh src/scripts/print_hosts.py "$SCRIPTS_DIR"/ RUN echo '#!/bin/bash' >>$STARTUP \ && echo 'printenv >/etc/remoteinstaller/environment' >>$STARTUP \ diff --git a/scripts/build.sh b/scripts/build.sh index dcfc234..a73cb9f 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -40,7 +40,6 @@ while getopts "hs" arg; do done docker build \ - --network=host \ --no-cache \ --force-rm \ --build-arg HTTP_PROXY="${http_proxy}" \ diff --git a/scripts/start.sh b/scripts/start.sh index ca3b7e4..1e3a6fe 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -34,15 +34,15 @@ error() help() { - echo -e "$(basename $0) [-h -a -c -i -r -s ] -b -e " - echo -e " -h display this help" - echo -e " -a rest API port, default $API_PORT" + echo -e "$(basename $0) [-h -a -c -i -r -s ] -b -e " + echo -e " -h display this help" + echo -e " -a rest API port, default $API_PORT" echo -e " -c container name, default $CONT_NAME" - echo -e " -b base directory, which contains images, certificates, etc." - echo -e " -e external ip address of the docker" - echo -e " -i secure https port, default $IMG_NAME" - echo -e " -p root password, default $ROOT_PW" - echo -e " -s secure https port, default $HTTPS_PORT" + echo -e " -b base directory, which contains images, certificates, etc." + echo -e " -e external ip address of the docker" + echo -e " -i secure https port, default $IMG_NAME" + echo -e " -p root password, default $ROOT_PW" + echo -e " -s secure https port, default $HTTPS_PORT" } while getopts "ha:b:e:s:c:p:i:" arg; do @@ -52,10 +52,10 @@ while getopts "ha:b:e:s:c:p:i:" arg; do exit 0 ;; b) - BASE_DIR="$OPTARG" + BASE_DIR="$OPTARG" ;; e) - EXT_IP="$OPTARG" + EXT_IP="$OPTARG" ;; s) HTTPS_PORT="$OPTARG" @@ -69,6 +69,9 @@ while getopts "ha:b:e:s:c:p:i:" arg; do i) IMG_NAME="$OPTARG" ;; + p) + ROOT_PW="$OPTARG" + ;; *) error "Unknow argument!" showhelp ;; diff --git a/src/remoteinstaller/client/remote-installer b/src/remoteinstaller/client/remote-installer new file mode 100755 index 0000000..f183ef4 --- /dev/null +++ b/src/remoteinstaller/client/remote-installer @@ -0,0 +1,173 @@ +#! /usr/bin/python + +# Copyright 2019 Nokia + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import argparse +import requests +import json + + +class Client(object): + DEFAULT_INSTALL_URL = 'http://{}:{}/v1/installations/' + DEFAULT_INSTALL_STATE_URL = 'http://{}:{}/v1/installations/{}/state' + DEFAULT_PATH = '/opt/remoteinstaller' + DEFAULT_PORT = '15101' + DEFAULT_HOST = 'localhost' + + def __init__(self): + self._verbose = None + self._host = Client.DEFAULT_HOST + self._port = Client.DEFAULT_PORT + self._client_cert_path = None + self._client_key_path = None + self._user_config = None + self._image = None + self._request_url = None + self._uuid = None + self._parser = None + self._args = self._parse_args(sys.argv[1:]) + self._debug(self._args) + + def _parse_args(self, args): + parser = argparse.ArgumentParser(description='Remote Installer Client',add_help=False) + self._parser = parser + subparsers = parser.add_subparsers(dest="subparsers") + + install_parser = subparsers.add_parser('install', description='Remote Installer Client: intall') + install_parser.add_argument('--image', + dest='image', required=True, + help='Full path to installation iso image') + install_parser.add_argument('--user-config', required=True, + dest='userconfig', + help='Full path to user config') + install_parser.set_defaults(func=self._install) + + query_parser = subparsers.add_parser('get-progress', description='Remote Installer Client: get-progress') + query_parser.add_argument('--uuid', required=True, + dest='uuid', + help='Installation uuid') + query_parser.set_defaults(func=self._query_progress) + + for name, subp in subparsers.choices.items(): + subp.add_argument('--debug', action='store_true', + required=False, dest='debug', help = "Debug mode") + + subp.add_argument('--host', + dest='host', required=False, + help='Remote installer server address. %s used if not specified.' % Client.DEFAULT_HOST) + + subp.add_argument('--port', required=False, + dest='port', + help='Remote installer server port. %s used if not specified.' % Client.DEFAULT_PORT) + + subp.add_argument('--client-key', required=True, + dest='client_key_path', + help='Full path to client key') + + subp.add_argument('--client-certificate', required=True, + dest='client_cert_path', + help='Full path to client certificate') + + # To be removed before publishing + subp.add_argument('--insecure', required=False, + dest='insecure', action='store_true', + help='Allow http insecure connection') + + _args = parser.parse_args(args) + return _args + + def _debug(self, message): + if self._args.debug: + print "DEBUG: {}".format(str(message)) + + def _process_args(self, args): + if args: + if args.client_cert_path: + self._client_cert_path = args.client_cert_path + if args.client_key_path: + self._client_key_path = args.client_key_path + if args.port: + self._port = args.port + if args.host: + self._host = args.host + + def run(self): + self._process_args(self._args) + self._args.func(self._args) + + def _query_progress(self, args): + self._debug("get-progress") + self._uuid = self._args.uuid + self._build_request_url('get-progress') + request_data = {'uuid': self._uuid} + _response = self._post_request(request_data) + self._process_response(_response, request_type='get-progress') + + def _install(self, args): + self._debug('install') + self._user_config = self._args.userconfig + self._image = self._args.image + self._build_request_url('install') + request_data = {'user-config': self._user_config, 'iso': self._image} + _response = self._post_request(request_data) + self._process_response(_response, request_type='install') + + def _cert_tuple(self): + cert_tuple = None + cert_tuple = (self._client_cert_path, self._client_key_path) + return None if None in cert_tuple else cert_tuple + + def _build_request_url(self, request_type): + if request_type == 'install': + self._request_url = Client.DEFAULT_INSTALL_URL.format(self._host, self._port) + elif request_type == 'get-progress': + self._request_url = Client.DEFAULT_INSTALL_STATE_URL.format(self._host, self._port, self._uuid) + + def _post_request(self, request_data): + if self._request_url: + response = None + cert_tuple = self._cert_tuple() if not self._args.insecure else None + try: + response = requests.post(self._request_url, json=request_data, cert=cert_tuple) + self._debug("post request %s %s %s" % (self._request_url, request_data, cert_tuple)) + except Exception as ex: + self._debug('Failed to send request: {}'.format(str(ex))) + + if response.status_code != requests.codes.ok: + self._debug('Failed to send requst: %s (%s)', str(response.reason), str(response.status_code)) + else: + self._debug('response: %s' % response.json()) + return response.json() + + def _process_response(self, response_content, request_type): + _json = response_content + if request_type == 'install': + _uuid = _json.get('uuid') + print "{}".format(_uuid) + elif request_type == 'get-progress': + for key in ['status', 'description', 'percentage']: + print "{}".format(str(_json.get(key))) + +def main(): + try: + client = Client() + client.run() + except Exception as exp: + print 'Failed with error: %s', str(exp) + return 1 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/remoteinstaller/installer/bmc_management/bmctools.py b/src/remoteinstaller/installer/bmc_management/bmctools.py index dfdeaea..4a6e383 100644 --- a/src/remoteinstaller/installer/bmc_management/bmctools.py +++ b/src/remoteinstaller/installer/bmc_management/bmctools.py @@ -306,6 +306,8 @@ class BMC(object): logging.info('Retry to expect a flag in console, %s seconds remaining', remaining_time) self.close() + raise BMCException('Expected message in console did not occur in time ({})'.format(flags)) + def _wait_for_bios_settings_done(self): logging.debug('Wait until BIOS settings are updated') diff --git a/src/remoteinstaller/installer/install.py b/src/remoteinstaller/installer/install.py index 1d60066..d453bad 100644 --- a/src/remoteinstaller/installer/install.py +++ b/src/remoteinstaller/installer/install.py @@ -38,27 +38,44 @@ class Installer(object): -o UserKnownHostsFile=/dev/null \ -o ServerAliveInterval=60' - def __init__(self, args): - self._yaml_path = args.yaml + def __init__(self, callback_server, callback_uuid, yaml, logdir, args=None): + self._callback_server = callback_server + self._callback_uuid = callback_uuid + self._yaml_path = yaml + self._uc = self._read_user_config(self._yaml_path) + self._logdir = logdir + + self._boot_iso_path = None + self._iso_url = None + self._callback_url = None + self._client_key = None + self._client_cert = None + self._ca_cert = None + self._own_ip = None + self._tag = None + if args: + self._set_arguments(args) + + # TODO + self._disable_bmc_initial_reset = False + self._disable_other_bmc_reset = True + + self._vip = None + self._first_controller = None + self._first_controller_ip = None + self._first_controller_bmc = None + + self._define_first_controller() + + def _set_arguments(self, args): self._boot_iso_path = args.boot_iso self._iso_url = args.iso - self._logdir = args.logdir self._callback_url = args.callback_url self._client_key = args.client_key self._client_cert = args.client_cert self._ca_cert = args.ca_cert self._own_ip = args.host_ip self._tag = args.tag - self._http_port = args.http_port - - # TODO - self._disable_bmc_initial_reset = True - self._disable_other_bmc_reset = True - - self._uc = self._read_user_config(self._yaml_path) - self._vip = None - self._first_controller_ip = None - self._first_controller_bmc = None @staticmethod def _read_user_config(config_file_path): @@ -95,22 +112,53 @@ class Installer(object): self._first_controller_bmc.attach_virtual_cd(self._own_ip, nfs_mount, os.path.basename(patched_iso_filename)) - def _create_cloud_config(self, first_controller): - logging.info('Create network config file') + def _setup_bmc_for_node(self, hw): + bmc_log_path = '{}/{}.log'.format(self._logdir, hw) + + host = self._uc['hosts'][hw]['hwmgmt']['address'] + user = self._uc['hosts'][hw]['hwmgmt']['user'] + passwd = self._uc['hosts'][hw]['hwmgmt']['password'] + + try: + hw_data = hw_detect.get_hw_data(host, user, passwd, False) + except HWException as e: + error = "Harware not detected for {}: {}".format(hw, str(e)) + logging.error(error) + raise BMCException(error) + + logging.debug("Hardware belongs to %s product family", (hw_data['product_family'])) + if 'Unknown' in hw_data['product_family']: + error = "Hardware not detected for %s" % hw + logging.error(error) + raise BMCException(error) + + bmc_mod_name = 'remoteinstaller.installer.bmc_management.{}'.format(hw_data['product_family'].lower()) + bmc_mod = importlib.import_module(bmc_mod_name) + bmc_class = getattr(bmc_mod, hw_data['product_family']) + bmc = bmc_class(host, user, passwd, bmc_log_path) + bmc.set_host_name(hw) + + return bmc + + def _define_first_controller(self): + for hw in sorted(self._uc['hosts']): + logging.debug('HW node name is %s', hw) + + if 'controller' in self._uc['hosts'][hw]['service_profiles'] or \ + 'caas_master' in self._uc['hosts'][hw]['service_profiles']: + self._first_controller = hw + break + + logging.info('First controller is %s', self._first_controller) + self._first_controller_bmc = self._setup_bmc_for_node(self._first_controller) - domain = self._uc['hosts'][first_controller].get('network_domain') + domain = self._uc['hosts'][self._first_controller].get('network_domain') extnet = self._uc['networking']['infra_external']['network_domains'][domain] - vlan = extnet.get('vlan') first_ip = extnet['ip_range_start'] - gateway = extnet['gateway'] - dns = self._uc['networking']['dns'][0] - cidr = extnet['cidr'] - prefix = IPNetwork(cidr).prefixlen - self._vip = str(IPAddress(first_ip)) - pre_allocated_ips = self._uc['hosts'][first_controller].get('pre_allocated_ips', None) + pre_allocated_ips = self._uc['hosts'][self._first_controller].get('pre_allocated_ips', None) if pre_allocated_ips: pre_allocated_infra_external_ip = pre_allocated_ips.get('infra_external', None) self._first_controller_ip = str(IPAddress(pre_allocated_infra_external_ip)) @@ -118,7 +166,19 @@ class Installer(object): if not self._first_controller_ip: self._first_controller_ip = str(IPAddress(first_ip)+1) - controller_network_profile = self._uc['hosts'][first_controller]['network_profiles'][0] + def _create_cloud_config(self): + logging.info('Create network config file') + + domain = self._uc['hosts'][self._first_controller].get('network_domain') + extnet = self._uc['networking']['infra_external']['network_domains'][domain] + + vlan = extnet.get('vlan') + gateway = extnet['gateway'] + dns = self._uc['networking']['dns'][0] + cidr = extnet['cidr'] + prefix = IPNetwork(cidr).prefixlen + + controller_network_profile = self._uc['hosts'][self._first_controller]['network_profiles'][0] mappings = self._uc['network_profiles'][controller_network_profile]['interface_net_mapping'] for interface in mappings: if 'infra_external' in mappings[interface]: @@ -141,7 +201,7 @@ class Installer(object): logging.debug('NAMESERVER=%s', dns) logging.debug('ISO_URL="%s"', self._iso_url) - network_config_filename = '{}/network_config'.format(os.getcwd()) + network_config_filename = '{}/network_config'.format(self._logdir) with open(network_config_filename, 'w') as f: if vlan: f.write('VLAN={}\n'.format(vlan)) @@ -152,23 +212,16 @@ class Installer(object): f.write('\n') f.write('ISO_URL="{}"'.format(self._iso_url)) + return network_config_filename + + def _create_callback_file(self): logging.debug('CALLBACK_URL="%s"', self._callback_url) - callback_url_filename = '{}/callback_url'.format(os.getcwd()) + callback_url_filename = '{}/callback_url'.format(self._logdir) with open(callback_url_filename, 'w') as f: f.write(self._callback_url) - if self._client_cert: - return [self._yaml_path, - network_config_filename, - callback_url_filename, - self._client_key, - self._client_cert, - self._ca_cert] - - return [self._yaml_path, - network_config_filename, - callback_url_filename] + return callback_url_filename def _patch_iso(self, iso_target, file_list): logging.info('Patch boot ISO') @@ -188,43 +241,43 @@ class Installer(object): file_name, user, ip, - to_file)) + to_file), 'put file') - def _get_file(self, log_dir, ip, user, passwd, file_name, recursive=False): + def _get_file(self, ip, user, passwd, file_name, recursive=False): if recursive: self._execute_shell('sshpass -p {} scp {} -r {}@{}:{} {}'.format(passwd, Installer.SSH_OPTS, user, ip, file_name, - log_dir)) + self._logdir), 'get files') else: self._execute_shell('sshpass -p {} scp {} {}@{}:{} {}'.format(passwd, Installer.SSH_OPTS, user, ip, file_name, - log_dir)) + self._logdir), 'get file') - def _get_node_logs(self, log_dir, ip, user, passwd): - self._get_file(log_dir, ip, user, passwd, '/srv/deployment/log/cm.log') - self._get_file(log_dir, ip, user, passwd, '/srv/deployment/log/bootstrap.log') - self._get_file(log_dir, ip, user, passwd, '/var/log/ironic', recursive=True) + def _get_node_logs(self, ip, user, passwd): + self._get_file(ip, user, passwd, '/srv/deployment/log/cm.log') + self._get_file(ip, user, passwd, '/srv/deployment/log/bootstrap.log') + self._get_file(ip, user, passwd, '/var/log/ironic', recursive=True) - def _get_journal_logs(self, log_dir, ip, user, passwd): - self._put_file(ip, user, passwd, '/opt/remoteinstaller/get_journals.sh') - self._put_file(ip, user, passwd, '/opt/remoteinstaller/print_hosts.py') + def _get_journal_logs(self, ip, user, passwd): + self._put_file(ip, user, passwd, '/opt/scripts/get_journals.sh') + self._put_file(ip, user, passwd, '/opt/scripts/print_hosts.py') - self._execute_shell('sh ./get_journals.sh') + self._execute_shell('sh ./get_journals.sh', 'run get_journals.sh') - self._get_file(log_dir, ip, user, passwd, '/tmp/node_journals.tgz') + self._get_file(ip, user, passwd, '/tmp/node_journals.tgz') - def _get_logs_from_console(self, log_dir, bmc, admin_user, admin_passwd): + def _get_logs_from_console(self, bmc, admin_user, admin_passwd): bmc_host = bmc.get_host() bmc_user = bmc.get_user() bmc_passwd = bmc.get_passwd() - log_file = '{}/cat_bootstrap.log'.format(log_dir) + log_file = '{}/cat_bootstrap.log'.format(self._logdir) try: cat_file = CatFile(bmc_host, bmc_user, bmc_passwd, admin_user, admin_passwd) cat_file.cat('/srv/deployment/log/bootstrap.log', log_file) @@ -234,22 +287,64 @@ class Installer(object): cat_file = CatFile(bmc_host, bmc_user, bmc_passwd, 'root', 'root') cat_file.cat('/srv/deployment/log/bootstrap.log', log_file) - def get_logs(self, log_dir, admin_passwd): + def get_logs(self, admin_passwd): admin_user = self._uc['users']['admin_user_name'] - ssh_command = 'nc -w1 {} 22 /dev/null'.format(self._first_controller_ip) - ssh_responds = self._execute_shell(ssh_command) + ssh_check_command = 'nc -w1 {} 22 /dev/null'.format(self._first_controller_ip) + ssh_check_fails = os.system(ssh_check_command) - if ssh_responds: - self._get_node_logs(log_dir, self._first_controller_ip, admin_user, admin_passwd) + if not ssh_check_fails: + self._get_node_logs(self._first_controller_ip, admin_user, admin_passwd) - self._get_journal_logs(log_dir, self._first_controller_ip, admin_user, admin_passwd) + self._get_journal_logs(self._first_controller_ip, admin_user, admin_passwd) else: - self._get_logs_from_console(log_dir, - self._first_controller_bmc, + self._get_logs_from_console(self._first_controller_bmc, admin_user, admin_passwd) + def _setup_bmcs(self): + other_bmcs = [] + for hw in sorted(self._uc['hosts']): + logging.info('HW node name is %s', hw) + + bmc = self._setup_bmc_for_node(hw) + bmc.setup_sol() + + if hw != self._first_controller: + other_bmcs.append(bmc) + bmc.power('off') + + if not self._disable_bmc_initial_reset: + self._first_controller_bmc.reset() + time_after_reset = int(time.time()) + + if not self._disable_other_bmc_reset: + for bmc in other_bmcs: + bmc.reset() + + if not self._disable_bmc_initial_reset: + # Make sure we sleep at least 6min after the first controller BMC reset + sleep_time = 6*60-int(time.time())-time_after_reset + if sleep_time > 0: + logging.debug('Waiting for first controller BMC to stabilize \ + (%s sec) after reset', sleep_time) + time.sleep(sleep_time) + + def get_access_info(self): + access_info = {'vip': self._vip, + 'installer_node_ip': self._first_controller_ip, + 'admin_user': self._uc['users']['admin_user_name']} + + return access_info + + def _set_progress(self, description, failed=False): + if failed: + state = 'failed' + else: + state = 'ongoing' + + self._callback_server.set_state(self._callback_uuid, state, description) + def install(self): try: logging.info('Start install') @@ -263,85 +358,37 @@ class Installer(object): else: self._logdir = '.' - other_bmcs = [] - first_controller = None - for hw in sorted(self._uc['hosts']): - logging.info('HW node name is %s', hw) - - if not first_controller: - if 'controller' in self._uc['hosts'][hw]['service_profiles'] or \ - 'caas_master' in self._uc['hosts'][hw]['service_profiles']: - first_controller = hw - logging.info('HW is first controller') - - host = self._uc['hosts'][hw]['hwmgmt']['address'] - user = self._uc['hosts'][hw]['hwmgmt']['user'] - passwd = self._uc['hosts'][hw]['hwmgmt']['password'] - - bmc_log_path = '{}/{}.log'.format(self._logdir, hw) - - try: - hw_data = hw_detect.get_hw_data(host, user, passwd, False) - except HWException as e: - error = "Harware not detected for {}: {}".format(hw, str(e)) - logging.error(error) - raise BMCException(error) - - logging.info("Hardware belongs to %s product family", (hw_data['product_family'])) - if 'Unknown' in hw_data['product_family']: - error = "Hardware not detected for %s" % hw - logging.error(error) - raise BMCException(error) - - bmc_mod_name = 'remoteinstaller.installer.bmc_management.{}'.format(hw_data['product_family'].lower()) - bmc_mod = importlib.import_module(bmc_mod_name) - bmc_class = getattr(bmc_mod, hw_data['product_family']) - bmc = bmc_class(host, user, passwd, bmc_log_path) - bmc.set_host_name(hw) - - bmc.setup_sol() - - if hw != first_controller: - other_bmcs.append(bmc) - bmc.power('off') - else: - self._first_controller_bmc = bmc - - logging.debug('First controller: %s', first_controller) - - if not self._disable_bmc_initial_reset: - self._first_controller_bmc.reset() - time_after_reset = int(time.time()) - - if not self._disable_other_bmc_reset: - for bmc in other_bmcs: - bmc.reset() - - if not self._disable_bmc_initial_reset: - # Make sure we sleep at least 6min after the first controller BMC reset - sleep_time = 6*60-int(time.time())-time_after_reset - if sleep_time > 0: - logging.debug('Waiting for first controller BMC to stabilize \ - (%s sec) after reset', sleep_time) - time.sleep(sleep_time) - - config_file_names = self._create_cloud_config(first_controller) + self._set_progress('Setup BMCs') + self._setup_bmcs() + + self._set_progress('Create config files') + network_config_filename = self._create_cloud_config() + callback_url_filename = self._create_callback_file() + + patch_files = [self._yaml_path, + network_config_filename, + callback_url_filename] + if self._client_cert: + patch_files.append(self._client_cert) + if self._client_key: + patch_files.append(self._client_key) + if self._ca_cert: + patch_files.append(self._ca_cert) + + self._set_progress('Setup boot options for virtual media') self._first_controller_bmc.setup_boot_options_for_virtual_media() - self._attach_iso_as_virtual_media(config_file_names) + self._set_progress('Attach iso as virtual media') + self._attach_iso_as_virtual_media(patch_files) + self._set_progress('Boot from virtual media') self._first_controller_bmc.boot_from_virtual_media() + self._set_progress('Wait for bootup') self._first_controller_bmc.wait_for_bootup() self._first_controller_bmc.close() - - access_info = {'vip': self._vip, - 'installer_node_ip': self._first_controller_ip, - 'admin_user': self._uc['users']['admin_user_name']} - - return access_info except BMCException as ex: logging.error('Installation failed: %s', str(ex)) raise InstallException(str(ex)) @@ -356,7 +403,7 @@ def main(): help='URL to ISO image') parser.add_argument('-d', '--debug', action='store_true', required=False, help='Debug level for logging') - parser.add_argument('-l', '--logdir', required=False, + parser.add_argument('-l', '--logdir', required=True, help='Directory path for log files') parser.add_argument('-c', '--callback-url', required=True, help='Callback URL for progress reporting') @@ -381,7 +428,7 @@ def main(): logging.basicConfig(stream=sys.stdout, level=log_level) logging.debug('args: %s', parsed_args) - installer = Installer(parsed_args) + installer = Installer(parsed_args.yaml, parsed_args.logdir, parsed_args) installer.install() diff --git a/src/remoteinstaller/server/server.py b/src/remoteinstaller/server/server.py index b925dd4..60ad195 100644 --- a/src/remoteinstaller/server/server.py +++ b/src/remoteinstaller/server/server.py @@ -36,31 +36,32 @@ class LoggingSSLSocket(ssl.SSLSocket): try: result = super(LoggingSSLSocket, self).accept(*args, **kwargs) except Exception as ex: - logging.warn('SSLSocket.accept raised exception: %s', str(ex)) + logging.warning('SSLSocket.accept raised exception: %s', str(ex)) raise return result class InstallationWorker(Thread): - def __init__(self, server, uuid, admin_passwd, logdir, args=None): + def __init__(self, server, uuid, admin_passwd, yaml, logdir, args=None): super(InstallationWorker, self).__init__(name=uuid) self._server = server self._uuid = uuid self._admin_passwd = admin_passwd + self._yaml = yaml self._logdir = logdir self._args = args def run(self): - access_info = None + installer = Installer(self._server, self._uuid, self._yaml, self._logdir, self._args) + access_info = installer.get_access_info() + if self._args: try: - installer = Installer(self._args) - #access_info = installer.install() - + installer.install() logging.info('Installation triggered for %s', self._uuid) except InstallException as ex: - logging.warn('Installation triggering failed for %s: %s', self._uuid, str(ex)) - self._server.set_state(self._uuid, 'failed', str(ex), 0) + logging.warning('Installation triggering failed for %s: %s', self._uuid, str(ex)) + self._server.set_state(self._uuid, 'failed', str(ex)) return installation_finished = False @@ -69,15 +70,18 @@ class InstallationWorker(Thread): if not state['status'] == 'ongoing': installation_finished = True else: + logging.info('Installation of %s still ongoing (%s%%): %s', + self._uuid, + state['percentage'], + state['description']) time.sleep(10) logging.info('Installation finished for %s: %s', self._uuid, state) - if access_info: - logging.info('Login details for installation %s: %s', self._uuid, str(access_info)) + logging.info('Login details for installation %s: %s', self._uuid, str(access_info)) - logging.info('Getting logs for installation %s...', uuid) - #installer.get_logs(self._logdir, self._admin_passwd) - logging.info('Logs retrieved for %s', uuid) + logging.info('Getting logs for installation %s...', self._uuid) + installer.get_logs(self._admin_passwd) + logging.info('Logs retrieved for %s', self._uuid) class Server(object): DEFAULT_PATH = '/opt/remoteinstaller' @@ -85,10 +89,18 @@ class Server(object): ISO_PATH = 'images' CERTIFICATE_PATH = 'certificates' INSTALLATIONS_PATH = 'installations' - #CLOUD_ISO_PATH = '{}/rec.iso'.format(ISO_PATH) - BOOT_ISO_PATH = '{}/boot.iso'.format(ISO_PATH) - - def __init__(self, host, port, cert=None, key=None, client_cert=None, client_key=None, ca_cert=None, path=None, http_port=None): + USER_CONFIG_NAME = 'user_config.yaml' + + def __init__(self, + host, + port, + cert=None, + key=None, + client_cert=None, + client_key=None, + ca_cert=None, + path=None, + http_port=None): self._host = host self._port = port self._http_port = http_port @@ -117,6 +129,16 @@ class Server(object): return admin_passwd + def _get_yaml_path_for_cloud(self, cloud_name): + yaml = '{}/{}/{}/{}'.format(self._path, + Server.USER_CONFIG_PATH, + cloud_name, + Server.USER_CONFIG_NAME) + if not os.path.isfile(yaml): + raise ServerError('YAML file {} not found'.format(yaml)) + + return yaml + def _load_states(self): uuid_list = os.listdir('{}/{}'.format(self._path, Server.INSTALLATIONS_PATH)) for uuid in uuid_list: @@ -130,14 +152,17 @@ class Server(object): logdir = '{}/{}/{}'.format(self._path, Server.INSTALLATIONS_PATH, uuid) cloud_name = self._ongoing_installations[uuid]['cloud_name'] admin_passwd = self._read_admin_passwd(cloud_name) - worker = InstallationWorker(self, uuid, admin_passwd, logdir) + yaml = self._get_yaml_path_for_cloud(cloud_name) + worker = InstallationWorker(self, uuid, admin_passwd, yaml, logdir) worker.start() - def _set_state(self, uuid, status, description, percentage, cloud_name=None): - self._ongoing_installations[uuid] = {} + def _set_state(self, uuid, status, description, percentage=None, cloud_name=None): + if not self._ongoing_installations.get(uuid, None): + self._ongoing_installations[uuid] = {} self._ongoing_installations[uuid]['status'] = status self._ongoing_installations[uuid]['description'] = description - self._ongoing_installations[uuid]['percentage'] = percentage + if percentage is not None: + self._ongoing_installations[uuid]['percentage'] = percentage if cloud_name: self._ongoing_installations[uuid]['cloud_name'] = cloud_name @@ -145,9 +170,9 @@ class Server(object): with open(state_file, 'w') as sf: sf.write(json.dumps(self._ongoing_installations[uuid])) - def set_state(self, uuid, status, description, percentage): - logging.info('uuid=%s, status=%s, description=%s, percentage=%s', - uuid, status, description, percentage) + def set_state(self, uuid, status, description, percentage=None): + logging.debug('set_state called for %s: status=%s, description=%s, percentage=%s', + uuid, status, description, percentage) if not uuid in self._ongoing_installations: raise ServerError('Installation id {} not found'.format(uuid)) @@ -158,7 +183,7 @@ class Server(object): self._set_state(uuid, status, description, percentage) def get_state(self, uuid): - logging.info('uuid=%s', uuid) + logging.debug('get_state called for %s', uuid) if not uuid in self._ongoing_installations: raise ServerError('Installation id {} not found'.format(uuid)) @@ -167,23 +192,23 @@ class Server(object): 'description': self._ongoing_installations[uuid]['description'], 'percentage': self._ongoing_installations[uuid]['percentage']} - def start_installation(self, cloud_name, iso): - logging.info('start_installation(%s, %s)', cloud_name, iso) + def start_installation(self, cloud_name, iso, boot_iso): + logging.debug('start_installation called with args: (%s, %s, %s)', cloud_name, iso, boot_iso) uuid = str(uuid_module.uuid4()) args = argparse.Namespace() - args.yaml = '{}/{}/{}/user_config.yml'.format(self._path, - Server.USER_CONFIG_PATH, - cloud_name) - if not os.path.isfile(args.yaml): - raise ServerError('YAML file {} not found'.format(args.yaml)) + args.yaml = self._get_yaml_path_for_cloud(cloud_name) iso_path = '{}/{}/{}'.format(self._path, Server.ISO_PATH, iso) if not os.path.isfile(iso_path): raise ServerError('ISO file {} not found'.format(iso_path)) + boot_iso_path = '{}/{}/{}'.format(self._path, Server.ISO_PATH, boot_iso) + if not os.path.isfile(boot_iso_path): + raise ServerError('Provisioning ISO file {} not found'.format(boot_iso_path)) + http_port_part = '' if self._http_port: http_port_part = ':{}'.format(self._http_port) @@ -194,12 +219,12 @@ class Server(object): os.makedirs(args.logdir) - args.boot_iso = '{}/{}'.format(self._path, Server.BOOT_ISO_PATH) + args.boot_iso = '{}/{}/{}'.format(self._path, Server.ISO_PATH, boot_iso) args.tag = uuid - args.callback_url = 'http://{}:{}/v1/installations/{}/state'.format(self._host, - self._port, - uuid) + args.callback_url = 'https://{}:{}/v1/installations/{}/state'.format(self._host, + self._port, + uuid) args.client_cert = self._client_cert args.client_key = self._client_key @@ -209,7 +234,7 @@ class Server(object): self._set_state(uuid, 'ongoing', '', 0, cloud_name) admin_passwd = self._read_admin_passwd(cloud_name) - worker = InstallationWorker(self, uuid, admin_passwd, args.logdir, args) + worker = InstallationWorker(self, uuid, admin_passwd, args.yaml, args.logdir, args) worker.start() return uuid @@ -336,6 +361,7 @@ class WSGIHandler(object): { 'cloud-name': , 'iso': , + 'provisioning-iso': } Response: http status set correctly { @@ -351,14 +377,14 @@ class WSGIHandler(object): request = json.loads(rpc.req_body) cloud_name = request['cloud-name'] iso = request['iso'] + boot_iso = request['provisioning-iso'] - uuid = self.server.start_installation(cloud_name, iso) + uuid = self.server.start_installation(cloud_name, iso, boot_iso) rpc.rep_status = HTTPErrors.get_ok_status() reply = {'uuid': uuid} rpc.rep_body = json.dumps(reply) except KeyError as ex: - rpc.rep_status = HTTPErrors.get_request_not_ok_status() raise ServerError('Missing request parameter: {}'.format(str(ex))) except Exception as exp: # pylint: disable=broad-except rpc.rep_status = HTTPErrors.get_internal_error_status() @@ -380,17 +406,13 @@ class WSGIHandler(object): logging.debug('_get_state called') try: - if not rpc.req_body: - rpc.rep_status = HTTPErrors.get_request_not_ok_status() - else: - uuid = rpc.req_params['uuid'] + uuid = rpc.req_params['uuid'] - reply = self.server.get_state(uuid) + reply = self.server.get_state(uuid) - rpc.rep_status = HTTPErrors.get_ok_status() - rpc.rep_body = json.dumps(reply) + rpc.rep_status = HTTPErrors.get_ok_status() + rpc.rep_body = json.dumps(reply) except KeyError as ex: - rpc.rep_status = HTTPErrors.get_request_not_ok_status() raise ServerError('Missing request parameter: {}'.format(str(ex))) except Exception as exp: # pylint: disable=broad-except rpc.rep_status = HTTPErrors.get_internal_error_status() @@ -410,7 +432,7 @@ class WSGIHandler(object): } """ - logging.debug('set_state called') + logging.debug('_set_state called') try: if not rpc.req_body: rpc.rep_status = HTTPErrors.get_request_not_ok_status() @@ -426,8 +448,6 @@ class WSGIHandler(object): rpc.rep_status = HTTPErrors.get_ok_status() reply = {} rpc.rep_body = json.dumps(reply) - except ServerError: - raise except KeyError as ex: rpc.rep_status = HTTPErrors.get_request_not_ok_status() raise ServerError('Missing request parameter: {}'.format(str(ex))) @@ -445,7 +465,11 @@ class WSGIHandler(object): rpc.req_filter = {} rpc.req_content_type = environ['CONTENT_TYPE'] try: - rpc.req_content_size = int(environ['CONTENT_LENGTH']) + content_len = environ['CONTENT_LENGTH'] + if not content_len: + rpc.req_content_size = 0 + else: + rpc.req_content_size = int(content_len) except KeyError: rpc.req_content_size = 0 @@ -497,12 +521,12 @@ class WSGIHandler(object): self._read_body(rpc, environ) - logging.info('Calling %s with rpc=%s', action, str(rpc)) + logging.debug('Calling %s with rpc=%s', action, str(rpc)) actionfunc = getattr(self, action) actionfunc(rpc) except ServerError as ex: rpc.rep_status = HTTPErrors.get_request_not_ok_status() - rpc.rep_status += ',' + rpc.rep_status += ', ' rpc.rep_status += str(ex) except AttributeError: rpc.rep_status = HTTPErrors.get_internal_error_status() @@ -510,10 +534,10 @@ class WSGIHandler(object): rpc.rep_status += 'Missing action function' except Exception as exp: # pylint: disable=broad-except rpc.rep_status = HTTPErrors.get_internal_error_status() - rpc.rep_status += ',' + rpc.rep_status += ', ' rpc.rep_status += str(exp) - logging.info('Replying with rpc=%s', str(rpc)) + logging.debug('Replying with rpc=%s', str(rpc)) response_headers = [('Content-type', 'application/json')] start_response(rpc.rep_status, response_headers) return [rpc.rep_body] @@ -555,8 +579,8 @@ def main(): else: log_level = logging.INFO - format = '%(asctime)s %(threadName)s:%(levelname)s %(message)s' - logging.basicConfig(stream=sys.stdout, level=log_level, format=format) + logformat = '%(asctime)s %(threadName)s:%(levelname)s %(message)s' + logging.basicConfig(stream=sys.stdout, level=log_level, format=logformat) logging.debug('args: %s', args) diff --git a/src/scripts/get_journals.sh b/src/scripts/get_journals.sh new file mode 100644 index 0000000..343a638 --- /dev/null +++ b/src/scripts/get_journals.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Copyright 2019 Nokia + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +hosts=$(python print_hosts.py) +for host in $hosts; do + if [ "${host}" != "$(hostname)" ]; then + ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=60 ${host} "sudo journalctl" > /tmp/journal_${host}.log + ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=60 ${host} "sudo journalctl -o json" > /tmp/journal_${host}_json.log + else + sudo journalctl > /tmp/journal_${host}.log + sudo journalctl -o json > /tmp/journal_${host}_json.log + fi +done + +cd /tmp +tar czf node_journals.tgz journal_*.log +rm -f journal_*.log +cd - diff --git a/src/scripts/print_hosts.py b/src/scripts/print_hosts.py new file mode 100644 index 0000000..66b2137 --- /dev/null +++ b/src/scripts/print_hosts.py @@ -0,0 +1,20 @@ +# Copyright 2019 Nokia + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml + +with open('/etc/userconfig/user_config.yaml') as f: + y = yaml.load(f) +for host in y['hosts'].keys(): + print host -- 2.16.6