From: Jyrki Aaltonen Date: Thu, 23 May 2019 09:37:52 +0000 (+0300) Subject: Fixes to server X-Git-Url: https://gerrit.akraino.org/r/gitweb?p=ta%2Fremote-installer.git;a=commitdiff_plain;h=5529dff990973f223eeff6601da4f466fead6a32 Fixes to server Added server logging to file. Extra install options for reseting BMCs before install. Changed installation uuid to be a shorter one. Fix no percentage in set progress call. Get logs correctly from target after successful install. Change-Id: I7f68ab0f3b2a0da04b00a453eac4324460e098e5 Signed-off-by: Jyrki Aaltonen --- diff --git a/docker-build/remote-installer/Dockerfile b/docker-build/remote-installer/Dockerfile index 98f6ceb..3c54056 100644 --- a/docker-build/remote-installer/Dockerfile +++ b/docker-build/remote-installer/Dockerfile @@ -52,7 +52,7 @@ RUN yum -y install systemd epel-release; yum clean all \ && yum install -y iproute wget openssh-server lighttpd nfs-utils \ python-setuptools python2-eventlet python-routes PyYAML \ python-netaddr pexpect net-tools tcpdump \ -ipmitool openssh-clients sshpass nmap-ncat \ +ipmitool openssh-clients sshpass nmap-ncat python-configparser\ # mod_ssl \ && systemctl enable sshd \ && systemctl enable lighttpd \ @@ -95,7 +95,7 @@ RUN pushd "$INSTALLER_MOUNT" \ RUN mkdir -p "$SCRIPTS_DIR" \ && mkdir -p "$ETC_REMOTE_INST" -COPY src/scripts/get_journals.sh src/scripts/print_hosts.py "$SCRIPTS_DIR"/ +COPY src/scripts/get_journals.sh "$SCRIPTS_DIR"/ RUN echo '#!/bin/bash -x' >>$STARTUP \ && echo "function handle_sigterm() {" >>$STARTUP \ diff --git a/src/remoteinstaller/installer/catfile.py b/src/remoteinstaller/installer/catfile.py index a653721..5d4fe3b 100644 --- a/src/remoteinstaller/installer/catfile.py +++ b/src/remoteinstaller/installer/catfile.py @@ -69,7 +69,7 @@ class CatFile(object): logging.debug('Command prompt found') return - except pexpect.exceptions.TIMEOUT as e: + except pexpect.TIMEOUT as e: pass try: @@ -88,7 +88,7 @@ class CatFile(object): self._sol.sendline() self._expect_cmd_prompt() logging.debug('Command prompt found') - except pexpect.exceptions.TIMEOUT as e: + except pexpect.TIMEOUT as e: logging.debug(e) raise diff --git a/src/remoteinstaller/installer/install.py b/src/remoteinstaller/installer/install.py index d453bad..eaa645b 100644 --- a/src/remoteinstaller/installer/install.py +++ b/src/remoteinstaller/installer/install.py @@ -19,6 +19,7 @@ import subprocess import os import importlib import time +import distutils.util from yaml import load from netaddr import IPNetwork @@ -34,9 +35,9 @@ class InstallException(Exception): pass class Installer(object): - SSH_OPTS = '-o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o ServerAliveInterval=60' + SSH_OPTS = ('-o StrictHostKeyChecking=no ' + '-o UserKnownHostsFile=/dev/null ' + '-o ServerAliveInterval=60') def __init__(self, callback_server, callback_uuid, yaml, logdir, args=None): self._callback_server = callback_server @@ -53,13 +54,12 @@ class Installer(object): self._ca_cert = None self._own_ip = None self._tag = None - if args: - self._set_arguments(args) - - # TODO self._disable_bmc_initial_reset = False self._disable_other_bmc_reset = True + if args: + self._set_arguments(args) + self._vip = None self._first_controller = None self._first_controller_ip = None @@ -67,7 +67,25 @@ class Installer(object): self._define_first_controller() + def _get_bool_arg(self, args, arg, default): + if hasattr(args, arg): + arg_value = vars(args)[arg] + if not isinstance(arg_value, bool): + if isinstance(arg_value, basestring): + try: + arg_value = bool(distutils.util.strtobool(arg_value)) + return arg_value + except ValueError: + logging.warning('Invalid value for %s: %s', arg, arg_value) + else: + return arg_value + + return default + def _set_arguments(self, args): + self._disable_bmc_initial_reset = self._get_bool_arg(args, 'disable_bmc_initial_reset', self._disable_bmc_initial_reset) + self._disable_other_bmc_reset = self._get_bool_arg(args, 'disable_other_bmc_reset', self._disable_other_bmc_reset) + self._boot_iso_path = args.boot_iso self._iso_url = args.iso self._callback_url = args.callback_url @@ -259,16 +277,34 @@ class Installer(object): file_name, self._logdir), 'get file') + def _run_node_command(self, ip, user, passwd, command): + self._execute_shell('sshpass -p {} ssh {} {}@{} {}'.format(passwd, + Installer.SSH_OPTS, + user, + ip, + command), 'run command: {}'.format(command)) + def _get_node_logs(self, ip, user, passwd): self._get_file(ip, user, passwd, '/srv/deployment/log/cm.log') self._get_file(ip, user, passwd, '/srv/deployment/log/bootstrap.log') self._get_file(ip, user, passwd, '/var/log/ironic', recursive=True) + def _create_hosts_file(self, file_name): + with open(file_name, 'w') as hosts_file: + for host in self._uc['hosts'].keys(): + hosts_file.write('{}\n'.format(host)) + def _get_journal_logs(self, ip, user, passwd): + hosts_file_name = 'host_names' + hosts_file_path = '{}/{}'.format(self._logdir, hosts_file_name) + self._create_hosts_file(hosts_file_name) + + host_list = ' '.join(self._uc['hosts'].keys()) + + self._put_file(ip, user, passwd, hosts_file_name) self._put_file(ip, user, passwd, '/opt/scripts/get_journals.sh') - self._put_file(ip, user, passwd, '/opt/scripts/print_hosts.py') - self._execute_shell('sh ./get_journals.sh', 'run get_journals.sh') + self._run_node_command(ip, user, passwd, 'sh ./get_journals.sh {}'.format(hosts_file_name)) self._get_file(ip, user, passwd, '/tmp/node_journals.tgz') @@ -388,6 +424,8 @@ class Installer(object): self._set_progress('Wait for bootup') self._first_controller_bmc.wait_for_bootup() + self._set_progress('Wait deployment start') + self._first_controller_bmc.close() except BMCException as ex: logging.error('Installation failed: %s', str(ex)) diff --git a/src/remoteinstaller/server/server.py b/src/remoteinstaller/server/server.py index 60ad195..e693556 100644 --- a/src/remoteinstaller/server/server.py +++ b/src/remoteinstaller/server/server.py @@ -14,7 +14,9 @@ import sys import argparse +from configparser import ConfigParser import logging +from logging.handlers import RotatingFileHandler import os from threading import Thread import time @@ -90,6 +92,7 @@ class Server(object): CERTIFICATE_PATH = 'certificates' INSTALLATIONS_PATH = 'installations' USER_CONFIG_NAME = 'user_config.yaml' + EXTRA_CONFIG_NAME = 'installation.ini' def __init__(self, host, @@ -125,7 +128,7 @@ class Server(object): with open('{}/{}/{}/admin_passwd'.format(self._path, Server.USER_CONFIG_PATH, cloud_name)) as pwf: - admin_passwd = pwf.readline() + admin_passwd = pwf.readline().strip() return admin_passwd @@ -192,13 +195,36 @@ class Server(object): 'description': self._ongoing_installations[uuid]['description'], 'percentage': self._ongoing_installations[uuid]['percentage']} + def _read_extra_args(self, cloud_name): + extra = {} + + extra_config_filename = '{}/{}/{}/{}'.format(self._path, + Server.USER_CONFIG_PATH, + cloud_name, + Server.EXTRA_CONFIG_NAME) + + if os.path.isfile(extra_config_filename): + logging.debug('Read extra installation args from: %s', extra_config_filename) + extra_config = ConfigParser() + with open(extra_config_filename, 'r') as extra_config_file: + extra_config.readfp(extra_config_file) + + if extra_config.has_section('extra'): + for key, value in extra_config.items('extra'): + extra[key] = value + + return extra + def start_installation(self, cloud_name, iso, boot_iso): logging.debug('start_installation called with args: (%s, %s, %s)', cloud_name, iso, boot_iso) - uuid = str(uuid_module.uuid4()) + uuid = str(uuid_module.uuid4())[:8] args = argparse.Namespace() + extra_args = self._read_extra_args(cloud_name) + vars(args).update(extra_args) + args.yaml = self._get_yaml_path_for_cloud(cloud_name) iso_path = '{}/{}/{}'.format(self._path, Server.ISO_PATH, iso) @@ -441,7 +467,7 @@ class WSGIHandler(object): uuid = rpc.req_params['uuid'] status = request['status'] description = request['description'] - percentage = request['percentage'] + percentage = request.get('percentage', None) self.server.set_state(uuid, status, description, percentage) @@ -558,19 +584,23 @@ def wrap_socket(sock, keyfile=None, certfile=None, def main(): parser = argparse.ArgumentParser() - parser.add_argument('-H', '--host', required=True, help='binding ip of the server') - parser.add_argument('-P', '--listen', required=True, help='binding port of the server') - parser.add_argument('-S', '--server', required=False, help='externally visible ip of the server') - parser.add_argument('-B', '--port', required=False, help='externally visible port of the server') - parser.add_argument('-C', '--cert', required=False, help='server cert file name') - parser.add_argument('-K', '--key', required=False, help='server private key file name') - parser.add_argument('-c', '--client-cert', required=False, help='client cert file name') - parser.add_argument('-k', '--client-key', required=False, help='client key file name') - parser.add_argument('-A', '--ca-cert', required=False, help='CA cert file name') - parser.add_argument('-p', '--path', required=False, help='path for remote installer files') - parser.add_argument('-T', '--http-port', required=False, help='port for HTTPD') - parser.add_argument('-d', '--debug', required=False, help='Debug level for logging', + parser = argparse.ArgumentParser() + parser.add_argument('-H', '--host', required=True, metavar='', help='binding ip of the server') + parser.add_argument('-P', '--listen', required=True, metavar='', help='binding port of the server') + parser.add_argument('-S', '--server', required=False, metavar='', help='externally visible ip of the server') + parser.add_argument('-B', '--port', required=False, metavar='', help='externally visible port of the server') + parser.add_argument('-C', '--cert', required=False, metavar='', help='path to server cert file') + parser.add_argument('-K', '--key', required=False, metavar='', help='path to server private key file') + parser.add_argument('-c', '--client-cert', required=False, metavar='', help='path to client cert file') + parser.add_argument('-k', '--client-key', required=False, metavar='', help='path to client key file') + parser.add_argument('-A', '--ca-cert', required=False, metavar='', help='path to CA cert file') + parser.add_argument('-p', '--path', required=False, metavar='', help='path to remote installer files') + parser.add_argument('-T', '--http-port', required=False, metavar='', help='port for HTTPD') + parser.add_argument('-d', '--debug', required=False, help='set debug level for logging', action='store_true') + parser.add_argument('--log-file', required=False, default='/var/log/remote-installer.log', metavar='', help='path to server log file') + parser.add_argument('--log-file-max-size', type=int, default=5, required=False, metavar='', help='server log file max size in MB') + parser.add_argument('--log-file-max-count', type=int, default=10, required=False, metavar='', help='server log file count') args = parser.parse_args() @@ -582,7 +612,15 @@ def main(): logformat = '%(asctime)s %(threadName)s:%(levelname)s %(message)s' logging.basicConfig(stream=sys.stdout, level=log_level, format=logformat) - logging.debug('args: %s', args) + log_file_handler = RotatingFileHandler(args.log_file, + maxBytes=(args.log_file_max_size*1024*1024), + backupCount=args.log_file_max_count) + log_file_handler.setFormatter(logging.Formatter(logformat)) + log_file_handler.setLevel(log_level) + logging.getLogger().addHandler(log_file_handler) + + logging.info('remote-installer started') + logging.debug('remote-installer args: %s', args) host = args.server if not host: diff --git a/src/scripts/get_journals.sh b/src/scripts/get_journals.sh index 343a638..3511eab 100644 --- a/src/scripts/get_journals.sh +++ b/src/scripts/get_journals.sh @@ -14,11 +14,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -hosts=$(python print_hosts.py) -for host in $hosts; do +hosts_names_file="$1" +for host in $(cat ${hosts_names_file}); do if [ "${host}" != "$(hostname)" ]; then - ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=60 ${host} "sudo journalctl" > /tmp/journal_${host}.log - ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=60 ${host} "sudo journalctl -o json" > /tmp/journal_${host}_json.log + address=${host} + if [ -e /etc/infra_internal_addresses ]; then + internal_ip=$(grep ${host} /etc/infra_internal_addresses | cut -d ' ' -f 2) + if [ -n "$internal_ip" ]; then + address=${internal_ip} + fi + fi + ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=60 ${address} "sudo journalctl" > /tmp/journal_${host}.log + ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=60 ${address} "sudo journalctl -o json" > /tmp/journal_${host}_json.log else sudo journalctl > /tmp/journal_${host}.log sudo journalctl -o json > /tmp/journal_${host}_json.log diff --git a/src/scripts/print_hosts.py b/src/scripts/print_hosts.py deleted file mode 100644 index 66b2137..0000000 --- a/src/scripts/print_hosts.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2019 Nokia - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import yaml - -with open('/etc/userconfig/user_config.yaml') as f: - y = yaml.load(f) -for host in y['hosts'].keys(): - print host