Fixes to server

Added server logging to file.
Extra install options for reseting BMCs before install.
Changed installation uuid to be a shorter one.
Fix no percentage in set progress call.
Get logs correctly from target after successful install.

Change-Id: I7f68ab0f3b2a0da04b00a453eac4324460e098e5
Signed-off-by: Jyrki Aaltonen <jyrki.aaltonen@nokia.com>

diff --git a/docker-build/remote-installer/Dockerfile b/docker-build/remote-installer/Dockerfile
index 98f6ceb..3c54056 100644 (file)
--- a/docker-build/remote-installer/Dockerfile
+++ b/docker-build/remote-installer/Dockerfile
@@ -52,7 +52,7 @@ RUN yum -y install systemd epel-release; yum clean all \
 && yum install -y iproute wget openssh-server lighttpd nfs-utils \
 python-setuptools python2-eventlet python-routes PyYAML \
 python-netaddr pexpect net-tools tcpdump \
 && yum install -y iproute wget openssh-server lighttpd nfs-utils \
 python-setuptools python2-eventlet python-routes PyYAML \
 python-netaddr pexpect net-tools tcpdump \

-ipmitool openssh-clients sshpass nmap-ncat \
+ipmitool openssh-clients sshpass nmap-ncat python-configparser\

 # mod_ssl \
 && systemctl enable sshd \
 && systemctl enable lighttpd \
 # mod_ssl \
 && systemctl enable sshd \
 && systemctl enable lighttpd \


@@ -95,7 +95,7 @@ RUN pushd "$INSTALLER_MOUNT" \
 RUN mkdir -p "$SCRIPTS_DIR" \
 && mkdir -p "$ETC_REMOTE_INST"
 
 RUN mkdir -p "$SCRIPTS_DIR" \
 && mkdir -p "$ETC_REMOTE_INST"
 

-COPY src/scripts/get_journals.sh src/scripts/print_hosts.py "$SCRIPTS_DIR"/
+COPY src/scripts/get_journals.sh "$SCRIPTS_DIR"/

 
 RUN echo '#!/bin/bash -x' >>$STARTUP \
 && echo "function handle_sigterm() {" >>$STARTUP \
 
 RUN echo '#!/bin/bash -x' >>$STARTUP \
 && echo "function handle_sigterm() {" >>$STARTUP \

diff --git a/src/remoteinstaller/installer/catfile.py b/src/remoteinstaller/installer/catfile.py
index a653721..5d4fe3b 100644 (file)
--- a/src/remoteinstaller/installer/catfile.py
+++ b/src/remoteinstaller/installer/catfile.py
@@ -69,7 +69,7 @@ class CatFile(object):
             logging.debug('Command prompt found')
 
             return
             logging.debug('Command prompt found')
 
             return

-        except pexpect.exceptions.TIMEOUT as e:
+        except pexpect.TIMEOUT as e:

             pass
 
         try:
             pass
 
         try:


@@ -88,7 +88,7 @@ class CatFile(object):
             self._sol.sendline()
             self._expect_cmd_prompt()
             logging.debug('Command prompt found')
             self._sol.sendline()
             self._expect_cmd_prompt()
             logging.debug('Command prompt found')

-        except pexpect.exceptions.TIMEOUT as e:
+        except pexpect.TIMEOUT as e:

             logging.debug(e)
             raise
 
             logging.debug(e)
             raise
 

diff --git a/src/remoteinstaller/installer/install.py b/src/remoteinstaller/installer/install.py
index d453bad..eaa645b 100644 (file)
--- a/src/remoteinstaller/installer/install.py
+++ b/src/remoteinstaller/installer/install.py
@@ -19,6 +19,7 @@ import subprocess
 import os
 import importlib
 import time
 import os
 import importlib
 import time

+import distutils.util

 
 from yaml import load
 from netaddr import IPNetwork
 
 from yaml import load
 from netaddr import IPNetwork


@@ -34,9 +35,9 @@ class InstallException(Exception):
     pass
 
 class Installer(object):
     pass
 
 class Installer(object):

-    SSH_OPTS = '-o StrictHostKeyChecking=no \
-                -o UserKnownHostsFile=/dev/null \
-                -o ServerAliveInterval=60'
+    SSH_OPTS = ('-o StrictHostKeyChecking=no '
+                '-o UserKnownHostsFile=/dev/null '
+                '-o ServerAliveInterval=60')

 
     def __init__(self, callback_server, callback_uuid, yaml, logdir, args=None):
         self._callback_server = callback_server
 
     def __init__(self, callback_server, callback_uuid, yaml, logdir, args=None):
         self._callback_server = callback_server


@@ -53,13 +54,12 @@ class Installer(object):
         self._ca_cert = None
         self._own_ip = None
         self._tag = None
         self._ca_cert = None
         self._own_ip = None
         self._tag = None

-        if args:
-            self._set_arguments(args)
-
-        # TODO

         self._disable_bmc_initial_reset = False
         self._disable_other_bmc_reset = True
 
         self._disable_bmc_initial_reset = False
         self._disable_other_bmc_reset = True
 

+        if args:
+            self._set_arguments(args)
+

         self._vip = None
         self._first_controller = None
         self._first_controller_ip = None
         self._vip = None
         self._first_controller = None
         self._first_controller_ip = None


@@ -67,7 +67,25 @@ class Installer(object):
 
         self._define_first_controller()
 
 
         self._define_first_controller()
 

+    def _get_bool_arg(self, args, arg, default):
+        if hasattr(args, arg):
+            arg_value = vars(args)[arg]
+            if not isinstance(arg_value, bool):
+                if isinstance(arg_value, basestring):
+                    try:
+                        arg_value = bool(distutils.util.strtobool(arg_value))
+                        return arg_value
+                    except ValueError:
+                        logging.warning('Invalid value for %s: %s', arg, arg_value)
+            else:
+                return arg_value
+
+        return default
+

     def _set_arguments(self, args):
     def _set_arguments(self, args):

+        self._disable_bmc_initial_reset = self._get_bool_arg(args, 'disable_bmc_initial_reset', self._disable_bmc_initial_reset)
+        self._disable_other_bmc_reset = self._get_bool_arg(args, 'disable_other_bmc_reset', self._disable_other_bmc_reset)
+

         self._boot_iso_path = args.boot_iso
         self._iso_url = args.iso
         self._callback_url = args.callback_url
         self._boot_iso_path = args.boot_iso
         self._iso_url = args.iso
         self._callback_url = args.callback_url


@@ -259,16 +277,34 @@ class Installer(object):
                                                                           file_name,
                                                                           self._logdir), 'get file')
 
                                                                           file_name,
                                                                           self._logdir), 'get file')
 

+    def _run_node_command(self, ip, user, passwd, command):
+        self._execute_shell('sshpass -p {} ssh {} {}@{} {}'.format(passwd,
+                                                                   Installer.SSH_OPTS,
+                                                                   user,
+                                                                   ip,
+                                                                   command), 'run command: {}'.format(command))
+

     def _get_node_logs(self, ip, user, passwd):
         self._get_file(ip, user, passwd, '/srv/deployment/log/cm.log')
         self._get_file(ip, user, passwd, '/srv/deployment/log/bootstrap.log')
         self._get_file(ip, user, passwd, '/var/log/ironic', recursive=True)
 
     def _get_node_logs(self, ip, user, passwd):
         self._get_file(ip, user, passwd, '/srv/deployment/log/cm.log')
         self._get_file(ip, user, passwd, '/srv/deployment/log/bootstrap.log')
         self._get_file(ip, user, passwd, '/var/log/ironic', recursive=True)
 

+    def _create_hosts_file(self, file_name):
+        with open(file_name, 'w') as hosts_file:
+            for host in self._uc['hosts'].keys():
+                hosts_file.write('{}\n'.format(host))
+

     def _get_journal_logs(self, ip, user, passwd):
     def _get_journal_logs(self, ip, user, passwd):

+        hosts_file_name = 'host_names'
+        hosts_file_path = '{}/{}'.format(self._logdir, hosts_file_name)
+        self._create_hosts_file(hosts_file_name)
+
+        host_list = ' '.join(self._uc['hosts'].keys())
+
+        self._put_file(ip, user, passwd, hosts_file_name)

         self._put_file(ip, user, passwd, '/opt/scripts/get_journals.sh')
         self._put_file(ip, user, passwd, '/opt/scripts/get_journals.sh')

-        self._put_file(ip, user, passwd, '/opt/scripts/print_hosts.py')

 
 

-        self._execute_shell('sh ./get_journals.sh', 'run get_journals.sh')
+        self._run_node_command(ip, user, passwd, 'sh ./get_journals.sh {}'.format(hosts_file_name))

 
         self._get_file(ip, user, passwd, '/tmp/node_journals.tgz')
 
 
         self._get_file(ip, user, passwd, '/tmp/node_journals.tgz')
 


@@ -388,6 +424,8 @@ class Installer(object):
             self._set_progress('Wait for bootup')
             self._first_controller_bmc.wait_for_bootup()
 
             self._set_progress('Wait for bootup')
             self._first_controller_bmc.wait_for_bootup()
 

+            self._set_progress('Wait deployment start')
+

             self._first_controller_bmc.close()
         except BMCException as ex:
             logging.error('Installation failed: %s', str(ex))
             self._first_controller_bmc.close()
         except BMCException as ex:
             logging.error('Installation failed: %s', str(ex))

diff --git a/src/remoteinstaller/server/server.py b/src/remoteinstaller/server/server.py
index 60ad195..e693556 100644 (file)
--- a/src/remoteinstaller/server/server.py
+++ b/src/remoteinstaller/server/server.py
@@ -14,7 +14,9 @@
 
 import sys
 import argparse
 
 import sys
 import argparse

+from configparser import ConfigParser

 import logging
 import logging

+from logging.handlers import RotatingFileHandler

 import os
 from threading import Thread
 import time
 import os
 from threading import Thread
 import time


@@ -90,6 +92,7 @@ class Server(object):
     CERTIFICATE_PATH = 'certificates'
     INSTALLATIONS_PATH = 'installations'
     USER_CONFIG_NAME = 'user_config.yaml'
     CERTIFICATE_PATH = 'certificates'
     INSTALLATIONS_PATH = 'installations'
     USER_CONFIG_NAME = 'user_config.yaml'

+    EXTRA_CONFIG_NAME = 'installation.ini'

 
     def __init__(self,
                  host,
 
     def __init__(self,
                  host,


@@ -125,7 +128,7 @@ class Server(object):
         with open('{}/{}/{}/admin_passwd'.format(self._path,
                                                  Server.USER_CONFIG_PATH,
                                                  cloud_name)) as pwf:
         with open('{}/{}/{}/admin_passwd'.format(self._path,
                                                  Server.USER_CONFIG_PATH,
                                                  cloud_name)) as pwf:

-            admin_passwd = pwf.readline()
+            admin_passwd = pwf.readline().strip()

 
         return admin_passwd
 
 
         return admin_passwd
 


@@ -192,13 +195,36 @@ class Server(object):
                 'description': self._ongoing_installations[uuid]['description'],
                 'percentage': self._ongoing_installations[uuid]['percentage']}
 
                 'description': self._ongoing_installations[uuid]['description'],
                 'percentage': self._ongoing_installations[uuid]['percentage']}
 

+    def _read_extra_args(self, cloud_name):
+        extra = {}
+
+        extra_config_filename = '{}/{}/{}/{}'.format(self._path,
+                                    Server.USER_CONFIG_PATH,
+                                    cloud_name,
+                                    Server.EXTRA_CONFIG_NAME)
+
+        if os.path.isfile(extra_config_filename):
+            logging.debug('Read extra installation args from: %s', extra_config_filename)
+            extra_config = ConfigParser()
+            with open(extra_config_filename, 'r') as extra_config_file:
+                extra_config.readfp(extra_config_file)
+
+            if extra_config.has_section('extra'):
+                for key, value in extra_config.items('extra'):
+                    extra[key] = value
+
+        return extra
+

     def start_installation(self, cloud_name, iso, boot_iso):
         logging.debug('start_installation called with args: (%s, %s, %s)', cloud_name, iso, boot_iso)
 
     def start_installation(self, cloud_name, iso, boot_iso):
         logging.debug('start_installation called with args: (%s, %s, %s)', cloud_name, iso, boot_iso)
 

-        uuid = str(uuid_module.uuid4())
+        uuid = str(uuid_module.uuid4())[:8]

 
         args = argparse.Namespace()
 
 
         args = argparse.Namespace()
 

+        extra_args = self._read_extra_args(cloud_name)
+        vars(args).update(extra_args)
+

         args.yaml = self._get_yaml_path_for_cloud(cloud_name)
 
         iso_path = '{}/{}/{}'.format(self._path, Server.ISO_PATH, iso)
         args.yaml = self._get_yaml_path_for_cloud(cloud_name)
 
         iso_path = '{}/{}/{}'.format(self._path, Server.ISO_PATH, iso)


@@ -441,7 +467,7 @@ class WSGIHandler(object):
                 uuid = rpc.req_params['uuid']
                 status = request['status']
                 description = request['description']
                 uuid = rpc.req_params['uuid']
                 status = request['status']
                 description = request['description']

-                percentage = request['percentage']
+                percentage = request.get('percentage', None)

 
                 self.server.set_state(uuid, status, description, percentage)
 
 
                 self.server.set_state(uuid, status, description, percentage)
 


@@ -558,19 +584,23 @@ def wrap_socket(sock, keyfile=None, certfile=None,
 
 def main():
     parser = argparse.ArgumentParser()
 
 def main():
     parser = argparse.ArgumentParser()

-    parser.add_argument('-H', '--host', required=True, help='binding ip of the server')
-    parser.add_argument('-P', '--listen', required=True, help='binding port of the server')
-    parser.add_argument('-S', '--server', required=False, help='externally visible ip of the server')
-    parser.add_argument('-B', '--port', required=False, help='externally visible port of the server')
-    parser.add_argument('-C', '--cert', required=False, help='server cert file name')
-    parser.add_argument('-K', '--key', required=False, help='server private key file name')
-    parser.add_argument('-c', '--client-cert', required=False, help='client cert file name')
-    parser.add_argument('-k', '--client-key', required=False, help='client key file name')
-    parser.add_argument('-A', '--ca-cert', required=False, help='CA cert file name')
-    parser.add_argument('-p', '--path', required=False, help='path for remote installer files')
-    parser.add_argument('-T', '--http-port', required=False, help='port for HTTPD')
-    parser.add_argument('-d', '--debug', required=False, help='Debug level for logging',
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-H', '--host', required=True, metavar='<bind ip>', help='binding ip of the server')
+    parser.add_argument('-P', '--listen', required=True, metavar='<bind port>', help='binding port of the server')
+    parser.add_argument('-S', '--server', required=False, metavar='<external ip>', help='externally visible ip of the server')
+    parser.add_argument('-B', '--port', required=False, metavar='<external port>', help='externally visible port of the server')
+    parser.add_argument('-C', '--cert', required=False, metavar='<server cert file>', help='path to server cert file')
+    parser.add_argument('-K', '--key', required=False, metavar='<server key file>', help='path to server private key file')
+    parser.add_argument('-c', '--client-cert', required=False, metavar='<client cert file>', help='path to client cert file')
+    parser.add_argument('-k', '--client-key', required=False, metavar='<client key file>', help='path to client key file')
+    parser.add_argument('-A', '--ca-cert', required=False, metavar='<CA cert file>', help='path to CA cert file')
+    parser.add_argument('-p', '--path', required=False, metavar='<path to installer files>', help='path to remote installer files')
+    parser.add_argument('-T', '--http-port', required=False, metavar='<HTTPD port>', help='port for HTTPD')
+    parser.add_argument('-d', '--debug', required=False, help='set debug level for logging',

                         action='store_true')
                         action='store_true')

+    parser.add_argument('--log-file', required=False, default='/var/log/remote-installer.log', metavar='<server log file>', help='path to server log file')
+    parser.add_argument('--log-file-max-size', type=int, default=5, required=False, metavar='<max size>', help='server log file max size in MB')
+    parser.add_argument('--log-file-max-count', type=int, default=10, required=False, metavar='<max count>', help='server log file count')

 
     args = parser.parse_args()
 
 
     args = parser.parse_args()
 


@@ -582,7 +612,15 @@ def main():
     logformat = '%(asctime)s %(threadName)s:%(levelname)s %(message)s'
     logging.basicConfig(stream=sys.stdout, level=log_level, format=logformat)
 
     logformat = '%(asctime)s %(threadName)s:%(levelname)s %(message)s'
     logging.basicConfig(stream=sys.stdout, level=log_level, format=logformat)
 

-    logging.debug('args: %s', args)
+    log_file_handler = RotatingFileHandler(args.log_file,
+                                           maxBytes=(args.log_file_max_size*1024*1024),
+                                           backupCount=args.log_file_max_count)
+    log_file_handler.setFormatter(logging.Formatter(logformat))
+    log_file_handler.setLevel(log_level)
+    logging.getLogger().addHandler(log_file_handler)
+
+    logging.info('remote-installer started')
+    logging.debug('remote-installer args: %s', args)

 
     host = args.server
     if not host:
 
     host = args.server
     if not host:

diff --git a/src/scripts/get_journals.sh b/src/scripts/get_journals.sh
index 343a638..3511eab 100644 (file)
--- a/src/scripts/get_journals.sh
+++ b/src/scripts/get_journals.sh
@@ -14,11 +14,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 # See the License for the specific language governing permissions and
 # limitations under the License.
 

-hosts=$(python print_hosts.py)
-for host in $hosts; do
+hosts_names_file="$1"
+for host in $(cat ${hosts_names_file}); do

     if [ "${host}" != "$(hostname)" ]; then
     if [ "${host}" != "$(hostname)" ]; then

-        ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=60 ${host} "sudo journalctl" > /tmp/journal_${host}.log
-        ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=60 ${host} "sudo journalctl -o json" > /tmp/journal_${host}_json.log
+        address=${host}
+        if [ -e /etc/infra_internal_addresses ]; then
+            internal_ip=$(grep ${host} /etc/infra_internal_addresses | cut -d ' ' -f 2)
+            if [ -n "$internal_ip" ]; then
+                address=${internal_ip}
+            fi
+        fi
+        ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=60 ${address} "sudo journalctl" > /tmp/journal_${host}.log
+        ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=60 ${address} "sudo journalctl -o json" > /tmp/journal_${host}_json.log

     else
         sudo journalctl > /tmp/journal_${host}.log
         sudo journalctl -o json > /tmp/journal_${host}_json.log
     else
         sudo journalctl > /tmp/journal_${host}.log
         sudo journalctl -o json > /tmp/journal_${host}_json.log

diff --git a/src/scripts/print_hosts.py b/src/scripts/print_hosts.py
deleted file mode 100644 (file)
index 66b2137..0000000
--- a/src/scripts/print_hosts.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# Copyright 2019 Nokia
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import yaml
-
-with open('/etc/userconfig/user_config.yaml') as f:
-    y = yaml.load(f)
-for host in y['hosts'].keys():
-    print host