60ad195c66e6b94e6de8e3730d85eeefd2b3383f
[ta/remote-installer.git] / src / remoteinstaller / server / server.py
1 # Copyright 2019 Nokia
2
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14
15 import sys
16 import argparse
17 import logging
18 import os
19 from threading import Thread
20 import time
21 import json
22 import urllib
23 import urlparse
24 import uuid as uuid_module
25 import ssl
26
27 from wsgiref.simple_server import make_server
28 import routes
29
30 from remoteinstaller.installer.install import Installer
31 from remoteinstaller.installer.install import InstallException
32
33
34 class LoggingSSLSocket(ssl.SSLSocket):
35     def accept(self, *args, **kwargs):
36         try:
37             result = super(LoggingSSLSocket, self).accept(*args, **kwargs)
38         except Exception as ex:
39             logging.warning('SSLSocket.accept raised exception: %s', str(ex))
40             raise
41         return result
42
43
44 class InstallationWorker(Thread):
45     def __init__(self, server, uuid, admin_passwd, yaml, logdir, args=None):
46         super(InstallationWorker, self).__init__(name=uuid)
47         self._server = server
48         self._uuid = uuid
49         self._admin_passwd = admin_passwd
50         self._yaml = yaml
51         self._logdir = logdir
52         self._args = args
53
54     def run(self):
55         installer = Installer(self._server, self._uuid, self._yaml, self._logdir, self._args)
56         access_info = installer.get_access_info()
57
58         if self._args:
59             try:
60                 installer.install()
61                 logging.info('Installation triggered for %s', self._uuid)
62             except InstallException as ex:
63                 logging.warning('Installation triggering failed for %s: %s', self._uuid, str(ex))
64                 self._server.set_state(self._uuid, 'failed', str(ex))
65                 return
66
67         installation_finished = False
68         while not installation_finished:
69             state = self._server.get_state(self._uuid)
70             if not state['status'] == 'ongoing':
71                 installation_finished = True
72             else:
73                 logging.info('Installation of %s still ongoing (%s%%): %s',
74                              self._uuid,
75                              state['percentage'],
76                              state['description'])
77                 time.sleep(10)
78
79         logging.info('Installation finished for %s: %s', self._uuid, state)
80         logging.info('Login details for installation %s: %s', self._uuid, str(access_info))
81
82         logging.info('Getting logs for installation %s...', self._uuid)
83         installer.get_logs(self._admin_passwd)
84         logging.info('Logs retrieved for %s', self._uuid)
85
86 class Server(object):
87     DEFAULT_PATH = '/opt/remoteinstaller'
88     USER_CONFIG_PATH = 'user-configs'
89     ISO_PATH = 'images'
90     CERTIFICATE_PATH = 'certificates'
91     INSTALLATIONS_PATH = 'installations'
92     USER_CONFIG_NAME = 'user_config.yaml'
93
94     def __init__(self,
95                  host,
96                  port,
97                  cert=None,
98                  key=None,
99                  client_cert=None,
100                  client_key=None,
101                  ca_cert=None,
102                  path=None,
103                  http_port=None):
104         self._host = host
105         self._port = port
106         self._http_port = http_port
107
108         self._path = path
109         if not self._path:
110             self._path = Server.DEFAULT_PATH
111
112         self._cert = '{}/{}/{}'.format(self._path, Server.CERTIFICATE_PATH, cert)
113         self._key = '{}/{}/{}'.format(self._path, Server.CERTIFICATE_PATH, key)
114         self._client_cert = '{}/{}/{}'.format(self._path, Server.CERTIFICATE_PATH, client_cert)
115         self._client_key = '{}/{}/{}'.format(self._path, Server.CERTIFICATE_PATH, client_key)
116         self._ca_cert = '{}/{}/{}'.format(self._path, Server.CERTIFICATE_PATH, ca_cert)
117
118         self._ongoing_installations = {}
119         self._load_states()
120
121     def get_server_keys(self):
122         return {'cert': self._cert, 'key': self._key, 'ca_cert': self._ca_cert}
123
124     def _read_admin_passwd(self, cloud_name):
125         with open('{}/{}/{}/admin_passwd'.format(self._path,
126                                                  Server.USER_CONFIG_PATH,
127                                                  cloud_name)) as pwf:
128             admin_passwd = pwf.readline()
129
130         return admin_passwd
131
132     def _get_yaml_path_for_cloud(self, cloud_name):
133         yaml = '{}/{}/{}/{}'.format(self._path,
134                                     Server.USER_CONFIG_PATH,
135                                     cloud_name,
136                                     Server.USER_CONFIG_NAME)
137         if not os.path.isfile(yaml):
138             raise ServerError('YAML file {} not found'.format(yaml))
139
140         return yaml
141
142     def _load_states(self):
143         uuid_list = os.listdir('{}/{}'.format(self._path, Server.INSTALLATIONS_PATH))
144         for uuid in uuid_list:
145             state_file_name = '{}/{}/{}.state'.format(self._path, Server.INSTALLATIONS_PATH, uuid)
146             if os.path.exists(state_file_name):
147                 with open(state_file_name) as sf:
148                     state_json = sf.readline()
149                     self._ongoing_installations[uuid] = json.loads(state_json)
150
151                 if self._ongoing_installations[uuid]['status'] == 'ongoing':
152                     logdir = '{}/{}/{}'.format(self._path, Server.INSTALLATIONS_PATH, uuid)
153                     cloud_name = self._ongoing_installations[uuid]['cloud_name']
154                     admin_passwd = self._read_admin_passwd(cloud_name)
155                     yaml = self._get_yaml_path_for_cloud(cloud_name)
156                     worker = InstallationWorker(self, uuid, admin_passwd, yaml, logdir)
157                     worker.start()
158
159     def _set_state(self, uuid, status, description, percentage=None, cloud_name=None):
160         if not self._ongoing_installations.get(uuid, None):
161             self._ongoing_installations[uuid] = {}
162         self._ongoing_installations[uuid]['status'] = status
163         self._ongoing_installations[uuid]['description'] = description
164         if percentage is not None:
165             self._ongoing_installations[uuid]['percentage'] = percentage
166         if cloud_name:
167             self._ongoing_installations[uuid]['cloud_name'] = cloud_name
168
169         state_file = '{}/{}/{}.state'.format(self._path, Server.INSTALLATIONS_PATH, uuid)
170         with open(state_file, 'w') as sf:
171             sf.write(json.dumps(self._ongoing_installations[uuid]))
172
173     def set_state(self, uuid, status, description, percentage=None):
174         logging.debug('set_state called for %s: status=%s, description=%s, percentage=%s',
175                       uuid, status, description, percentage)
176
177         if not uuid in self._ongoing_installations:
178             raise ServerError('Installation id {} not found'.format(uuid))
179
180         if not status in ['ongoing', 'failed', 'completed']:
181             raise ServerError('Invalid state: {}'.format(status))
182
183         self._set_state(uuid, status, description, percentage)
184
185     def get_state(self, uuid):
186         logging.debug('get_state called for %s', uuid)
187
188         if not uuid in self._ongoing_installations:
189             raise ServerError('Installation id {} not found'.format(uuid))
190
191         return {'status': self._ongoing_installations[uuid]['status'],
192                 'description': self._ongoing_installations[uuid]['description'],
193                 'percentage': self._ongoing_installations[uuid]['percentage']}
194
195     def start_installation(self, cloud_name, iso, boot_iso):
196         logging.debug('start_installation called with args: (%s, %s, %s)', cloud_name, iso, boot_iso)
197
198         uuid = str(uuid_module.uuid4())
199
200         args = argparse.Namespace()
201
202         args.yaml = self._get_yaml_path_for_cloud(cloud_name)
203
204         iso_path = '{}/{}/{}'.format(self._path, Server.ISO_PATH, iso)
205         if not os.path.isfile(iso_path):
206             raise ServerError('ISO file {} not found'.format(iso_path))
207
208         boot_iso_path = '{}/{}/{}'.format(self._path, Server.ISO_PATH, boot_iso)
209         if not os.path.isfile(boot_iso_path):
210             raise ServerError('Provisioning ISO file {} not found'.format(boot_iso_path))
211
212         http_port_part = ''
213         if self._http_port:
214             http_port_part = ':{}'.format(self._http_port)
215
216         args.iso = 'https://{}{}/{}/{}'.format(self._host, http_port_part, Server.ISO_PATH, iso)
217
218         args.logdir = '{}/{}/{}'.format(self._path, Server.INSTALLATIONS_PATH, uuid)
219
220         os.makedirs(args.logdir)
221
222         args.boot_iso = '{}/{}/{}'.format(self._path, Server.ISO_PATH, boot_iso)
223
224         args.tag = uuid
225         args.callback_url = 'https://{}:{}/v1/installations/{}/state'.format(self._host,
226                                                                              self._port,
227                                                                              uuid)
228
229         args.client_cert = self._client_cert
230         args.client_key = self._client_key
231         args.ca_cert = self._ca_cert
232         args.host_ip = self._host
233
234         self._set_state(uuid, 'ongoing', '', 0, cloud_name)
235
236         admin_passwd = self._read_admin_passwd(cloud_name)
237         worker = InstallationWorker(self, uuid, admin_passwd, args.yaml, args.logdir, args)
238         worker.start()
239
240         return uuid
241
242
243 class ServerError(Exception):
244     pass
245
246
247 class HTTPErrors(object):
248     # response for a successful GET, PUT, PATCH, DELETE,
249     # can also be used for POST that does not result in creation.
250     HTTP_OK = 200
251     # response to a POST which results in creation.
252     HTTP_CREATED = 201
253     # response to a successfull request that won't be returning any body like a DELETE request
254     HTTP_NO_CONTENT = 204
255     # used when http caching headers are in play
256     HTTP_NOT_MODIFIED = 304
257     # the request is malformed such as if the body does not parse
258     HTTP_BAD_REQUEST = 400
259     # when no or invalid authentication details are provided.
260     # also useful to trigger an auth popup API is used from a browser
261     HTTP_UNAUTHORIZED_OPERATION = 401
262     # when authentication succeeded but authenticated user doesn't have access to the resource
263     HTTP_FORBIDDEN = 403
264     # when a non-existent resource is requested
265     HTTP_NOT_FOUND = 404
266     # when an http method is being requested that isn't allowed for the authenticated user
267     HTTP_METHOD_NOT_ALLOWED = 405
268     # indicates the resource at this point is no longer available
269     HTTP_GONE = 410
270     # if incorrect content type was provided as part of the request
271     HTTP_UNSUPPORTED_MEDIA_TYPE = 415
272     # used for validation errors
273     HTTP_UNPROCESSABLE_ENTITY = 422
274     # when request is rejected due to rate limiting
275     HTTP_TOO_MANY_REQUESTS = 429
276     # Other errrors
277     HTTP_INTERNAL_ERROR = 500
278
279     @staticmethod
280     def get_ok_status():
281         return '%d OK' % HTTPErrors.HTTP_OK
282
283     @staticmethod
284     def get_object_created_successfully_status():
285         return '%d Created' % HTTPErrors.HTTP_CREATED
286
287     @staticmethod
288     def get_request_not_ok_status():
289         return '%d Bad request' % HTTPErrors.HTTP_BAD_REQUEST
290
291     @staticmethod
292     def get_resource_not_found_status():
293         return '%d Not found' % HTTPErrors.HTTP_NOT_FOUND
294
295     @staticmethod
296     def get_unsupported_content_type_status():
297         return '%d Unsupported content type' % HTTPErrors.HTTP_UNSUPPORTED_MEDIA_TYPE
298
299     @staticmethod
300     def get_validation_error_status():
301         return '%d Validation error' % HTTPErrors.HTTP_UNPROCESSABLE_ENTITY
302
303     @staticmethod
304     def get_internal_error_status():
305         return '%d Internal error' % HTTPErrors.HTTP_INTERNAL_ERROR
306
307
308 class HTTPRPC(object):
309     def __init__(self):
310         self.req_body = ''
311         self.req_filter = ''
312         self.req_params = {}
313         self.req_method = ''
314         self.req_content_type = ''
315         self.req_content_size = 0
316         self.req_path = ''
317
318         self.rep_body = ''
319         self.rep_status = ''
320
321     def __str__(self):
322         return str.format('REQ: body:{body} filter:{filter} '
323                           'params:{params} method:{method} path:{path} '
324                           'content_type:{content_type} content_size:{content_size} '
325                           'REP: body:{rep_body} status:{status}',
326                           body=self.req_body, filter=self.req_filter,
327                           params=str(self.req_params), method=self.req_method, path=self.req_path,
328                           content_type=self.req_content_type, content_size=self.req_content_size,
329                           rep_body=self.rep_body, status=self.rep_status)
330
331 class WSGIHandler(object):
332     def __init__(self, server):
333         logging.debug('WSGIHandler constructor called')
334
335         self.server = server
336
337         self.mapper = routes.Mapper()
338         self.mapper.connect(None, '/apis', action='get_apis')
339         self.mapper.connect(None, '/{api}/installations', action='handle_installations')
340         self.mapper.connect(None, '/{api}/installations/{uuid}/state', action='handle_state')
341
342     def handle_installations(self, rpc):
343         if rpc.req_method == 'POST':
344             self._start_installation(rpc)
345         else:
346             rpc.rep_status = HTTPErrors.get_request_not_ok_status()
347             rpc.rep_status += ', only POST are possible to this resource'
348
349     def handle_state(self, rpc):
350         if rpc.req_method == 'GET':
351             self._get_state(rpc)
352         elif rpc.req_method == 'POST':
353             self._set_state(rpc)
354         else:
355             rpc.rep_status = HTTPErrors.get_request_not_ok_status()
356             rpc.rep_status += ', only GET/POST are possible to this resource'
357
358     def _start_installation(self, rpc):
359         """
360             Request: POST http://<ip:port>/v1/installations
361                 {
362                     'cloud-name': <name of the cloud>,
363                     'iso': <iso image name>,
364                     'provisioning-iso': <boot iso image name>
365                 }
366             Response: http status set correctly
367                 {
368                     'uuid': <operation identifier>
369                 }
370         """
371
372         logging.debug('_start_installation called')
373         try:
374             if not rpc.req_body:
375                 rpc.rep_status = HTTPErrors.get_request_not_ok_status()
376             else:
377                 request = json.loads(rpc.req_body)
378                 cloud_name = request['cloud-name']
379                 iso = request['iso']
380                 boot_iso = request['provisioning-iso']
381
382                 uuid = self.server.start_installation(cloud_name, iso, boot_iso)
383
384                 rpc.rep_status = HTTPErrors.get_ok_status()
385                 reply = {'uuid': uuid}
386                 rpc.rep_body = json.dumps(reply)
387         except KeyError as ex:
388             raise ServerError('Missing request parameter: {}'.format(str(ex)))
389         except Exception as exp:  # pylint: disable=broad-except
390             rpc.rep_status = HTTPErrors.get_internal_error_status()
391             rpc.rep_status += ','
392             rpc.rep_status += str(exp)
393
394     def _get_state(self, rpc):
395         """
396             Request: GET http://<ip:port>/v1/installations/<uuid>/state
397                 {
398                 }
399             Response: http status set correctly
400                 {
401                     'status': <ongoing|completed|failed>,
402                     'description': <description about the progress>,
403                     'percentage': <percentage completed of the installation>
404                 }
405         """
406
407         logging.debug('_get_state called')
408         try:
409             uuid = rpc.req_params['uuid']
410
411             reply = self.server.get_state(uuid)
412
413             rpc.rep_status = HTTPErrors.get_ok_status()
414             rpc.rep_body = json.dumps(reply)
415         except KeyError as ex:
416             raise ServerError('Missing request parameter: {}'.format(str(ex)))
417         except Exception as exp:  # pylint: disable=broad-except
418             rpc.rep_status = HTTPErrors.get_internal_error_status()
419             rpc.rep_status += ','
420             rpc.rep_status += str(exp)
421
422     def _set_state(self, rpc):
423         """
424             Request: POST http://<ip:port>/v1/installations/<uuid>/state
425                 {
426                     'status': <ongoing|completed|failed>,
427                     'description': <description about the progress>,
428                     'percentage': <percentage completed of the installation>
429                 }
430             Response: http status set correctly
431                 {
432                 }
433         """
434
435         logging.debug('_set_state called')
436         try:
437             if not rpc.req_body:
438                 rpc.rep_status = HTTPErrors.get_request_not_ok_status()
439             else:
440                 request = json.loads(rpc.req_body)
441                 uuid = rpc.req_params['uuid']
442                 status = request['status']
443                 description = request['description']
444                 percentage = request['percentage']
445
446                 self.server.set_state(uuid, status, description, percentage)
447
448                 rpc.rep_status = HTTPErrors.get_ok_status()
449                 reply = {}
450                 rpc.rep_body = json.dumps(reply)
451         except KeyError as ex:
452             rpc.rep_status = HTTPErrors.get_request_not_ok_status()
453             raise ServerError('Missing request parameter: {}'.format(str(ex)))
454         except Exception as exp:  # pylint: disable=broad-except
455             rpc.rep_status = HTTPErrors.get_internal_error_status()
456             rpc.rep_status += ','
457             rpc.rep_status += str(exp)
458
459     def _read_header(self, rpc, environ):
460         rpc.req_method = environ['REQUEST_METHOD']
461         rpc.req_path = environ['PATH_INFO']
462         try:
463             rpc.req_filter = urlparse.parse_qs(urllib.unquote(environ['QUERY_STRING']))
464         except KeyError:
465             rpc.req_filter = {}
466         rpc.req_content_type = environ['CONTENT_TYPE']
467         try:
468             content_len = environ['CONTENT_LENGTH']
469             if not content_len:
470                 rpc.req_content_size = 0
471             else:
472                 rpc.req_content_size = int(content_len)
473         except KeyError:
474             rpc.req_content_size = 0
475
476     def _get_action(self, rpc):
477         # get the action to be done
478         action = ''
479         match_result = self.mapper.match(rpc.req_path)
480         if not match_result:
481             rpc.rep_status = HTTPErrors.get_resource_not_found_status()
482             raise ServerError('URL does not match')
483
484         resultdict = {}
485         if isinstance(match_result, dict):
486             resultdict = match_result
487         else:
488             resultdict = match_result[0]
489
490         try:
491             action = resultdict['action']
492             for key, value in resultdict.iteritems():
493                 if key != 'action':
494                     rpc.req_params[key] = value
495         except KeyError:
496             rpc.rep_status = HTTPErrors.get_internal_error_status()
497             raise ServerError('No action found')
498
499         return action
500
501     def _read_body(self, rpc, environ):
502         # get the body if available
503         if rpc.req_content_size:
504             if rpc.req_content_type == 'application/json':
505                 rpc.req_body = environ['wsgi.input'].read(rpc.req_content_size)
506             else:
507                 rpc.rep_status = HTTPErrors.get_unsupported_content_type_status()
508                 raise ServerError('Content type is not json')
509
510     def __call__(self, environ, start_response):
511         logging.debug('Handling request started, environ=%s', str(environ))
512
513         # For request and resonse data
514         rpc = HTTPRPC()
515         rpc.rep_status = HTTPErrors.get_ok_status()
516
517         try:
518             self._read_header(rpc, environ)
519
520             action = self._get_action(rpc)
521
522             self._read_body(rpc, environ)
523
524             logging.debug('Calling %s with rpc=%s', action, str(rpc))
525             actionfunc = getattr(self, action)
526             actionfunc(rpc)
527         except ServerError as ex:
528             rpc.rep_status = HTTPErrors.get_request_not_ok_status()
529             rpc.rep_status += ', '
530             rpc.rep_status += str(ex)
531         except AttributeError:
532             rpc.rep_status = HTTPErrors.get_internal_error_status()
533             rpc.rep_status += ','
534             rpc.rep_status += 'Missing action function'
535         except Exception as exp:  # pylint: disable=broad-except
536             rpc.rep_status = HTTPErrors.get_internal_error_status()
537             rpc.rep_status += ', '
538             rpc.rep_status += str(exp)
539
540         logging.debug('Replying with rpc=%s', str(rpc))
541         response_headers = [('Content-type', 'application/json')]
542         start_response(rpc.rep_status, response_headers)
543         return [rpc.rep_body]
544
545 def wrap_socket(sock, keyfile=None, certfile=None,
546                 server_side=False, cert_reqs=ssl.CERT_NONE,
547                 ssl_version=ssl.PROTOCOL_SSLv23, ca_certs=None,
548                 do_handshake_on_connect=True,
549                 suppress_ragged_eofs=True,
550                 ciphers=None):
551
552     return LoggingSSLSocket(sock=sock, keyfile=keyfile, certfile=certfile,
553                             server_side=server_side, cert_reqs=cert_reqs,
554                             ssl_version=ssl_version, ca_certs=ca_certs,
555                             do_handshake_on_connect=do_handshake_on_connect,
556                             suppress_ragged_eofs=suppress_ragged_eofs,
557                             ciphers=ciphers)
558
559 def main():
560     parser = argparse.ArgumentParser()
561     parser.add_argument('-H', '--host', required=True, help='binding ip of the server')
562     parser.add_argument('-P', '--listen', required=True, help='binding port of the server')
563     parser.add_argument('-S', '--server', required=False, help='externally visible ip of the server')
564     parser.add_argument('-B', '--port', required=False, help='externally visible port of the server')
565     parser.add_argument('-C', '--cert', required=False, help='server cert file name')
566     parser.add_argument('-K', '--key', required=False, help='server private key file name')
567     parser.add_argument('-c', '--client-cert', required=False, help='client cert file name')
568     parser.add_argument('-k', '--client-key', required=False, help='client key file name')
569     parser.add_argument('-A', '--ca-cert', required=False, help='CA cert file name')
570     parser.add_argument('-p', '--path', required=False, help='path for remote installer files')
571     parser.add_argument('-T', '--http-port', required=False, help='port for HTTPD')
572     parser.add_argument('-d', '--debug', required=False, help='Debug level for logging',
573                         action='store_true')
574
575     args = parser.parse_args()
576
577     if args.debug:
578         log_level = logging.DEBUG
579     else:
580         log_level = logging.INFO
581
582     logformat = '%(asctime)s %(threadName)s:%(levelname)s %(message)s'
583     logging.basicConfig(stream=sys.stdout, level=log_level, format=logformat)
584
585     logging.debug('args: %s', args)
586
587     host = args.server
588     if not host:
589         host = args.host
590
591     port = args.port
592     if not port:
593         port = args.listen
594
595     server = Server(host, port, args.cert, args.key, args.client_cert, args.client_key, args.ca_cert, args.path, args.http_port)
596
597     wsgihandler = WSGIHandler(server)
598
599     wsgi_server = make_server(args.host, int(args.listen), wsgihandler)
600
601     if args.cert:
602         server_keys = server.get_server_keys()
603         wsgi_server.socket = wrap_socket(wsgi_server.socket,
604                                          certfile=server_keys['cert'],
605                                          keyfile=server_keys['key'],
606                                          server_side=True,
607                                          ca_certs=server_keys['ca_cert'],
608                                          cert_reqs=ssl.CERT_REQUIRED)
609
610     wsgi_server.serve_forever()
611
612 if __name__ == "__main__":
613     sys.exit(main())