From 90e74b60a35a20d9a91e09672664022b64b619b9 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Ferenc=20T=C3=B3th?= Date: Wed, 7 Aug 2019 18:01:17 +0200 Subject: [PATCH] Add CaaS Logging REST API and CLI plugins MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Change-Id: I7f22a113bfa215ff27cbdde5404385f170681d42 Signed-off-by: Ferenc Tóth --- SPECS/caas-logging.spec | 59 ++++++ src/__init__.py | 15 ++ src/caas_logging/__init__.py | 15 ++ src/caas_logging/activator/caasactivator.py | 45 ++++ src/caas_logging/cli/__init__.py | 14 ++ src/caas_logging/cli/caas.py | 148 +++++++++++++ src/caas_logging/rest-plugin/__init__.py | 14 ++ src/caas_logging/rest-plugin/caas.ini | 16 ++ src/caas_logging/rest-plugin/caashandler.py | 308 ++++++++++++++++++++++++++++ src/setup.py | 41 ++++ 10 files changed, 675 insertions(+) create mode 100644 SPECS/caas-logging.spec create mode 100644 src/__init__.py create mode 100644 src/caas_logging/__init__.py create mode 100644 src/caas_logging/activator/caasactivator.py create mode 100644 src/caas_logging/cli/__init__.py create mode 100644 src/caas_logging/cli/caas.py create mode 100644 src/caas_logging/rest-plugin/__init__.py create mode 100644 src/caas_logging/rest-plugin/caas.ini create mode 100644 src/caas_logging/rest-plugin/caashandler.py create mode 100644 src/setup.py diff --git a/SPECS/caas-logging.spec b/SPECS/caas-logging.spec new file mode 100644 index 0000000..512800d --- /dev/null +++ b/SPECS/caas-logging.spec @@ -0,0 +1,59 @@ +# Copyright 2019 Nokia + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Name: caas-logging +Version: %{_version} +Release: 1%{?dist} +Summary: CaaS Logging restful API and CLI plugins +License: %{_platform_license} + +Vendor: %{_platform_vendor} +Source0: %{name}-%{version}.tar.gz +BuildArch: noarch +Requires: python-flask, python2-flask-restful, python2-configparser +BuildRequires: python python-setuptools + +%description +This RPM contains CaaS Logging components (restful API and CLI plugins) for Akraino REC + +%prep +%autosetup + +%install +mkdir -p %{buildroot}%{_python_site_packages_path}/caas_logging + +mkdir -p %{buildroot}%{_python_site_packages_path}/yarf/handlers/caas_logging +rsync -ra src/caas_logging/rest-plugin/* %{buildroot}/%{_python_site_packages_path}/yarf/handlers/caas_logging + +mkdir -p %{buildroot}/opt/cmframework/activators/ +rsync -ra src/caas_logging/activator/* %{buildroot}/opt/cmframework/activators + +cd src && python setup.py install --root %{buildroot} --no-compile --install-purelib %{_python_site_packages_path} --install-scripts %{_platform_bin_path} && cd - + +%files +%defattr(0755,root,root) +%{_python_site_packages_path}/caas_logging* +%{_python_site_packages_path}/yarf/handlers/caas_logging* +/opt/cmframework/activators/caasactivator.py* + +%pre + +%post + +%preun + +%postun + +%clean +rm -rf %{buildroot} diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..287b513 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2019 Nokia + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__import__('pkg_resources').declare_namespace(__name__) diff --git a/src/caas_logging/__init__.py b/src/caas_logging/__init__.py new file mode 100644 index 0000000..287b513 --- /dev/null +++ b/src/caas_logging/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2019 Nokia + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__import__('pkg_resources').declare_namespace(__name__) diff --git a/src/caas_logging/activator/caasactivator.py b/src/caas_logging/activator/caasactivator.py new file mode 100644 index 0000000..762ddf3 --- /dev/null +++ b/src/caas_logging/activator/caasactivator.py @@ -0,0 +1,45 @@ +#! /usr/bin/python + +# Copyright 2019 Nokia + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from cmframework.apis import cmactivator + + +class caasactivator(cmactivator.CMGlobalActivator): + playbooks = dict( + infra_chart_reconfig="/opt/openstack-ansible/playbooks/infra_chart_reconfig_fluentd.yaml" + ) + + def __init__(self): + super(caasactivator, self).__init__() + + def get_subscription_info(self): + return 'cloud.caas' + + def activate_set(self, props): + self._activate() + + def activate_delete(self, props): + self._activate() + + def activate_full(self, target): + self._activate(target=target) + + def _activate(self, target=None): + if 'CONFIG_PHASE' in os.environ: + return + self.run_playbook(self.playbooks['infra_chart_reconfig'], target) diff --git a/src/caas_logging/cli/__init__.py b/src/caas_logging/cli/__init__.py new file mode 100644 index 0000000..78c5878 --- /dev/null +++ b/src/caas_logging/cli/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Nokia + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/caas_logging/cli/caas.py b/src/caas_logging/cli/caas.py new file mode 100644 index 0000000..f8a6057 --- /dev/null +++ b/src/caas_logging/cli/caas.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python + +# Copyright 2019 Nokia + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long, too-few-public-methods + +from copy import deepcopy + +from hostcli.helper import ListerHelper, ShowOneHelper, CommandHelper + +API_VERSION = 'v1' +RESOURCE_PREFIX = 'caas/%s/' % API_VERSION +ID = 'id' +NAMESPACE = 'namespace' +PLUGIN = 'plugin' +TARGETURL = 'target_url' +STREAM = 'stream' + + +FIELDMAP = { + ID: {'display': 'ID', + 'help': 'The ID of the log entry'}, + NAMESPACE: {'display': 'namespace', + 'help': 'The kubernetes namespace where the log entry applies'}, + PLUGIN: {'display': 'plugin', + 'help': 'The fluentd plugin which is used for forwarding log entries. ' + 'Should be one of remote_syslog, elasticsearch'}, + TARGETURL: {'display': 'target_url', + 'help': 'The URL of the log storage where fluentd will send log entries'}, + STREAM: {'display': 'stream', + 'help': 'The stream which will be logged by fluentd. ' + 'Should be one of stdout, stderr, both'} +} + + +class CaasCliLister(ListerHelper): + """Helper class for Lister""" + def __init__(self, app, app_args, cmd_name=None): + super(CaasCliLister, self).__init__(app, app_args, cmd_name) + self.fieldmap = deepcopy(FIELDMAP) + self.resource_prefix = RESOURCE_PREFIX + + +class CaasCliShowOne(ShowOneHelper): + """Helper class for ShowOne""" + def __init__(self, app, app_args, cmd_name=None): + super(CaasCliShowOne, self).__init__(app, app_args, cmd_name) + self.fieldmap = deepcopy(FIELDMAP) + self.resource_prefix = RESOURCE_PREFIX + + +class CaasCliCommand(CommandHelper): + """Helper class for Command""" + def __init__(self, app, app_args, cmd_name=None): + super(CaasCliCommand, self).__init__(app, app_args, cmd_name) + self.fieldmap = deepcopy(FIELDMAP) + self.resource_prefix = RESOURCE_PREFIX + + +class CreateAppLogBackend(CaasCliCommand): + """A command for adding a new CaaS application log forwarding entry.""" + + def __init__(self, app, app_args, cmd_name=None): + super(CreateAppLogBackend, self).__init__(app, app_args, cmd_name) + self.operation = 'post' + self.endpoint = 'log/apps' + self.mandatory_positional = True + self.positional_count = 4 + self.arguments = [NAMESPACE, PLUGIN, TARGETURL, STREAM] + self.message = 'Entry has been added.' + + +class ChangeAppLogBackend(CaasCliCommand): + """A command for modifying a CaaS application log forwarding entry.""" + + def __init__(self, app, app_args, cmd_name=None): + super(ChangeAppLogBackend, self).__init__(app, app_args, cmd_name) + self.operation = 'put' + self.endpoint = 'log/apps' + self.mandatory_positional = True + self.positional_count = 1 + self.arguments = [ID, NAMESPACE, PLUGIN, TARGETURL, STREAM] + self.message = 'Entry has been updated.' + + +class DeleteAppLogBackend(CaasCliCommand): + """A command for removing a CaaS application log forwarding entry.""" + + def __init__(self, app, app_args, cmd_name=None): + super(DeleteAppLogBackend, self).__init__(app, app_args, cmd_name) + self.operation = 'delete' + self.endpoint = 'log/apps' + self.mandatory_positional = True + self.positional_count = 1 + self.arguments = [ID] + self.message = 'Entry has been deleted.' + + +class ShowAppLogBackend(CaasCliShowOne): + """A command for showing detail of a CaaS application log forwarding entry.""" + + def __init__(self, app, app_args, cmd_name=None): + super(ShowAppLogBackend, self).__init__(app, app_args, cmd_name) + self.operation = 'get' + self.endpoint = 'log/apps' + self.mandatory_positional = True + self.positional_count = 1 + self.arguments = [ID] + self.columns = [ID, NAMESPACE, PLUGIN, TARGETURL, STREAM] + self.default_sort = [ID, 'asc'] + + +class ListAppLogBackend(CaasCliLister): + """A command for listing existing CaaS application log forwarding entries.""" + + def __init__(self, app, app_args, cmd_name=None): + super(ListAppLogBackend, self).__init__(app, app_args, cmd_name) + self.operation = 'get' + self.endpoint = 'log/apps' + self.no_positional = True + self.columns = [ID, NAMESPACE, PLUGIN, TARGETURL, STREAM] + self.default_sort = [ID, 'asc'] + + +class ListAppLogBackendForNamespace(CaasCliLister): + """A command for listing existing CaaS application log forwarding entries for a namespace""" + + def __init__(self, app, app_args, cmd_name=None): + super(ListAppLogBackendForNamespace, self).__init__(app, app_args, cmd_name) + self.operation = 'get' + self.endpoint = 'log/apps' + self.mandatory_positional = True + self.positional_count = 1 + self.arguments = [NAMESPACE] + self.columns = [ID, NAMESPACE, PLUGIN, TARGETURL, STREAM] + self.default_sort = [ID, 'asc'] diff --git a/src/caas_logging/rest-plugin/__init__.py b/src/caas_logging/rest-plugin/__init__.py new file mode 100644 index 0000000..78c5878 --- /dev/null +++ b/src/caas_logging/rest-plugin/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Nokia + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/src/caas_logging/rest-plugin/caas.ini b/src/caas_logging/rest-plugin/caas.ini new file mode 100644 index 0000000..b96c574 --- /dev/null +++ b/src/caas_logging/rest-plugin/caas.ini @@ -0,0 +1,16 @@ +# Copyright 2019 Nokia + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[v1] +handlers=AppLogStoreHandler diff --git a/src/caas_logging/rest-plugin/caashandler.py b/src/caas_logging/rest-plugin/caashandler.py new file mode 100644 index 0000000..d506a14 --- /dev/null +++ b/src/caas_logging/rest-plugin/caashandler.py @@ -0,0 +1,308 @@ +# Copyright 2019 Nokia + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +from cmframework.apis import cmclient + +from yarf.restresource import RestResource + +MISSING_LOG_STORE_ENTRY_INDEX = 'Missing log store entry index parameter' +PATTERN_SHOULD_BE_ONE_OF = '{} should be one of: {}' + +ID = 'id' +NAMESPACE = 'namespace' +SUPPORTED_BACKENDS = ('elasticsearch', 'remote_syslog') +SUPPORTED_STREAMS = ('stdout', 'stderr', 'both') + + +class CaasHandler(RestResource): + KEY_CAAS = 'cloud.caas' + + def _get_caas_config(self): + api = cmclient.CMClient() + return json.loads(api.get_property(self.KEY_CAAS)) + + def _update_caas_config(self, updated): + api = cmclient.CMClient() + api.set_property(self.KEY_CAAS, json.dumps(updated)) + + @staticmethod + def _response(return_code, data='', error_desc=''): + return {"code": return_code, "description": error_desc, "data": data} + + def _result(self, data=None): + return self._response(0, data=data) + + def _error(self, exc): + error_desc = str(exc).replace('Internal error, ', '') + return self._response(1, error_desc=error_desc) + + def _modify_caas_config(self, modifier_method): + try: + caas = self._get_caas_config() + modifier_method(self, caas, self.get_args()) + self._update_caas_config(caas) + return self._result('') + except Exception as exc: + return self._error(exc) + + +class AppLogStoreHandler(CaasHandler): + KEY_NAME = 'log_forwarding' + + parser_arguments = [ID, NAMESPACE, 'plugin', 'target_url', 'stream'] + endpoints = ['log/apps'] + + def post(self): + """ + .. :quickref: CaaS - Log forwarding;Add the log forwarding configuration entry + + **Add the new containerized application log forwarding entry to configuration**: + + **Example request**: + + .. sourcecode:: http + + POST /caas/v1/log/apps HTTP/1.1 + Host: haproxyvip:61200 + Content-Type: application/json + Accept: application/json + + { + "namespace": "app1", + "plugin": "elasticsearch", + "target_url": "http://elasticsearch.elk.svc.nokia.net:9200", + "stream": "stderr" + } + + :json int code: The status code, 0 when OK other when error + :>json string description: The error description, present if code is non zero + :>json string data: Empty string in all cases + :statuscode 200: no error + :statuscode 401: on authentication error + + """ + def func(self, caas, args): + entries = caas[self.KEY_NAME] + args.pop(ID) + if args['plugin'] not in SUPPORTED_BACKENDS: + raise ValueError(PATTERN_SHOULD_BE_ONE_OF.format('plugin', ', '.join(SUPPORTED_BACKENDS))) + if args['stream'] not in SUPPORTED_STREAMS: + raise ValueError(PATTERN_SHOULD_BE_ONE_OF.format('stream', ', '.join(SUPPORTED_STREAMS))) + if entries.count(args): + raise ValueError('Already exists') + entries.append(args) + + return self._modify_caas_config(func) + + def get(self): + """ + .. :quickref: CaaS - Log forwarding;Get the existing log forwarding configuration + + **Get the existing containerized application log forwarding configuration**: + + Optionally request could filter entries by namespace. + + **Example request**: + + .. sourcecode:: http + + GET /caas/v1/apps HTTP/1.1 + Host: haproxyvip:61200 + Content-Type: application/json + Accept: application/json + + { + "namespace": "app1" + } + + :json int code: The status code, 0 when OK other when error + :>json string description: The error description, present if code is non zero + :>json string data: Contains the log forwarding configuration + :>json string id: The identifier of the log forwarding entry + :>json string namespace: Apply forwarding for containers deployed in this namespace + :>json string plugin: Plugin that handles log forwarding, one of ``elasticsearch``, ``remote_syslog`` + :>json string target_url: Log storage service URL (eg. ``protocol://ip-or-fqdn:port``) + :>json string stream: Output stream source of gathered logs, one of ``stdout``, ``stderr``, ``both`` + :statuscode 200: no error + :statuscode 401: on authentication error + + """ + try: + caas = self._get_caas_config() + value = caas[self.KEY_NAME] + args = self.get_args() + data = {} + for idx, v in enumerate(value, start=1): + v[ID] = idx + data[idx] = v + if args[ID] is not None: + return self._result({args[ID]: value[int(args[ID]) - 1]}) + elif args[NAMESPACE] is not None: + return self._result({k: v for (k, v) in data.items() if v[NAMESPACE] == args[NAMESPACE]}) + else: + return self._result(data) + + except Exception as exc: + return self._error(exc) + + def put(self): + """ + .. :quickref: CaaS - Log forwarding;Change the existing log forwarding entry configuration + + **Change the existing containerized application log forwarding entry configuration**: + + **Example request**: + + .. sourcecode:: http + + PUT /caas/v1/apps HTTP/1.1 + Host: haproxyvip:61200 + Content-Type: application/json + Accept: application/json + + { + "id": 1, + "namespace": "app2" + } + + :json int code: The status code, 0 when OK other when error + :>json string description: The error description, present if code is non zero + :>json string data: Empty string in all cases + :statuscode 200: no error + :statuscode 401: on authentication error + + """ + def func(self, caas, args): + if args[ID] is None: + raise ValueError(MISSING_LOG_STORE_ENTRY_INDEX) + else: + idx = int(args[ID]) - 1 + for k, v in args.items(): + if not k == ID and v is not None: + caas[self.KEY_NAME][idx][k] = v + + return self._modify_caas_config(func) + + def delete(self): + """ + .. :quickref: CaaS - Log forwarding;Delete the entry from log forwarding configuration + + **Delete the entry from containerized application log forwarding configuration**: + + **Example request**: + + .. sourcecode:: http + + DELETE /caas/v1/apps HTTP/1.1 + Host: haproxyvip:61200 + Content-Type: application/json + Accept: application/json + + { + "id": "1" + } + + :json int code: The status code, 0 when OK other when error + :>json string description: The error description, present if code is non zero + :>json string data: Empty string in all cases + :statuscode 200: no error + :statuscode 401: on authentication error + + """ + def func(self, caas, args): + if args[ID] is None: + raise ValueError(MISSING_LOG_STORE_ENTRY_INDEX) + else: + caas[self.KEY_NAME].pop(int(args[ID]) - 1) + + return self._modify_caas_config(func) diff --git a/src/setup.py b/src/setup.py new file mode 100644 index 0000000..c0c5502 --- /dev/null +++ b/src/setup.py @@ -0,0 +1,41 @@ +# Copyright 2019 Nokia + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from setuptools import setup, find_packages +setup( + name='caas_logging', + version='1.0', + license='Apache-2.0', + author='Gabor Mate', + author_email='gabor.mate@nokia.com', + platforms=['Any'], + scripts=[], + provides=[], + namespace_packages=['caas_logging'], + packages=find_packages(), + include_package_data=True, + description='CaaS Logging for Akraino REC', + install_requires=['flask', 'flask-restful', 'hostcli'], + entry_points={ + 'hostcli.commands': [ + 'caas applications log add = caas_logging.cli.caas:CreateAppLogBackend', + 'caas applications log change = caas_logging.cli.caas:ChangeAppLogBackend', + 'caas applications log delete = caas_logging.cli.caas:DeleteAppLogBackend', + 'caas applications log show = caas_logging.cli.caas:ShowAppLogBackend', + 'caas applications log list = caas_logging.cli.caas:ListAppLogBackend', + 'caas applications log list namespace = caas_logging.cli.caas:ListAppLogBackendForNamespace', + ], + }, + zip_safe=False, +) -- 2.16.6