From 67a7cb59e0ff498eff5e8fe23fdd19db77ed8a22 Mon Sep 17 00:00:00 2001 From: xinhuili Date: Wed, 14 Aug 2019 13:59:25 +0800 Subject: [PATCH] Add Revised Tosca-loader This patch is to add new tosca-loader Signed-off-by: XINHUI LI Change-Id: I249006d8f4961fe88601cdcf4c5004170292ddca --- src/tosca-loader/.dockerignore | 6 + src/tosca-loader/.gitignore | 16 ++ src/tosca-loader/Dockerfile | 48 ++++ src/tosca-loader/Jenkinsfile | 42 ++++ src/tosca-loader/Makefile | 32 +++ src/tosca-loader/README.md | 34 +++ src/tosca-loader/VERSION | 1 + src/tosca-loader/book.json | 4 + src/tosca-loader/ci_scripts/push_containers.sh | 7 + src/tosca-loader/ci_scripts/push_manifest.sh | 8 + src/tosca-loader/docs/GLOSSARY.md | 2 + src/tosca-loader/docs/README.md | 152 ++++++++++++ src/tosca-loader/docs/SUMMARY.md | 4 + src/tosca-loader/docs/devel.md | 27 ++ src/tosca-loader/loader/Dockerfile.tosca-loader | 40 +++ src/tosca-loader/loader/tosca-loader.sh | 31 +++ src/tosca-loader/src/grpc_client/KEYS.reference.py | 75 ++++++ src/tosca-loader/src/grpc_client/__init__.py | 16 ++ src/tosca-loader/src/grpc_client/main.py | 80 ++++++ .../src/grpc_client/models_accessor.py | 87 +++++++ src/tosca-loader/src/grpc_client/resources.py | 18 ++ src/tosca-loader/src/main.py | 64 +++++ src/tosca-loader/src/tosca/__init__.py | 16 ++ src/tosca-loader/src/tosca/custom_types/.gitignore | 2 + src/tosca-loader/src/tosca/default.py | 21 ++ src/tosca-loader/src/tosca/generator.py | 64 +++++ src/tosca-loader/src/tosca/parser.py | 270 ++++++++++++++++++++ src/tosca-loader/src/tosca/xtarget/tosca.xtarget | 72 ++++++ .../src/tosca/xtarget/tosca_keys.xtarget | 5 + src/tosca-loader/src/web_server/__init__.py | 16 ++ src/tosca-loader/src/web_server/main.py | 106 ++++++++ src/tosca-loader/src/xos-tosca-config-schema.yaml | 28 +++ src/tosca-loader/test/helpers.py | 21 ++ src/tosca-loader/test/out/.gitignore | 2 + src/tosca-loader/test/test_config.yaml | 13 + src/tosca-loader/test/test_grpc_models_accessor.py | 221 +++++++++++++++++ src/tosca-loader/test/test_tosca_generator.py | 80 ++++++ src/tosca-loader/test/test_tosca_parser.py | 221 +++++++++++++++++ src/tosca-loader/test/test_tosca_parser_e2e.py | 274 +++++++++++++++++++++ src/tosca-loader/test/tosca/link.yaml | 160 ++++++++++++ src/tosca-loader/test/tosca/must_exist.yaml | 49 ++++ src/tosca-loader/test/tosca/test.yaml | 57 +++++ src/tosca-loader/unittest.cfg | 7 + 43 files changed, 2499 insertions(+) create mode 100644 src/tosca-loader/.dockerignore create mode 100644 src/tosca-loader/.gitignore create mode 100644 src/tosca-loader/Dockerfile create mode 100644 src/tosca-loader/Jenkinsfile create mode 100644 src/tosca-loader/Makefile create mode 100644 src/tosca-loader/README.md create mode 100644 src/tosca-loader/VERSION create mode 100644 src/tosca-loader/book.json create mode 100755 src/tosca-loader/ci_scripts/push_containers.sh create mode 100755 src/tosca-loader/ci_scripts/push_manifest.sh create mode 100644 src/tosca-loader/docs/GLOSSARY.md create mode 100644 src/tosca-loader/docs/README.md create mode 100644 src/tosca-loader/docs/SUMMARY.md create mode 100644 src/tosca-loader/docs/devel.md create mode 100644 src/tosca-loader/loader/Dockerfile.tosca-loader create mode 100755 src/tosca-loader/loader/tosca-loader.sh create mode 100644 src/tosca-loader/src/grpc_client/KEYS.reference.py create mode 100644 src/tosca-loader/src/grpc_client/__init__.py create mode 100644 src/tosca-loader/src/grpc_client/main.py create mode 100644 src/tosca-loader/src/grpc_client/models_accessor.py create mode 100644 src/tosca-loader/src/grpc_client/resources.py create mode 100644 src/tosca-loader/src/main.py create mode 100644 src/tosca-loader/src/tosca/__init__.py create mode 100644 src/tosca-loader/src/tosca/custom_types/.gitignore create mode 100644 src/tosca-loader/src/tosca/default.py create mode 100644 src/tosca-loader/src/tosca/generator.py create mode 100644 src/tosca-loader/src/tosca/parser.py create mode 100644 src/tosca-loader/src/tosca/xtarget/tosca.xtarget create mode 100644 src/tosca-loader/src/tosca/xtarget/tosca_keys.xtarget create mode 100644 src/tosca-loader/src/web_server/__init__.py create mode 100644 src/tosca-loader/src/web_server/main.py create mode 100644 src/tosca-loader/src/xos-tosca-config-schema.yaml create mode 100644 src/tosca-loader/test/helpers.py create mode 100644 src/tosca-loader/test/out/.gitignore create mode 100644 src/tosca-loader/test/test_config.yaml create mode 100644 src/tosca-loader/test/test_grpc_models_accessor.py create mode 100644 src/tosca-loader/test/test_tosca_generator.py create mode 100644 src/tosca-loader/test/test_tosca_parser.py create mode 100644 src/tosca-loader/test/test_tosca_parser_e2e.py create mode 100644 src/tosca-loader/test/tosca/link.yaml create mode 100644 src/tosca-loader/test/tosca/must_exist.yaml create mode 100644 src/tosca-loader/test/tosca/test.yaml create mode 100644 src/tosca-loader/unittest.cfg diff --git a/src/tosca-loader/.dockerignore b/src/tosca-loader/.dockerignore new file mode 100644 index 0000000..5465185 --- /dev/null +++ b/src/tosca-loader/.dockerignore @@ -0,0 +1,6 @@ +.idea +cover +docs +test +Dockerfile +Makefile diff --git a/src/tosca-loader/.gitignore b/src/tosca-loader/.gitignore new file mode 100644 index 0000000..24478e5 --- /dev/null +++ b/src/tosca-loader/.gitignore @@ -0,0 +1,16 @@ +.idea +*.pyc +.noseids +local_certs.crt +src/tosca/tmp.yaml +src/grpc_client/KEYS.py + +# Test output +nose2-junit.xml +.coverage +cover +coverage.xml + +# hack +xos +_book diff --git a/src/tosca-loader/Dockerfile b/src/tosca-loader/Dockerfile new file mode 100644 index 0000000..36b9223 --- /dev/null +++ b/src/tosca-loader/Dockerfile @@ -0,0 +1,48 @@ +# docker build -t xosproject/xos-tosca:candidate . + +# xosproject/xos-tosca +FROM cachengo/xos-client:2.1.22 + +# Set environment variables +ENV CODE_SOURCE . +ENV CODE_DEST /opt/xos-tosca +WORKDIR ${CODE_DEST} + +# Add XOS-TOSCA code +COPY ${CODE_SOURCE}/ ${CODE_DEST}/ + +# Install dependencies +RUN pip install klein==16.12.0 + +EXPOSE 9102 + +# Label image +ARG org_label_schema_schema_version=1.0 +ARG org_label_schema_name=gui-extension-sample +ARG org_label_schema_version=unknown +ARG org_label_schema_vcs_url=unknown +ARG org_label_schema_vcs_ref=unknown +ARG org_label_schema_build_date=unknown +ARG org_opencord_vcs_commit_date=unknown +ARG org_opencord_component_chameleon_version=unknown +ARG org_opencord_component_chameleon_vcs_url=unknown +ARG org_opencord_component_chameleon_vcs_ref=unknown +ARG org_opencord_component_xos_version=unknown +ARG org_opencord_component_xos_vcs_url=unknown +ARG org_opencord_component_xos_vcs_ref=unknown + +LABEL org.label-schema.schema-version=$org_label_schema_schema_version \ + org.label-schema.name=$org_label_schema_name \ + org.label-schema.version=$org_label_schema_version \ + org.label-schema.vcs-url=$org_label_schema_vcs_url \ + org.label-schema.vcs-ref=$org_label_schema_vcs_ref \ + org.label-schema.build-date=$org_label_schema_build_date \ + org.opencord.vcs-commit-date=$org_opencord_vcs_commit_date \ + org.opencord.component.chameleon.version=$org_opencord_component_chameleon_version \ + org.opencord.component.chameleon.vcs-url=$org_opencord_component_chameleon_vcs_url \ + org.opencord.component.chameleon.vcs-ref=$org_opencord_component_chameleon_vcs_ref \ + org.opencord.component.xos.version=$org_opencord_component_xos_version \ + org.opencord.component.xos.vcs-url=$org_opencord_component_xos_vcs_url \ + org.opencord.component.xos.vcs-ref=$org_opencord_component_xos_vcs_ref + +ENTRYPOINT [ "/usr/bin/python", "src/main.py" ] diff --git a/src/tosca-loader/Jenkinsfile b/src/tosca-loader/Jenkinsfile new file mode 100644 index 0000000..a58adb4 --- /dev/null +++ b/src/tosca-loader/Jenkinsfile @@ -0,0 +1,42 @@ +pipeline { + agent any + stages { + stage('Build') { + parallel { + stage('Build aarch64') { + agent { + node { + label 'aarch64' + } + + } + steps { + withDockerRegistry([ credentialsId: "fcf9c294-b8a9-4f7e-87d6-d0446f712411", url: "https://index.docker.io/v1/" ]) { + sh 'ci_scripts/push_containers.sh' + } + } + } + stage('Build x86') { + agent { + node { + label 'x86_64' + } + + } + steps { + withDockerRegistry([ credentialsId: "fcf9c294-b8a9-4f7e-87d6-d0446f712411", url: "https://index.docker.io/v1/" ]) { + sh 'ci_scripts/push_containers.sh' + } + } + } + } + } + stage('Push Manifest') { + steps { + withDockerRegistry([ credentialsId: "fcf9c294-b8a9-4f7e-87d6-d0446f712411", url: "https://index.docker.io/v1/" ]) { + sh 'ci_scripts/push_manifest.sh' + } + } + } + } +} diff --git a/src/tosca-loader/Makefile b/src/tosca-loader/Makefile new file mode 100644 index 0000000..b91ee28 --- /dev/null +++ b/src/tosca-loader/Makefile @@ -0,0 +1,32 @@ +help: + @echo "tests: Run unit tests (need the xos dev virtual-env activated)" + @echo "tosca: Generate tosca definition from core.xproto" + @echo "build: Build the docker image for xos-tosca" + @echo "start: Run an xos-tosca container" + @echo "clean: Remove the xos-tosca container (if any), and the image (if any)" + @echo "test-create: Send a sample tosca recipe" + @echo "test-delete: Delete a sample tosca recipe" + +tests: tosca + nose2 --verbose --coverage-report xml --coverage-report term --junit-xml + +build: + docker build -t xosproject/xos-tosca . + docker tag xosproject/xos-tosca:latest xosproject/xos-tosca:candidate + +start: build + docker run -p 9200:9200 --name xos-tosca -d xosproject/xos-tosca + +clean: + docker rm -f xos-tosca || true + docker rmi -f xosproject/xos-tosca || true + +test-create: + curl -H "xos-username: xosadmin@opencord.org" -H "xos-password: rk1UYDHZXbu6KVCMkhmV" -X POST --data-binary @test/tosca/test.yaml 127.0.0.1:9102/run + +test-delete: + curl -H "xos-username: xosadmin@opencord.org" -H "xos-password: rk1UYDHZXbu6KVCMkhmV" -X POST --data-binary @test/tosca/test.yaml 127.0.0.1:9102/delete + +tosca: + xosgenx --target=src/tosca/xtarget/tosca.xtarget --output=src/tosca/custom_types --write-to-file=target ../xos/xos/core/models/core.xproto + xosgenx --target=src/tosca/xtarget/tosca_keys.xtarget --output=src/grpc_client/ --write-to-file=single --dest-file=KEYS.py ../xos/xos/core/models/core.xproto diff --git a/src/tosca-loader/README.md b/src/tosca-loader/README.md new file mode 100644 index 0000000..9f2c88c --- /dev/null +++ b/src/tosca-loader/README.md @@ -0,0 +1,34 @@ +# XOS TOSCA + +Welcome to the XOS TOSCA. + +## Documentation +You can find the documentation in the `docs` folder. It has been created using `gitbook` and can be consumed as a local website. +To bring it up, just open a termina pointing to this folder and execute: `gitbook serve`, then open a browser at `http://localhost:4000` + +## Support + +For support please refer to: + +**Slack**
+[slackin.opencord.org](https://slackin.opencord.org/) + +**Mailing List**
+[CORD Discuss](https://groups.google.com/a/opencord.org/forum/#!forum/cord-discuss)
+[CORD Developers](https://groups.google.com/a/opencord.org/forum/#!forum/cord-dev) + +**Wiki**
+[wiki.opencord.org](https://wiki.opencord.org/) + + +## Testing + +To run tests, you must first create a virtualenv with the XOS dependencies: + +```shell +cd ../xos +./scripts/setup_venv.sh +source venv-xos/bin/activate +cd ../xos-tosca +make tests +``` diff --git a/src/tosca-loader/VERSION b/src/tosca-loader/VERSION new file mode 100644 index 0000000..e25d8d9 --- /dev/null +++ b/src/tosca-loader/VERSION @@ -0,0 +1 @@ +1.1.5 diff --git a/src/tosca-loader/book.json b/src/tosca-loader/book.json new file mode 100644 index 0000000..abbf32f --- /dev/null +++ b/src/tosca-loader/book.json @@ -0,0 +1,4 @@ +{ + "title": "XOS-TOSCA User Documentation", + "root": "./docs" +} diff --git a/src/tosca-loader/ci_scripts/push_containers.sh b/src/tosca-loader/ci_scripts/push_containers.sh new file mode 100755 index 0000000..f354143 --- /dev/null +++ b/src/tosca-loader/ci_scripts/push_containers.sh @@ -0,0 +1,7 @@ +export IMAGE_TAG=$(cat VERSION) +export AARCH=`uname -m` +docker build -t cachengo/xos-tosca-$AARCH:$IMAGE_TAG . +docker push cachengo/xos-tosca-$AARCH:$IMAGE_TAG +cd loader +docker build -f Dockerfile.tosca-loader -t cachengo/tosca-loader-$AARCH:$IMAGE_TAG . +docker push cachengo/tosca-loader-$AARCH:$IMAGE_TAG diff --git a/src/tosca-loader/ci_scripts/push_manifest.sh b/src/tosca-loader/ci_scripts/push_manifest.sh new file mode 100755 index 0000000..ea2fbdc --- /dev/null +++ b/src/tosca-loader/ci_scripts/push_manifest.sh @@ -0,0 +1,8 @@ +export IMAGE_TAG=$(cat VERSION) +export DOCKER_CLI_EXPERIMENTAL=enabled + +docker manifest create --amend cachengo/xos-tosca:$IMAGE_TAG cachengo/xos-tosca-x86_64:$IMAGE_TAG cachengo/xos-tosca-aarch64:$IMAGE_TAG +docker manifest push cachengo/xos-tosca:$IMAGE_TAG + +docker manifest create --amend cachengo/tosca-loader:$IMAGE_TAG cachengo/tosca-loader-x86_64:$IMAGE_TAG cachengo/tosca-loader-aarch64:$IMAGE_TAG +docker manifest push cachengo/tosca-loader:$IMAGE_TAG diff --git a/src/tosca-loader/docs/GLOSSARY.md b/src/tosca-loader/docs/GLOSSARY.md new file mode 100644 index 0000000..3bef333 --- /dev/null +++ b/src/tosca-loader/docs/GLOSSARY.md @@ -0,0 +1,2 @@ +# XOS-TOSCA Glossary + diff --git a/src/tosca-loader/docs/README.md b/src/tosca-loader/docs/README.md new file mode 100644 index 0000000..6dd692f --- /dev/null +++ b/src/tosca-loader/docs/README.md @@ -0,0 +1,152 @@ +# TOSCA Interface + +A TOSCA interface is available for configuring and controlling CORD. It is +auto-generated from the set of [models](../xos/README.md) configured into the +POD manifest, and includes both core and service-specific models. + +## What is TOSCA? + +Topology and Orchestration Specification for Cloud Applications (TOSCA) is an +OASIS standard language to describe a topology of cloud based web services, +their components, relationships, and the processes that manage them. The TOSCA +standard includes specifications to describe processes that create or modify +web services. You can read more about it on the +[OASIS](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=tosca) +website. + +CORD extends the TOSCA specification to support custom models for services, +allow operators to manage them with a simple and well-known YAML interface. + +## Difference between `xos-tosca` and `*-tosca-loader` + +When you deploy CORD using helm charts you'll notice that there are two containers +that contains the name `tosca`. There is quite a big difference between them: + +- `xos-tosca` contains the TOSCA engine and parser, and exposes a REST api to let you push TOSCA recipes into XOS +- `*-tosca-loader` is a convenience container that pushes a set of recipes into `xos-tosca` and then exits. + +## Internals + +The `xos-tosca` container autogenerates the TOSCA interface starting from the +`xproto` definition. When the `xos-tosca` container starts, it connects to +`xos-core` via the `gRPC` API to fetch all the `xproto` definition of the +onboarded models. This includes both `core` and `service` models. Then using +the `xos-genx` toolchain, it will generates the corresponding TOSCA +specifications. + +For example, the `xproto` definition of a compute node in `XOS` is: + +```protobuf +message Node::node_policy (XOSBase) { + required string name = 1 [max_length = 200, content_type = "stripped", blank = False, help_text = "Name of the Node", null = False, db_index = False]; + required manytoone site_deployment->SiteDeployment:nodes = 2 [db_index = True, null = False, blank = False]; +} +``` + +which is then transformed in the following TOSCA spec: + +```yaml +tosca_definitions_version: tosca_simple_yaml_1_0 + +node_types: + + tosca.nodes.Node: + derived_from: tosca.nodes.Root + description: "An XOS Node" + capabilities: + node: + type: tosca.capabilities.xos.Node + properties: + must-exist: + type: boolean + default: false + description: Allow to reference existing models in TOSCA recipes + name: + type: string + required: false + description: "Name of the Node" + + + tosca.relationships.BelongsToOne: + derived_from: tosca.relationships.Root + valid_target_types: [ tosca.capabilities.xos.SiteDeployment ] + + + tosca.capabilities.xos.Node: + derived_from: tosca.capabilities.Root + description: Node +``` + +In TOSCA terminology, the above woule be called a `TOSCA node type` +(although confusingly, it's defined for the `Node` model in CORD, +which represents a server). + +## Using TOSCA + +Once CORD is up and running, a `node` can be added to a POD +using the TOSCA interface by uploading the following recipe: + +```yaml +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: Load a compute node in XOS + +imports: + - custom_types/node.yaml + +topology_template: + node_templates: + + # A compute node + GratefulVest: + type: tosca.nodes.Node + properties: + name: Grateful Vest +``` + +In TOSCA terminology, the above would be called a `TOSCA node template`. + +### Where to find the generated specs? + +On any running CORD POD, the TOSCA apis are accessible as: + +```shell +curl http://:/xos-tosca | python -m json.tool +``` + +And it will return a list of all the recipes with the related url: + +```json +{ + "image": "/custom_type/image", + "site": "/custom_type/site", + ... +} +``` + +For examples, to site the TOSCA spec of the Site model, you can use the URL: + +```shell +curl http://:/xos-tosca/custom_type/site +``` + +If you have a running `xos-tosca` container you can also find generated copies +of the specs in `/opt/xos-tosca/src/tosca/custom_types`. + +### How to load a TOSCA recipe in the system + +The `xos-tosca` container exposes two endpoint: + +```shell +POST http://:/run +POST http://:/delete +``` + +To load a recipe via `curl` you can use this command: + +```shell +curl -H "xos-username: xosadmin@opencord.org" -H "xos-password: " -X POST --data-binary @ http://:/run +``` + +_If you installed the `xos-core` charts without modifications, the `tosca-port` is `30007` + diff --git a/src/tosca-loader/docs/SUMMARY.md b/src/tosca-loader/docs/SUMMARY.md new file mode 100644 index 0000000..263112f --- /dev/null +++ b/src/tosca-loader/docs/SUMMARY.md @@ -0,0 +1,4 @@ +# Summary + +* [Introduction](README.md) + diff --git a/src/tosca-loader/docs/devel.md b/src/tosca-loader/docs/devel.md new file mode 100644 index 0000000..7586f3c --- /dev/null +++ b/src/tosca-loader/docs/devel.md @@ -0,0 +1,27 @@ +# Development + +To run a development environment locally, you'll need to have: + +* an XOS configuration running in the frontend vm +* source the xos virtual env (from `xos` root: `source scripts/setup_venv.sh`) +* install `xos-tosca` specific dependencies: `pip install -r + pip_requirements.txt` +* an entry in the `/etc/hosts` file that point `xos-core.opencord.org` to you + local environment + +## Run the xos-tosca process + +You can run this either from an IDE or: + +```bash +python scr/main.py +``` + +## Sample call + +To send an example request to `xos-tosca`: + +```bash +curl -X POST --data-binary @test.yaml 127.0.0.1:9200 +``` + diff --git a/src/tosca-loader/loader/Dockerfile.tosca-loader b/src/tosca-loader/loader/Dockerfile.tosca-loader new file mode 100644 index 0000000..fd55517 --- /dev/null +++ b/src/tosca-loader/loader/Dockerfile.tosca-loader @@ -0,0 +1,40 @@ +# Copyright 2018-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# xosproject/tosca-loader + +FROM alpine:3.7 + +RUN apk add --no-cache httpie=0.9.9-r0 + +COPY tosca-loader.sh /usr/local/bin/tosca-loader.sh + +# Label image +ARG org_label_schema_schema_version=1.0 +ARG org_label_schema_name=tosca-loader +ARG org_label_schema_version=unknown +ARG org_label_schema_vcs_url=unknown +ARG org_label_schema_vcs_ref=unknown +ARG org_label_schema_build_date=unknown +ARG org_opencord_vcs_commit_date=unknown + +LABEL org.label-schema.schema-version=$org_label_schema_schema_version \ + org.label-schema.name=$org_label_schema_name \ + org.label-schema.version=$org_label_schema_version \ + org.label-schema.vcs-url=$org_label_schema_vcs_url \ + org.label-schema.vcs-ref=$org_label_schema_vcs_ref \ + org.label-schema.build-date=$org_label_schema_build_date \ + org.opencord.vcs-commit-date=$org_opencord_vcs_commit_date + +CMD [ "/bin/sh", "/usr/local/bin/tosca-loader.sh" ] diff --git a/src/tosca-loader/loader/tosca-loader.sh b/src/tosca-loader/loader/tosca-loader.sh new file mode 100755 index 0000000..9f27a8b --- /dev/null +++ b/src/tosca-loader/loader/tosca-loader.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# tosca-loader.sh +# loads TOSCA files found in /opt/tosca into XOS + +echo "Starting TOSCA loader using httpie version: $(http --version)" + +for recipe in /opt/tosca/* +do + echo "Loading: $recipe" + http --check-status --ignore-stdin \ + POST "http://xos-tosca:$XOS_TOSCA_SERVICE_PORT/run" \ + "xos-username:$XOS_USER" \ + "xos-password:$XOS_PASSWD" \ + "@$recipe" || exit 1 + echo '' +done diff --git a/src/tosca-loader/src/grpc_client/KEYS.reference.py b/src/tosca-loader/src/grpc_client/KEYS.reference.py new file mode 100644 index 0000000..fd8c776 --- /dev/null +++ b/src/tosca-loader/src/grpc_client/KEYS.reference.py @@ -0,0 +1,75 @@ +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +######################################################################### +# # +# This file is here for reference, the used one is generate by xos-genx # +# # +######################################################################### + +TOSCA_KEYS = { + 'XOSBase': [], + 'User': ['email'], + 'Privilege': [], + 'AddressPool': ['name'], + 'Controller': ['name'], + 'ControllerImages': [], + 'ControllerNetwork': [], + 'ControllerRole': [], + 'ControllerSite': [], + 'ControllerPrivilege': [], + 'ControllerSitePrivilege': [], + 'ControllerSlice': [], + 'ControllerSlicePrivilege': [], + 'ControllerUser': [], + 'Deployment': ['name'], + 'DeploymentPrivilege': [], + 'DeploymentRole': [], + 'Diag': ['name'], + 'Flavor': ['name'], + 'Image': ['name'], + 'ImageDeployments': [], + 'Instance': ['name'], + 'Network': ['name'], + 'NetworkParameter': [], + 'NetworkParameterType': ['name'], + 'NetworkSlice': ['network', 'slice'], + 'NetworkTemplate': ['name'], + 'Node': ['name'], + 'NodeLabel': ['name'], + 'Port': [], + 'Role': [], + 'Service': ['name'], + 'ServiceAttribute': ['name'], + 'ServiceDependency': ['provider_service'], + 'ServiceMonitoringAgentInfo': ['name'], + 'ServicePrivilege': [], + 'ServiceRole': [], + 'Site': ['name'], + 'SiteDeployment': ['site', 'deployment'], + 'SitePrivilege': ['site', 'role'], + 'SiteRole': [], + 'Slice': ['name'], + 'SlicePrivilege': [], + 'SliceRole': [], + 'Tag': ['name'], + 'InterfaceType': ['name'], + 'ServiceInterface': ['service', 'interface_type'], + 'ServiceInstance': ['name'], + 'ServiceInstanceLink': ['provider_service_instance'], + 'ServiceInstanceAttribute': ['name'], + 'TenantWithContainer': ['name'], + 'XOS': ['name'], + 'XOSGuiExtension': ['name'], +} \ No newline at end of file diff --git a/src/tosca-loader/src/grpc_client/__init__.py b/src/tosca-loader/src/grpc_client/__init__.py new file mode 100644 index 0000000..d4e8062 --- /dev/null +++ b/src/tosca-loader/src/grpc_client/__init__.py @@ -0,0 +1,16 @@ + +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + diff --git a/src/tosca-loader/src/grpc_client/main.py b/src/tosca-loader/src/grpc_client/main.py new file mode 100644 index 0000000..3633797 --- /dev/null +++ b/src/tosca-loader/src/grpc_client/main.py @@ -0,0 +1,80 @@ + +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import functools +from xosapi.xos_grpc_client import SecureClient, InsecureClient +from twisted.internet import defer +from resources import RESOURCES +from xosconfig import Config +from twisted.internet import reactor + +from xosconfig import Config +from multistructlog import create_logger +log = create_logger(Config().get('logging')) + +class GRPC_Client: + def __init__(self): + self.client = None + + insecure = Config.get('gprc_endpoint') + secure = Config.get('gprc_endpoint') + + self.grpc_secure_endpoint = secure + ":50051" + self.grpc_insecure_endpoint = insecure + ":50055" + + def setup_resources(self, client, key, deferred, recipe): + log.info("[XOS-TOSCA] Loading resources for authenticated user") + if key not in RESOURCES: + RESOURCES[key] = {} + for k in client.xos_orm.all_model_names: + RESOURCES[key][k] = getattr(client.xos_orm, k) + reactor.callLater(0, deferred.callback, recipe) + + def start(self): + log.info("[XOS-TOSCA] Connecting to xos-core") + + deferred = defer.Deferred() + + if self.client: + self.client.stop() + self.client.session_change = True + + self.client = InsecureClient(endpoint=self.grpc_insecure_endpoint) + self.client.restart_on_disconnect = True + + self.client.set_reconnect_callback(functools.partial(deferred.callback, self.client)) + self.client.start() + + return deferred + + def create_secure_client(self, username, password, recipe): + """ + This method will check if this combination of username/password already has stored orm classes in RESOURCES, otherwise create them + """ + deferred = defer.Deferred() + key = "%s~%s" % (username, password) + if key in RESOURCES: + reactor.callLater(0, deferred.callback, recipe) + else: + local_cert = Config.get('local_cert') + client = SecureClient(endpoint=self.grpc_secure_endpoint, username=username, password=password, cacert=local_cert) + client.restart_on_disconnect = True + # SecureClient is preceeded by an insecure client, so treat all secure clients as previously connected + # See CORD-3152 + client.was_connected = True + client.set_reconnect_callback(functools.partial(self.setup_resources, client, key, deferred, recipe)) + client.start() + return deferred diff --git a/src/tosca-loader/src/grpc_client/models_accessor.py b/src/tosca-loader/src/grpc_client/models_accessor.py new file mode 100644 index 0000000..2c96c48 --- /dev/null +++ b/src/tosca-loader/src/grpc_client/models_accessor.py @@ -0,0 +1,87 @@ + +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from xosconfig import Config +from multistructlog import create_logger +log = create_logger(Config().get('logging')) + +from resources import RESOURCES + +class GRPCModelsAccessor: + """ + This class provide the glue between the models managed by TOSCA and the ones living in xos-core + """ + + @staticmethod + def get_model_from_classname(class_name, data, username, password): + """ + Give a Model Class Name and some data, check if that exits or instantiate a new one + """ + + # NOTE: we need to import this later as it's generated by main.py + from KEYS import TOSCA_KEYS + + # get the key for this model + try: + filter_keys = TOSCA_KEYS[class_name] + except KeyError, e: + raise Exception("[XOS-TOSCA] Model %s doesn't have a tosca_key specified" % (class_name)) + + if len(filter_keys) == 0: + raise Exception("[XOS-TOSCA] Model %s doesn't have a tosca_key specified" % (class_name)) + + filter = {} + for k in filter_keys: + if isinstance(k, str): + try: + filter[k] = data[k] + except KeyError, e: + raise Exception("[XOS-TOSCA] Model %s doesn't have a property for the specified tosca_key (%s)" % (class_name, e)) + elif isinstance(k, list): + # one of they keys in this list has to be set + one_of_key = None + for i in k: + if i in data: + one_of_key = i + one_of_key_val = data[i] + if not one_of_key: + raise Exception("[XOS-TOSCA] Model %s doesn't have a property for the specified tosca_key_one_of (%s)" % (class_name, k)) + else: + filter[one_of_key] = one_of_key_val + + key = "%s~%s" % (username, password) + if not key in RESOURCES: + raise Exception("[XOS-TOSCA] User '%s' does not have ready resources" % username) + if class_name not in RESOURCES[key]: + raise Exception('[XOS-TOSCA] The model you are trying to create (class: %s, properties, %s) is not know by xos-core' % (class_name, str(filter))) + + cls = RESOURCES[key][class_name] + + models = cls.objects.filter(**filter) + + if len(models) == 1: + model = models[0] + log.info("[XOS-Tosca] Model of class %s and properties %s already exist, retrieving instance..." % (class_name, str(filter)), model=model) + elif len(models) == 0: + + if 'must-exist' in data and data['must-exist']: + raise Exception("[XOS-TOSCA] Model of class %s and properties %s has property 'must-exist' but cannot be found" % (class_name, str(filter))) + + model = cls.objects.new() + log.info("[XOS-Tosca] Model (%s) is new, creating new instance..." % str(filter)) + else: + raise Exception("[XOS-Tosca] Model of class %s and properties %s has multiple instances, I can't handle it" % (class_name, str(filter))) + + return model \ No newline at end of file diff --git a/src/tosca-loader/src/grpc_client/resources.py b/src/tosca-loader/src/grpc_client/resources.py new file mode 100644 index 0000000..cfe3e2e --- /dev/null +++ b/src/tosca-loader/src/grpc_client/resources.py @@ -0,0 +1,18 @@ + +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# NOTE will add all the resources in this dictionary +RESOURCES = {} \ No newline at end of file diff --git a/src/tosca-loader/src/main.py b/src/tosca-loader/src/main.py new file mode 100644 index 0000000..6b0359d --- /dev/null +++ b/src/tosca-loader/src/main.py @@ -0,0 +1,64 @@ + +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +from xosconfig import Config +from multistructlog import create_logger + +current_dir = os.path.dirname(os.path.realpath(__file__)) +config_file = os.path.join(current_dir, 'xos-tosca.config.yaml') +config_schema = os.path.join(current_dir, 'xos-tosca-config-schema.yaml') + +Config.init(config_file, config_schema) +log = create_logger(Config().get('logging')) + +from grpc_client.main import GRPC_Client +from tosca.generator import TOSCA_Generator +from web_server.main import TOSCA_WebServer +from twisted.internet import defer + + +class Main: + + def __init__(self): + self.grpc_client = None + + def generate_tosca(self, client): + + deferred = defer.Deferred() + + TOSCA_Generator().generate(client) + + return deferred + + def start(self): + log.info("[XOS-TOSCA] Starting") + + # Remove generated TOSCA and KEYS that may have been downloaded by a previous session. This is done here, rather + # than in the generator, to cover the case where the TOSCA engine is restarted and a web request is received + # and processed before generate_tosca() has completed. + TOSCA_Generator().clean() + TOSCA_Generator().clean_keys() + + grpc_setup = GRPC_Client().start() + grpc_setup.addCallback(self.generate_tosca) + + # NOTE that TOSCA_WebServer create a Klein app that call reactor.run() + TOSCA_WebServer() + + +if __name__ == '__main__': + Main().start() \ No newline at end of file diff --git a/src/tosca-loader/src/tosca/__init__.py b/src/tosca-loader/src/tosca/__init__.py new file mode 100644 index 0000000..d4e8062 --- /dev/null +++ b/src/tosca-loader/src/tosca/__init__.py @@ -0,0 +1,16 @@ + +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + diff --git a/src/tosca-loader/src/tosca/custom_types/.gitignore b/src/tosca-loader/src/tosca/custom_types/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/src/tosca-loader/src/tosca/custom_types/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/src/tosca-loader/src/tosca/default.py b/src/tosca-loader/src/tosca/default.py new file mode 100644 index 0000000..4de0975 --- /dev/null +++ b/src/tosca-loader/src/tosca/default.py @@ -0,0 +1,21 @@ + +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +TOSCA_DEFS_DIR = os.path.dirname(os.path.realpath(__file__)) + "/custom_types" +TOSCA_RECIPES_DIR = os.path.dirname(os.path.realpath(__file__)) + "/" +TOSCA_KEYS_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../grpc_client") \ No newline at end of file diff --git a/src/tosca-loader/src/tosca/generator.py b/src/tosca-loader/src/tosca/generator.py new file mode 100644 index 0000000..1c2dccb --- /dev/null +++ b/src/tosca-loader/src/tosca/generator.py @@ -0,0 +1,64 @@ + +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from xosconfig import Config +from multistructlog import create_logger +log = create_logger(Config().get('logging')) + +import os +from default import TOSCA_DEFS_DIR, TOSCA_KEYS_DIR +from xosgenx.generator import XOSProcessor, XOSProcessorArgs +from xosapi.xos_grpc_client import Empty + +current_dir = os.path.dirname(os.path.realpath(__file__)) + +class TOSCA_Generator: + + def clean(self, dir=TOSCA_DEFS_DIR): + filesToRemove = [f for f in os.listdir(dir)] + for f in filesToRemove: + if not f.startswith('.'): + os.remove(dir + '/' + f) + + def clean_keys(self, dir=TOSCA_KEYS_DIR): + keys_fn = os.path.join(dir, "KEYS.py") + if os.path.exists(keys_fn): + os.remove(keys_fn) + + def generate(self, client): + log.info("[XOS-TOSCA] Generating TOSCA") + + try: + xproto = client.utility.GetXproto(Empty()) + args = XOSProcessorArgs(output = TOSCA_DEFS_DIR, + inputs = str(xproto.xproto), + target = os.path.join(current_dir, 'xtarget/tosca.xtarget'), + write_to_file = 'target') + XOSProcessor.process(args) + log.info("[XOS-TOSCA] Recipes generated in %s" % args.output) + except Exception as e: + log.exception("[XOS-TOSCA] Failed to generate TOSCA") + + try: + xproto = client.utility.GetXproto(Empty()) + args = XOSProcessorArgs(output = TOSCA_KEYS_DIR, + inputs = str(xproto.xproto), + target = os.path.join(current_dir, 'xtarget/tosca_keys.xtarget'), + write_to_file = 'single', + dest_file = 'KEYS.py') + XOSProcessor.process(args) + log.info("[XOS-TOSCA] TOSCA Keys generated in %s" % args.output) + except Exception as e: + log.exception("[XOS-TOSCA] Failed to generate TOSCA Keys") diff --git a/src/tosca-loader/src/tosca/parser.py b/src/tosca-loader/src/tosca/parser.py new file mode 100644 index 0000000..3f44c50 --- /dev/null +++ b/src/tosca-loader/src/tosca/parser.py @@ -0,0 +1,270 @@ + +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +from tempfile import NamedTemporaryFile +from xosconfig import Config +from multistructlog import create_logger +log = create_logger(Config().get('logging')) + +from toscaparser.tosca_template import ToscaTemplate, ValidationError +from default import TOSCA_RECIPES_DIR +from grpc_client.resources import RESOURCES +from grpc_client.models_accessor import GRPCModelsAccessor +from grpc._channel import _Rendezvous +import json +import traceback + +class TOSCA_Parser: + + def compute_dependencies(self, template, models_by_name): + """ + NOTE this method is augmenting self.template, isn't there a more explicit way to achieve it? + """ + for nodetemplate in template.nodetemplates: + nodetemplate.dependencies = [] + nodetemplate.dependencies_names = [] + for reqs in nodetemplate.requirements: + for (k,v) in reqs.items(): + name = v["node"] + if (name in models_by_name): + nodetemplate.dependencies.append(models_by_name[name]) + nodetemplate.dependencies_names.append(name) + + # go another level deep, as our requirements can have requirements... + # NOTE do we still need to go deep? + for sd_req in v.get("requirements",[]): + for (sd_req_k, sd_req_v) in sd_req.items(): + name = sd_req_v["node"] + if (name in models_by_name): + nodetemplate.dependencies.append(models_by_name[name]) + nodetemplate.dependencies_names.append(name) + + @staticmethod + def topsort_dependencies(g): + + # Get set of all nodes, including those without outgoing edges + keys = set(g.keys()) + values = set({}) + for v in g.values(): + values = values | set(v.dependencies_names) + + all_nodes = list(keys | values) + steps = all_nodes + + + # Final order + order = [] + + # DFS stack, not using recursion + stack = [] + + # Unmarked set + unmarked = all_nodes + + # visiting = [] - skip, don't expect 1000s of nodes, |E|/|V| is small + + while unmarked: + stack.insert(0, unmarked[0]) # push first unmarked + + while (stack): + n = stack[0] + add = True + try: + for m in g[n].dependencies_names: + if (m in unmarked): + add = False + stack.insert(0, m) + except KeyError: + pass + if (add): + if (n in steps and n not in order): + order.append(n) + item = stack.pop(0) + try: + unmarked.remove(item) + except ValueError: + pass + + noorder = list(set(steps) - set(order)) + return order + noorder + + @staticmethod + def populate_model(model, data): + for k,v in data.iteritems(): + # NOTE must-exists is a TOSCA implementation choice, remove it before saving the model + if k != "must-exist": + try: + setattr(model, k, v) + except TypeError, e: + raise Exception('Failed to set %s on field %s for class %s, Exception was: "%s"' % (v, k, model.model_name, e)) + return model + + @staticmethod + def _translate_exception(msg): + readable = [] + for line in msg.splitlines(): + if line.strip().startswith('MissingRequiredFieldError') or \ + line.strip().startswith('UnknownFieldError') or \ + line.strip().startswith('ImportError') or \ + line.strip().startswith('InvalidTypeError') or \ + line.strip().startswith('TypeMismatchError'): + readable.append(line) + + if len(readable) > 0: + return '\n'.join(readable) + '\n' + else: + return msg + + @staticmethod + def get_tosca_models_by_name(template): + models_by_name = {} + for node in template.nodetemplates: + models_by_name[node.name] = node + return models_by_name + + @staticmethod + def get_ordered_models_template(ordered_models_name, templates_by_model_name): + ordered_models_templates = [] + for name in ordered_models_name: + if name in templates_by_model_name: + ordered_models_templates.append(templates_by_model_name[name]) + return ordered_models_templates + + @staticmethod + def populate_dependencies(model, requirements, saved_models): + for dep in requirements: + class_name = dep.keys()[0] + related_model = saved_models[dep[class_name]['node']] + setattr(model, "%s_id" % class_name, related_model.id) + return model + + @staticmethod + def add_dependencies(data, requirements, saved_models): + for dep in requirements: + class_name = dep.keys()[0] + related_model = saved_models[dep[class_name]['node']] + data["%s_id" % class_name] = related_model.id + return data + + def __init__(self, recipe, username, password, **kwargs): + + self.delete = False + if 'delete' in kwargs: + self.delete = True + + # store username/password combination to read resources + self.username = username + self.password = password + + # the template returned by TOSCA-Parser + self.template = None + # dictionary containing the models in the recipe and their template + self.templates_by_model_name = None + # list of models ordered by requirements + self.ordered_models_name = [] + # dictionary containing the saved model + self.saved_model_by_name = {} + + self.ordered_models_template = [] + + self.recipe = recipe + + def execute(self): + + try: + # [] save the recipe to a tmp file + with NamedTemporaryFile(delete=False, suffix=".yaml", dir=TOSCA_RECIPES_DIR) as recipe_file: + try: + recipe_file.write(self.recipe) + recipe_file.close() + + # [] parse the recipe with TOSCA Parse + self.template = ToscaTemplate(recipe_file.name) + finally: + # [] Make sure the temporary file is cleaned up + os.remove(recipe_file.name) + + # [] get all models in the recipe + self.templates_by_model_name = self.get_tosca_models_by_name(self.template) + # [] compute requirements + self.compute_dependencies(self.template, self.templates_by_model_name) + # [] topsort requirements + self.ordered_models_name = self.topsort_dependencies(self.templates_by_model_name) + # [] topsort templates + self.ordered_models_template = self.get_ordered_models_template(self.ordered_models_name, self.templates_by_model_name) + + for recipe in self.ordered_models_template: + try: + # get properties from tosca + if not 'properties' in recipe.templates[recipe.name]: + data = {} + else: + data = recipe.templates[recipe.name]['properties'] + if data == None: + data = {} + # [] get model by class name + class_name = recipe.type.replace("tosca.nodes.", "") + + # augemnt data with relations + data = self.add_dependencies(data, recipe.requirements, self.saved_model_by_name) + + model = GRPCModelsAccessor.get_model_from_classname(class_name, data, self.username, self.password) + # [] populate model with data + model = self.populate_model(model, data) + # [] check if the model has requirements + # [] if it has populate them + model = self.populate_dependencies(model, recipe.requirements, self.saved_model_by_name) + # [] save, update or delete + + reference_only = False + if 'must-exist' in data: + reference_only = True + + if self.delete and not model.is_new and not reference_only: + log.info("[XOS-Tosca] Deleting model %s[%s]" % (class_name, model.id)) + model.delete() + elif not self.delete: + log.info("[XOS-Tosca] Saving model %s[%s]" % (class_name, model.id)) + model.save() + + + self.saved_model_by_name[recipe.name] = model + except Exception, e: + log.exception("[XOS-TOSCA] Failed to save model: %s [%s]" % (class_name, recipe.name)) + raise e + + except ValidationError as e: + if e.message: + exception_msg = TOSCA_Parser._translate_exception(e.message) + else: + exception_msg = TOSCA_Parser._translate_exception(str(e)) + raise Exception(exception_msg) + + except _Rendezvous, e: + try: + details = json.loads(e._state.details) + exception_msg = details["error"] + if "specific_error" in details: + exception_msg = "%s: %s" % (exception_msg, details["specific_error"]) + except Exception: + exception_msg = e._state.details + raise Exception(exception_msg) + except Exception, e: + log.exception(e) + raise Exception(e) + + diff --git a/src/tosca-loader/src/tosca/xtarget/tosca.xtarget b/src/tosca-loader/src/tosca/xtarget/tosca.xtarget new file mode 100644 index 0000000..f0eb78b --- /dev/null +++ b/src/tosca-loader/src/tosca/xtarget/tosca.xtarget @@ -0,0 +1,72 @@ + +{% for m in proto.messages %} + +tosca_definitions_version: tosca_simple_yaml_1_0 + +node_types: + + # Example usage: + # + # : + # type: tosca.nodes.{{ m.name }} + # properties: + # must-exist: true # optional to reference models created in other recipes +{%- for f in (m.fields + xproto_base_fields(m, proto.message_table)) | sort(attribute='name') %} +{%- if not f.link and xproto_tosca_required(f.options.null, f.options.blank, f.options.default) %} + # {{ f.name }}: +{%- endif -%} +{%- endfor %} + + + + tosca.nodes.{{ m.name }}: + derived_from: tosca.nodes.Root + description: {% if m.options.description -%}{{ m.options.description}}{% else%}"An XOS {{ m.name }}"{%- endif %} + capabilities: + {{ m.name|lower }}: + type: tosca.capabilities.xos.{{ m.name }} + properties: + must-exist: + type: boolean + default: false + description: Allow to reference existing models in TOSCA recipes + {% for f in (m.fields + xproto_base_fields(m, proto.message_table)) | sort(attribute='name') %} + {%- if not f.link -%} + {{ f.name }}: + type: {{ xproto_tosca_field_type(f.type) }} + required: {{ xproto_tosca_required(f.options.null, f.options.blank, f.options.default) }} + description: {{ f.options.help_text }} + {% endif %} + {%- endfor %} + + {% for l in m.links %} + {%- if l.link_type == "manytoone" -%} + + + # Identify a {{ l.peer.name }} that belongs to a {{ m.name }} + # + # example usage: + # requirements: + # - {{ l.src_port }}: + # node: : + # relationship: tosca.relationships.BelongsToOne + + tosca.relationships.BelongsToOne: + derived_from: tosca.relationships.Root + valid_target_types: [ tosca.capabilities.xos.{{ l.peer.name }} ] + {%- endif%} + {%- if l.link_type == "manytomany" -%} + tosca.relationships.BelongsToMany: + derived_from: tosca.relationships.Root + valid_target_types: [ tosca.capabilities.xos.{{ l.peer.name }} ] + {%- endif%} + {% endfor %} + + tosca.capabilities.xos.{{ m.name }}: + derived_from: tosca.capabilities.Root + description: {{ m.name }} + ++++ {{ m.name }}.yaml + +{%- endfor %} + diff --git a/src/tosca-loader/src/tosca/xtarget/tosca_keys.xtarget b/src/tosca-loader/src/tosca/xtarget/tosca_keys.xtarget new file mode 100644 index 0000000..9569061 --- /dev/null +++ b/src/tosca-loader/src/tosca/xtarget/tosca_keys.xtarget @@ -0,0 +1,5 @@ +TOSCA_KEYS = { +{%- for m in proto.messages %} + '{{ m.name }}': {{ xproto_fields_to_tosca_keys(m.fields + xproto_base_fields(m, proto.message_table), m) }}, +{%- endfor %} +} \ No newline at end of file diff --git a/src/tosca-loader/src/web_server/__init__.py b/src/tosca-loader/src/web_server/__init__.py new file mode 100644 index 0000000..d4e8062 --- /dev/null +++ b/src/tosca-loader/src/web_server/__init__.py @@ -0,0 +1,16 @@ + +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + diff --git a/src/tosca-loader/src/web_server/main.py b/src/tosca-loader/src/web_server/main.py new file mode 100644 index 0000000..9a6c9b3 --- /dev/null +++ b/src/tosca-loader/src/web_server/main.py @@ -0,0 +1,106 @@ + +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from xosconfig import Config +from multistructlog import create_logger +log = create_logger(Config().get('logging')) + +from grpc_client.main import GRPC_Client +from klein import Klein +import os +from tosca.parser import TOSCA_Parser +from tosca.default import TOSCA_DEFS_DIR +import json + +BANNER = """ + _ ______ _____ __________ _____ _________ + | |/ / __ \/ ___/ /_ __/ __ \/ ___// ____/ | + | / / / /\__ \ / / / / / /\__ \/ / / /| | + / / /_/ /___/ / / / / /_/ /___/ / /___/ ___ | +/_/|_\____//____/ /_/ \____//____/\____/_/ |_| +""" + +class TOSCA_WebServer: + + current_dir = os.path.dirname(os.path.realpath(__file__)) + template_dir = os.path.join(current_dir, 'templates/') + + app = Klein() + + def execute_tosca(self, recipe): + self.parser.execute() + if self.parser.delete: + response_text = "Deleted models: %s" % str(self.parser.ordered_models_name) + else: + response_text = "Created models: %s" % str(self.parser.ordered_models_name) + return response_text + + def errorCallback(self, failure, request): + request.setResponseCode(500) + try: + f = failure.getErrorMessage() + if f.startswith("XOSPermissionDenied"): + request.setResponseCode(401) + log.info("[XOS-TOSCA] Error while loading TOSCA: \n\n", failure=f) + return f + except Exception: + log.info("[XOS-TOSCA] Fatal Error: \n\n", failure=failure) + return "Internal server error, please report this along with the failed recipe." + + @app.route('/', methods=['GET']) + def index(self, request): + request.responseHeaders.addRawHeader(b"content-type", b"application/json") + tosca_defs = [f for f in os.listdir(TOSCA_DEFS_DIR) if not f.startswith('.')] + + response = {} + for d in tosca_defs: + name = d.replace('.yaml', '') + response[name] = "/custom_type/%s" % name + return json.dumps(response) + + @app.route("/custom_type/") + def custom_type(self, request, name): + request.responseHeaders.addRawHeader(b"content-type", b"text/plain") + custom_type = open(TOSCA_DEFS_DIR + '/' + name + '.yaml').read() + return custom_type + + @app.route('/run', methods=['POST']) + def run(self, request): + recipe = request.content.read() + headers = request.getAllHeaders() + username = headers['xos-username'] + password = headers['xos-password'] + + d = GRPC_Client().create_secure_client(username, password, recipe) + self.parser = TOSCA_Parser(recipe, username, password) + tosca_execution = d.addCallback(self.execute_tosca) + tosca_execution.addErrback(self.errorCallback, request) + return d + + @app.route('/delete', methods=['POST']) + def delete(self, request): + recipe = request.content.read() + headers = request.getAllHeaders() + username = headers['xos-username'] + password = headers['xos-password'] + + d = GRPC_Client().create_secure_client(username, password, recipe) + self.parser = TOSCA_Parser(recipe, username, password, delete=True) + tosca_execution = d.addCallback(self.execute_tosca) + tosca_execution.addErrback(self.errorCallback, request) + return d + + def __init__(self): + self.app.run('0.0.0.0', '9102') \ No newline at end of file diff --git a/src/tosca-loader/src/xos-tosca-config-schema.yaml b/src/tosca-loader/src/xos-tosca-config-schema.yaml new file mode 100644 index 0000000..db68a43 --- /dev/null +++ b/src/tosca-loader/src/xos-tosca-config-schema.yaml @@ -0,0 +1,28 @@ + +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +map: + name: + type: str + required: True + gprc_endpoint: + type: str + required: True + local_cert: + type: str + required: True + logging: + type: any diff --git a/src/tosca-loader/test/helpers.py b/src/tosca-loader/test/helpers.py new file mode 100644 index 0000000..2a07499 --- /dev/null +++ b/src/tosca-loader/test/helpers.py @@ -0,0 +1,21 @@ +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from xosconfig import Config +current_dir = os.path.dirname(os.path.realpath(__file__)) +config_file = os.path.join(current_dir, 'test_config.yaml') +config_schema = os.path.join(current_dir, '../src/xos-tosca-config-schema.yaml') +Config.clear() +Config.init(config_file, config_schema) \ No newline at end of file diff --git a/src/tosca-loader/test/out/.gitignore b/src/tosca-loader/test/out/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/src/tosca-loader/test/out/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/src/tosca-loader/test/test_config.yaml b/src/tosca-loader/test/test_config.yaml new file mode 100644 index 0000000..698538c --- /dev/null +++ b/src/tosca-loader/test/test_config.yaml @@ -0,0 +1,13 @@ +name: xos-tosca +gprc_endpoint: "xos-core" +local_cert: /usr/local/share/ca-certificates/local_certs.crt +logging: + version: 1 + handlers: + console: + class: logging.StreamHandler + loggers: + 'multistructlog': + handlers: + - console + level: ERROR \ No newline at end of file diff --git a/src/tosca-loader/test/test_grpc_models_accessor.py b/src/tosca-loader/test/test_grpc_models_accessor.py new file mode 100644 index 0000000..a385f86 --- /dev/null +++ b/src/tosca-loader/test/test_grpc_models_accessor.py @@ -0,0 +1,221 @@ + +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from helpers import * +import unittest +from mock import patch, MagicMock +from grpc_client.models_accessor import GRPCModelsAccessor +from grpc_client.resources import RESOURCES +from grpc_client.KEYS import TOSCA_KEYS + +class FakeObj: + new = None + filter = None + +class FakeResource: + objects = FakeObj + +class FakeModel: + pass + +class FakeExistingModel: + pass + +mock_resources = { + 'username~pass': { + 'test-model': FakeResource, + 'single-key': FakeResource, + 'double-key': FakeResource, + 'one-of-key': FakeResource + } +} + +mock_keys = { + 'i-do-not-exists': ['name'], + 'test-model': ['name'], + 'empty-key': [], + 'single-key': ['fake_key'], + 'double-key': ['key_1', 'key_2'], + 'one-of-key': ['key_1', ['key_2', 'key_3']], +} + +USERNAME = 'username' +PASSWORD = 'pass' + +class GRPCModelsAccessor_Create_or_update_Test(unittest.TestCase): + + @patch.dict(TOSCA_KEYS, mock_keys, clear=True) + def test_unkown_user(self): + """ + [GRPCModelsAccessor] get_model_from_classname: If a user does not have orm classes, raise + """ + data = { + "name": "test" + } + with self.assertRaises(Exception) as e: + GRPCModelsAccessor.get_model_from_classname('i-do-not-exists', data, USERNAME, PASSWORD) + self.assertEqual(e.exception.message, "[XOS-TOSCA] User 'username' does not have ready resources") + + @patch.dict(RESOURCES, mock_resources, clear=True) + @patch.dict(TOSCA_KEYS, mock_keys, clear=True) + def test_unkown_module(self): + """ + [GRPCModelsAccessor] get_model_from_classname: If a model is not know by the grpc api, raise + """ + data = { + "name": "test" + } + with self.assertRaises(Exception) as e: + GRPCModelsAccessor.get_model_from_classname('i-do-not-exists', data, USERNAME, PASSWORD) + self.assertEqual(e.exception.message, "[XOS-TOSCA] The model you are trying to create (class: i-do-not-exists, properties, {'name': 'test'}) is not know by xos-core") + + def test_unkown_tosca_key(self): + """ + [GRPCModelsAccessor] get_model_from_classname: If a model does not have a tosca_key, raise + """ + data = { + "name": "test" + } + with self.assertRaises(Exception) as e: + GRPCModelsAccessor.get_model_from_classname('no-key', data, USERNAME, PASSWORD) + self.assertEqual(e.exception.message, "[XOS-TOSCA] Model no-key doesn't have a tosca_key specified") + + @patch.dict(TOSCA_KEYS, mock_keys, clear=True) + def test_empty_tosca_key(self): + """ + [GRPCModelsAccessor] get_model_from_classname: If a model does not have a tosca_key, raise + """ + data = { + "name": "test" + } + with self.assertRaises(Exception) as e: + GRPCModelsAccessor.get_model_from_classname('empty-key', data, USERNAME, PASSWORD) + self.assertEqual(e.exception.message, "[XOS-TOSCA] Model empty-key doesn't have a tosca_key specified") + + @patch.dict(TOSCA_KEYS, mock_keys, clear=True) + def test_tosca_key_are_defined(self): + """ + [GRPCModelsAccessor] get_model_from_classname: a model should have a property for it's tosca_key + """ + data = { + "name": "test", + } + with self.assertRaises(Exception) as e: + GRPCModelsAccessor.get_model_from_classname('single-key', data, USERNAME, PASSWORD) + self.assertEqual(e.exception.message, "[XOS-TOSCA] Model single-key doesn't have a property for the specified tosca_key ('fake_key')") + + @patch.object(FakeResource.objects, "filter") + @patch.object(FakeResource.objects, "new", MagicMock(return_value=FakeModel)) + @patch.dict(TOSCA_KEYS, mock_keys, clear=True) + def test_composite_key(self, mock_filter): + """ + [GRPCModelsAccessor] get_model_from_classname: should use a composite key to lookup a model + """ + data = { + "name": "test", + "key_1": "key1", + "key_2": "key2" + } + with patch.dict(RESOURCES, mock_resources, clear=True): + model = GRPCModelsAccessor.get_model_from_classname('double-key', data, USERNAME, PASSWORD) + mock_filter.assert_called_with(key_1="key1", key_2="key2") + self.assertEqual(model, FakeModel) + + @patch.object(FakeResource.objects, "filter") + @patch.object(FakeResource.objects, "new", MagicMock(return_value=FakeModel)) + @patch.dict(TOSCA_KEYS, mock_keys, clear=True) + def test_one_of_key(self, mock_filter): + """ + [GRPCModelsAccessor] get_model_from_classname: should use a composite with one_of key to lookup a model + """ + # NOTE it should be valid for items with either one of the keys + data2 = { + "name": "test", + "key_1": "key1", + "key_2": "key2" + } + with patch.dict(RESOURCES, mock_resources, clear=True): + model = GRPCModelsAccessor.get_model_from_classname('one-of-key', data2, USERNAME, PASSWORD) + mock_filter.assert_called_with(key_1="key1", key_2="key2") + self.assertEqual(model, FakeModel) + + data3 = { + "name": "test", + "key_1": "key1", + "key_3": "key3" + } + with patch.dict(RESOURCES, mock_resources, clear=True): + model = GRPCModelsAccessor.get_model_from_classname('one-of-key', data3, USERNAME, PASSWORD) + mock_filter.assert_called_with(key_1="key1", key_3="key3") + self.assertEqual(model, FakeModel) + + @patch.object(FakeResource.objects, "filter") + @patch.object(FakeResource.objects, "new", MagicMock(return_value=FakeModel)) + @patch.dict(TOSCA_KEYS, mock_keys, clear=True) + def test_one_of_key_error(self, mock_filter): + data = { + "name": "test", + "key_1": "key1" + } + with self.assertRaises(Exception) as e: + GRPCModelsAccessor.get_model_from_classname('one-of-key', data, USERNAME, PASSWORD) + self.assertEqual(e.exception.message, "[XOS-TOSCA] Model one-of-key doesn't have a property for the specified tosca_key_one_of (['key_2', 'key_3'])") + + @patch.object(FakeResource.objects, "filter") + @patch.object(FakeResource.objects, "new", MagicMock(return_value=FakeModel)) + @patch.dict(TOSCA_KEYS, mock_keys, clear=True) + def test_new_model(self, mock_filter): + """ + [GRPCModelsAccessor] get_model_from_classname: should create a new model + """ + data = { + "name": "test", + "fake_key": "key" + } + with patch.dict(RESOURCES, mock_resources, clear=True): + model = GRPCModelsAccessor.get_model_from_classname('single-key', data, USERNAME, PASSWORD) + mock_filter.assert_called_with(fake_key="key") + self.assertEqual(model, FakeModel) + + @patch.object(FakeResource.objects, "filter", MagicMock(return_value=[FakeExistingModel])) + @patch.dict(TOSCA_KEYS, mock_keys, clear=True) + def test_existing_model(self): + """ + [GRPCModelsAccessor] get_model_from_classname: should update an existing model + """ + data = { + "name": "test", + "fake_key": "key" + } + with patch.dict(RESOURCES, mock_resources, clear=True): + model = GRPCModelsAccessor.get_model_from_classname('single-key', data, USERNAME, PASSWORD) + self.assertEqual(model, FakeExistingModel) + + @patch.object(FakeResource.objects, "filter", MagicMock(return_value=['a', 'b'])) + @patch.dict(TOSCA_KEYS, mock_keys, clear=True) + def test_multiple_models(self): + """ + [GRPCModelsAccessor] get_model_from_classname: should raise an exception if multiple instances are found + """ + data = { + "name": "test" + } + with patch.dict(RESOURCES, mock_resources, clear=True): + with self.assertRaises(Exception) as e: + GRPCModelsAccessor.get_model_from_classname('test-model', data, USERNAME, PASSWORD) + self.assertEqual(e.exception.message, "[XOS-Tosca] Model of class test-model and properties {'name': 'test'} has multiple instances, I can't handle it") + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/src/tosca-loader/test/test_tosca_generator.py b/src/tosca-loader/test/test_tosca_generator.py new file mode 100644 index 0000000..f76dce5 --- /dev/null +++ b/src/tosca-loader/test/test_tosca_generator.py @@ -0,0 +1,80 @@ + +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from helpers import * +import unittest +import os +from xosgenx.generator import XOSProcessor, XOSProcessorArgs + +current_dir = os.path.dirname(os.path.realpath(__file__)) +OUTPUT_DIR = os.path.join(current_dir, 'out'); +print OUTPUT_DIR + +class TOSCA_Generator_Test(unittest.TestCase): + + def test_generate_basic_tosca(self): + """ + [TOSCA_xtarget] Should generate a basic TOSCA recipe + """ + xproto = \ + """ + option app_label = "core"; + + message XOSGuiExtension (XOSBase) { + option verbose_name="XOS GUI Extension"; + option description="This model holds the instruction to load an extension in the GUI"; + required string name = 1 [max_length = 200, content_type = "stripped", blank = False, help_text = "Name of the GUI Extensions", null = False, db_index = False]; + required string files = 2 [max_length = 1024, content_type = "stripped", blank = False, help_text = "List of comma separated file composing the view", null = False, db_index = False]; + } + """ + args = XOSProcessorArgs(inputs = xproto, + target = os.path.join(current_dir, '../src/tosca/xtarget/tosca.xtarget'), + output = OUTPUT_DIR, + write_to_file = "single", + dest_file = "basic.yaml", + quiet = False) + output = XOSProcessor.process(args) + self.assertIn("name:", output) + self.assertIn("files:", output) + + def test_generate_inherithed_tosca(self): + """ + [TOSCA_xtarget] Should generate a TOSCA recipe for a models that inherits from another model + """ + xproto = \ + """ + option app_label = "core"; + + message Service (XosBase) { + option verbose_name="Basic Service"; + required string name = 1 [max_length = 200, content_type = "stripped", blank = False, null = False, db_index = False]; + } + + message MyService (Service) { + option verbose_name="Extending service"; + required string prop = 1 [max_length = 200, content_type = "stripped", blank = False, null = False, db_index = False]; + } + """ + args = XOSProcessorArgs(inputs = xproto, + target = os.path.join(current_dir, '../src/tosca/xtarget/tosca.xtarget'), + output = OUTPUT_DIR, + write_to_file = 'target', + quiet = False) + output = XOSProcessor.process(args) + self.assertEqual(output.count("name:"), 4) + self.assertIn("prop:", output) + +if __name__ == '__main__': + unittest.main() diff --git a/src/tosca-loader/test/test_tosca_parser.py b/src/tosca-loader/test/test_tosca_parser.py new file mode 100644 index 0000000..9641380 --- /dev/null +++ b/src/tosca-loader/test/test_tosca_parser.py @@ -0,0 +1,221 @@ + +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from helpers import * +import unittest +import os +from tosca.parser import TOSCA_Parser + +class TOSCA_Parser_Test(unittest.TestCase): + + def test_get_tosca_models_by_name(self): + """ + [TOSCA_Parser] get_tosca_models_by_name: should extract models from the TOSCA recipe and store them in a dict + """ + class FakeNode: + def __init__(self, name): + self.name = name + + class FakeTemplate: + nodetemplates = [ + FakeNode('model1'), + FakeNode('model2') + ] + + + res = TOSCA_Parser.get_tosca_models_by_name(FakeTemplate) + self.assertIsInstance(res['model1'], FakeNode) + self.assertIsInstance(res['model2'], FakeNode) + + self.assertEqual(res['model1'].name, 'model1') + self.assertEqual(res['model2'].name, 'model2') + + def test_populate_dependencies(self): + """ + [TOSCA_Parser] populate_dependencies: if a recipe has dependencies, it should find the ID of the requirements and add it to the model + """ + class FakeRecipe: + requirements = [ + { + 'site': { + 'node': 'site_onlab', + 'relationship': 'tosca.relationship.BelongsToOne' + } + } + ] + + class FakeSite: + id = 1 + name = 'onlab' + + class FakeModel: + name = 'test@opencord.org' + + saved_models = { + 'site_onlab': FakeSite + } + + model = TOSCA_Parser.populate_dependencies(FakeModel, FakeRecipe.requirements, saved_models) + self.assertEqual(model.site_id, 1) + + def test_get_ordered_models_template(self): + """ + [TOSCA_Parser] get_ordered_models_template: Create a list of templates based on topsorted models + """ + ordered_models = ['foo', 'bar'] + + templates = { + 'foo': 'foo_template', + 'bar': 'bar_template' + } + + ordered_templates = TOSCA_Parser.get_ordered_models_template(ordered_models, templates) + + self.assertEqual(ordered_templates[0], 'foo_template') + self.assertEqual(ordered_templates[1], 'bar_template') + + def test_topsort_dependencies(self): + """ + [TOSCA_Parser] topsort_dependencies: Create a list of models based on dependencies + """ + class FakeTemplate: + def __init__(self, name, deps): + self.name = name + self.dependencies_names = deps + + + templates = { + 'deps': FakeTemplate('deps', ['main']), + 'main': FakeTemplate('main', []), + } + + sorted = TOSCA_Parser.topsort_dependencies(templates) + + self.assertEqual(sorted[0], 'main') + self.assertEqual(sorted[1], 'deps') + + def test_compute_dependencies(self): + """ + [TOSCA_Parser] compute_dependencies: augment the TOSCA nodetemplate with information on requirements (aka related models) + """ + + parser = TOSCA_Parser('', 'user', 'pass') + + class FakeNode: + def __init__(self, name, requirements): + self.name = name + self.requirements = requirements + + main = FakeNode('main', []) + dep = FakeNode('dep', [{'relation': {'node': 'main'}}]) + + models_by_name = { + 'main': main, + 'dep': dep + } + + class FakeTemplate: + nodetemplates = [dep, main] + + parser.compute_dependencies(FakeTemplate, models_by_name) + + templates = FakeTemplate.nodetemplates + augmented_dep = templates[0] + augmented_main = templates[1] + + self.assertIsInstance(augmented_dep.dependencies[0], FakeNode) + self.assertEqual(augmented_dep.dependencies[0].name, 'main') + self.assertEqual(augmented_dep.dependencies_names[0], 'main') + + self.assertEqual(len(augmented_main.dependencies), 0) + self.assertEqual(len(augmented_main.dependencies_names), 0) + + def test_populate_model(self): + """ + [TOSCA_Parser] populate_model: augment the GRPC model with data from TOSCA + """ + class FakeModel: + pass + + data = { + 'name': 'test', + 'foo': 'bar', + 'number': 1 + } + + model = TOSCA_Parser.populate_model(FakeModel, data) + + self.assertEqual(model.name, 'test') + self.assertEqual(model.foo, 'bar') + self.assertEqual(model.number, 1) + + def test_populate_model_error(self): + """ + [TOSCA_Parser] populate_model: should print a meaningful error message + """ + + class FakeModel: + + model_name = "FakeModel" + + def __setattr__(self, name, value): + if name == 'foo': + raise TypeError('reported exception') + else: + super(FakeModel, self).__setattr__(name, value) + + data = { + 'name': 'test', + 'foo': None, + 'number': 1 + } + + + model = FakeModel() + + with self.assertRaises(Exception) as e: + model = TOSCA_Parser.populate_model(model, data) + self.assertEqual(e.exception.message, 'Failed to set None on field foo for class FakeModel, Exception was: "reported exception"') + + def test_translate_exception(self): + """ + [TOSCA_Parser] translate_exception: convert a TOSCA Parser exception in a user readable string + """ + e = TOSCA_Parser._translate_exception("Non tosca exception") + self.assertEqual(e, "Non tosca exception") + + e = TOSCA_Parser._translate_exception(""" +MissingRequiredFieldError: some message + followed by unreadable + and mystic + python error + starting at line + 38209834 of some file +UnknownFieldError: with some message + followed by useless things +ImportError: with some message + followed by useless things +InvalidTypeError: with some message + followed by useless things +TypeMismatchError: with some message + followed by useless things + """) + self.assertEqual(e, """MissingRequiredFieldError: some message +UnknownFieldError: with some message +ImportError: with some message +InvalidTypeError: with some message +TypeMismatchError: with some message +""") diff --git a/src/tosca-loader/test/test_tosca_parser_e2e.py b/src/tosca-loader/test/test_tosca_parser_e2e.py new file mode 100644 index 0000000..124e160 --- /dev/null +++ b/src/tosca-loader/test/test_tosca_parser_e2e.py @@ -0,0 +1,274 @@ + +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from helpers import * +import unittest +from mock import patch, MagicMock +from tosca.parser import TOSCA_Parser +from grpc_client.resources import RESOURCES + +class FakeObj: + new = None + filter = None + +class FakeModel: + save = None + delete = None + is_new = False + id = 1 + +class FakeGuiExt: + objects = FakeObj + +class FakeSite: + objects = FakeObj + +class FakeInstance: + objects = FakeObj + +class FakeUser: + objects = FakeObj + +class FakeNode: + objects = FakeObj + +USERNAME = "username" +PASSWORD = "pass" + +mock_resources = {} +mock_resources["%s~%s" % (USERNAME, PASSWORD)] = { + 'XOSGuiExtension': FakeGuiExt, + 'Site': FakeSite, + 'User': FakeUser, + 'Instance': FakeInstance, + 'Node': FakeNode +} + +class TOSCA_Parser_E2E(unittest.TestCase): + + @patch.dict(RESOURCES, mock_resources, clear=True) + @patch.object(FakeGuiExt.objects, 'filter', MagicMock(return_value=[FakeModel])) + @patch.object(FakeModel, 'save') + def test_basic_creation(self, mock_save): + """ + [TOSCA_Parser] Should save models defined in a TOSCA recipe + """ + recipe = """ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: Persist xos-sample-gui-extension + +imports: + - custom_types/xosguiextension.yaml + +topology_template: + node_templates: + + # UI Extension + test: + type: tosca.nodes.XOSGuiExtension + properties: + name: test + files: /spa/extensions/test/vendor.js, /spa/extensions/test/app.js +""" + + parser = TOSCA_Parser(recipe, USERNAME, PASSWORD) + + parser.execute() + + # checking that the model has been saved + mock_save.assert_called() + + self.assertIsNotNone(parser.templates_by_model_name['test']) + self.assertEqual(parser.ordered_models_name, ['test']) + + # check that the model was saved with the expected values + saved_model = parser.saved_model_by_name['test'] + self.assertEqual(saved_model.name, 'test') + self.assertEqual(saved_model.files, '/spa/extensions/test/vendor.js, /spa/extensions/test/app.js') + + @patch.dict(RESOURCES, mock_resources, clear=True) + @patch.object(FakeGuiExt.objects, 'filter', MagicMock(return_value=[FakeModel])) + @patch.object(FakeNode.objects, 'filter', MagicMock(return_value=[FakeModel])) + @patch.object(FakeModel, 'delete') + def test_basic_deletion(self, mock_model): + """ + [TOSCA_Parser] Should delete models defined in a TOSCA recipe + """ + recipe = """ + tosca_definitions_version: tosca_simple_yaml_1_0 + + description: Persist xos-sample-gui-extension + + imports: + - custom_types/node.yaml + - custom_types/xosguiextension.yaml + + topology_template: + node_templates: + + should_stay: + type: tosca.nodes.Node + properties: + name: should_stay + must-exist: true + + test: + type: tosca.nodes.XOSGuiExtension + properties: + name: test + files: /spa/extensions/test/vendor.js, /spa/extensions/test/app.js + """ + + parser = TOSCA_Parser(recipe, USERNAME, PASSWORD, delete=True) + + parser.execute() + + # checking that the model has been saved + mock_model.assert_called_once() + + self.assertIsNotNone(parser.templates_by_model_name['test']) + self.assertEqual(parser.ordered_models_name, ['test', 'should_stay']) + + @patch.dict(RESOURCES, mock_resources, clear=True) + @patch.object(FakeSite.objects, 'filter', MagicMock(return_value=[FakeModel])) + @patch.object(FakeUser.objects, 'filter', MagicMock(return_value=[FakeModel])) + @patch.object(FakeModel, 'save') + def test_related_models_creation(self, mock_save): + """ + [TOSCA_Parser] Should save related models defined in a TOSCA recipe + """ + + recipe = """ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: Create a new site with one user + +imports: + - custom_types/user.yaml + - custom_types/site.yaml + +topology_template: + node_templates: + + # Site + site_onlab: + type: tosca.nodes.Site + properties: + name: Open Networking Lab + site_url: http://onlab.us/ + hosts_nodes: True + + # User + usertest: + type: tosca.nodes.User + properties: + username: test@opencord.org + email: test@opencord.org + password: mypwd + firstname: User + lastname: Test + is_admin: True + requirements: + - site: + node: site_onlab + relationship: tosca.relationships.BelongsToOne +""" + + parser = TOSCA_Parser(recipe, USERNAME, PASSWORD) + + parser.execute() + + self.assertEqual(mock_save.call_count, 2) + + self.assertIsNotNone(parser.templates_by_model_name['site_onlab']) + self.assertIsNotNone(parser.templates_by_model_name['usertest']) + self.assertEqual(parser.ordered_models_name, ['site_onlab', 'usertest']) + + # check that the model was saved with the expected values + saved_site = parser.saved_model_by_name['site_onlab'] + self.assertEqual(saved_site.name, 'Open Networking Lab') + + saved_user = parser.saved_model_by_name['usertest'] + self.assertEqual(saved_user.firstname, 'User') + self.assertEqual(saved_user.site_id, 1) + + @patch.dict(RESOURCES, mock_resources, clear=True) + @patch.object(FakeSite.objects, 'filter', MagicMock(return_value=[])) + def test_must_exist_fail(self): + """ + [TOSCA_Parser] Should throw an error if an object with 'must_exist' does not exist + """ + recipe = """ + tosca_definitions_version: tosca_simple_yaml_1_0 + + description: Create a new site with one user + + imports: + - custom_types/site.yaml + + topology_template: + node_templates: + + # Site + site_onlab: + type: tosca.nodes.Site + properties: + name: Open Networking Lab + must-exist: True + """ + + parser = TOSCA_Parser(recipe, USERNAME, PASSWORD) + + with self.assertRaises(Exception) as e: + parser.execute() + + self.assertEqual(e.exception.message.message, "[XOS-TOSCA] Model of class Site and properties {'name': 'Open Networking Lab'} has property 'must-exist' but cannot be found") + + @patch.dict(RESOURCES, mock_resources, clear=True) + @patch.object(FakeInstance.objects, 'filter', MagicMock(return_value=[FakeModel])) + @patch.object(FakeModel, 'save') + def test_number_param(self, mock_save): + """ + [TOSCA_Parser] Should correctly parse number parameters + """ + recipe = """ + tosca_definitions_version: tosca_simple_yaml_1_0 + + description: Create a new site with one user + + imports: + - custom_types/instance.yaml + + topology_template: + node_templates: + + # Site + instance#test_instance: + type: tosca.nodes.Instance + properties: + name: test_instance + numberCores: 10 + """ + parser = TOSCA_Parser(recipe, USERNAME, PASSWORD) + parser.execute() + + # checking that the model has been saved + mock_save.assert_called() + + # check that the model was saved with the expected values + saved_model = parser.saved_model_by_name['instance#test_instance'] + self.assertEqual(saved_model.name, 'test_instance') + self.assertEqual(saved_model.numberCores, 10) diff --git a/src/tosca-loader/test/tosca/link.yaml b/src/tosca-loader/test/tosca/link.yaml new file mode 100644 index 0000000..f2c000f --- /dev/null +++ b/src/tosca-loader/test/tosca/link.yaml @@ -0,0 +1,160 @@ + +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: Persist xos-sample-gui-extension + +imports: + - custom_types/slice.yaml + - custom_types/network.yaml + - custom_types/networkslice.yaml + - custom_types/networktemplate.yaml + - custom_types/site.yaml + - custom_types/image.yaml + - custom_types/service.yaml + - custom_types/serviceinstance.yaml + - custom_types/serviceinstancelink.yaml + +topology_template: + node_templates: + + service#mcord: + type: tosca.nodes.Service + properties: + name: mcord + + test1: + type: tosca.nodes.ServiceInstance + properties: + name: test1 + + test2: + type: tosca.nodes.ServiceInstance + properties: + name: test2 + + link1: + type: tosca.nodes.ServiceInstanceLink + requirements: + - provider_service_instance: + node: test1 + relationship: tosca.relationships.BelongsToOne + - subscriber_service_instance: + node: test2 + relationship: tosca.relationships.BelongsToOne + + link2: + type: tosca.nodes.ServiceInstanceLink + requirements: + - subscriber_service: + node: service#mcord + relationship: tosca.relationships.BelongsToOne + - provider_service_instance: + node: test2 + relationship: tosca.relationships.BelongsToOne + + # Site + mysite: + type: tosca.nodes.Site + properties: + must-exist: true + name: mysite + + # Images + image#trusty-server-multi-nic: + type: tosca.nodes.Image + properties: + must-exist: true + name: trusty-server-multi-nic + + # slices + slice#slice1: + type: tosca.nodes.Slice + properties: + name: mysite_slice1 + requirements: + - site: + node: mysite + relationship: tosca.relationships.BelongsToOne + - default_image: + node: image#trusty-server-multi-nic + relationship: tosca.relationships.BelongsToOne + + slice#slice2: + type: tosca.nodes.Slice + properties: + name: mysite_slice2 + requirements: + - site: + node: mysite + relationship: tosca.relationships.BelongsToOne + - default_image: + node: image#trusty-server-multi-nic + relationship: tosca.relationships.BelongsToOne + + # networks + + shared_template: + type: tosca.nodes.NetworkTemplate + properties: + must-exist: true + name: shared_template + + network#network1: + type: tosca.nodes.Network + properties: + name: network1 + requirements: + - template: + node: shared_template + relationship: tosca.relationships.BelongsToOne + - owner: + node: slice#slice1 + relationship: tosca.relationships.BelongsToOne + + network#network2: + type: tosca.nodes.Network + properties: + name: network2 + requirements: + - template: + node: shared_template + relationship: tosca.relationships.BelongsToOne + - owner: + node: slice#slice2 + relationship: tosca.relationships.BelongsToOne + + networkslice#slice1_network2: + type: tosca.nodes.NetworkSlice + requirements: + - network: + node: network#network2 + relationship: tosca.relationships.BelongsToOne + - slice: + node: slice#slice1 + relationship: tosca.relationships.BelongsToOne + + networkslice#slice2_network1: + type: tosca.nodes.NetworkSlice + requirements: + - network: + node: network#network1 + relationship: tosca.relationships.BelongsToOne + - slice: + node: slice#slice2 + relationship: tosca.relationships.BelongsToOne + diff --git a/src/tosca-loader/test/tosca/must_exist.yaml b/src/tosca-loader/test/tosca/must_exist.yaml new file mode 100644 index 0000000..2042229 --- /dev/null +++ b/src/tosca-loader/test/tosca/must_exist.yaml @@ -0,0 +1,49 @@ + +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: Persist xos-sample-gui-extension + +imports: + - custom_types/user.yaml + - custom_types/site.yaml + - custom_types/xosguiextension.yaml + +topology_template: + node_templates: + + # Site + site_onlab: + type: tosca.nodes.Site + properties: + name: Open Networking Lab + must-exist: True + + # User + user_test: + type: tosca.nodes.User + properties: + username: test@opencord.org + email: test@opencord.org + password: mypwd + firstname: User + lastname: Test + is_admin: True + requirements: + - site: + node: site_onlab + relationship: tosca.relationships.BelongsToOne \ No newline at end of file diff --git a/src/tosca-loader/test/tosca/test.yaml b/src/tosca-loader/test/tosca/test.yaml new file mode 100644 index 0000000..ff735e0 --- /dev/null +++ b/src/tosca-loader/test/tosca/test.yaml @@ -0,0 +1,57 @@ + +# Copyright 2017-present Open Networking Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# curl -H "xos-username: admin@opencord.org" -H "xos-password: letmein" -X POST --data-binary @test.yaml http://192.168.99.100:30007/run +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: Persist xos-sample-gui-extension + +imports: + - custom_types/user.yaml + - custom_types/site.yaml + - custom_types/xosguiextension.yaml + +topology_template: + node_templates: + + # UI Extension + test: + type: tosca.nodes.XOSGuiExtension + properties: + name: test + files: /spa/extensions/test/vendor.js, /spa/extensions/test/app.js + + # Site + site_onlab: + type: tosca.nodes.Site + properties: + name: Open Networking Lab + site_url: http://onlab.us/ + hosts_nodes: True + + # User + user_test: + type: tosca.nodes.User + properties: + username: test@opencord.org + email: test@opencord.org + password: mypwd + firstname: User + lastname: Test + is_admin: True + requirements: + - site: + node: site_onlab + relationship: tosca.relationships.BelongsToOne \ No newline at end of file diff --git a/src/tosca-loader/unittest.cfg b/src/tosca-loader/unittest.cfg new file mode 100644 index 0000000..f7d64b7 --- /dev/null +++ b/src/tosca-loader/unittest.cfg @@ -0,0 +1,7 @@ +[unittest] +plugins=nose2.plugins.junitxml + +[coverage] +always-on = True +coverage-report = term +coverage-report = xml -- 2.16.6