skip, show_stopper, all tested

* skip, If you want to skip any testcase you can just give "True"
  for this.

* show_stopper, testcase failed meaning we should not continue.
  If you are expecting too many failures because of this failure,
  you mark it as show_stopper: True

* all is special layer, that means all layers.

* -d/--delegate is an option to delegate validation to docker
  containers. docker containers will never use this.

Change-Id: I92b1fc0ac47a6ca3b75e1e2e75c9aea0d206fdaf
Signed-off-by: Naga Sugguna <ns156u@att.com>

diff --git a/.coafile b/.coafile
index 0514a52..d92f581 100644 (file)
--- a/.coafile
+++ b/.coafile
@@ -1,5 +1,6 @@
 [all]
 ignore = .tox/**,
+    .py35/**,
     .git/**,
     .gitignore,
     .gitreview,

diff --git a/.gitignore b/.gitignore
index 2d802ce..2e45b8c 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -42,12 +42,12 @@ bin/
 .project
 .idea
 *~
-pmd-bin-6.15.0*
+pmd-bin-*
+.vscode
 
 .*
 !/.gitignore
 !/.coafile
 !/ui/src/main/webapp/.eslintrc
-
 # git submodule patching mechanism local artifacts
 .submodules_*

diff --git a/bluval/bluval-dummy.yaml b/bluval/bluval-dummy.yaml
index 11e8fc2..4b5fa76 100644 (file)
--- a/bluval/bluval-dummy.yaml
+++ b/bluval/bluval-dummy.yaml
@@ -25,14 +25,18 @@ blueprint:
     hardware: &hardware_dummy
         -
             name: bios_version
-            # something that can be evaluated to true or false
-            condition: true
+            # To skip this test case keep it as True
+            skip: "True"
             what: bios_version
-            # true or false, default is false
-            show_stopper: false
+            # True or False, default is False
+            show_stopper: "False"
         -
             name: hp_baremetal
             what: hp_baremetal
+            # To skip this test case keep it as False
+            skip: "False"
+            # True or False, default is False
+            show_stopper: "False"
     os: &os_base
         -
             name: cyclictest

diff --git a/bluval/bluval.py b/bluval/bluval.py
index 7855816..1580a40 100644 (file)
--- a/bluval/bluval.py
+++ b/bluval/bluval.py
@@ -21,13 +21,31 @@ testcase
 
 import subprocess
 from pathlib import Path
+import sys
+import traceback
 import click
 import yaml
 
+class BluvalError(Exception):
+    """Base class for exceptions in this module."""
+    pass
+
+
+class ShowStopperError(Exception):
+    """Showstopper test case failed"""
+    pass
+
+
 def run_testcase(testcase):
     """Runs a single testcase
     """
-    show_stopper = testcase.get('show_stopper', False)
+    name = testcase.get('name')
+    skip = testcase.get('skip', "False")
+    if skip.lower() == "true":
+        # skip is mentioned and true.
+        print('Skipping {}'.format(name))
+        return
+    show_stopper = testcase.get('show_stopper', "False")
     what = testcase.get('what')
     mypath = Path(__file__).absolute()
     results_path = mypath.parents[2].joinpath("results/"+testcase.get('layer')+"/"+what)
@@ -42,19 +60,16 @@ def run_testcase(testcase):
     # run the test
     args = ["robot", "-V", str(variables_file), "-d", str(results_path), str(test_path)]
 
-    print('Executing testcase {}'.format(testcase['name']))
-    print('          show_stopper {}'.format(show_stopper))
+    print('Executing testcase {}'.format(name))
+    print('show_stopper {}'.format(show_stopper))
     print('Invoking {}'.format(args))
     try:
         status = subprocess.call(args, shell=False)
-        if status != 0 and show_stopper:
-            print('Show stopper testcase failed')
-            return status
+        if status != 0 and show_stopper.lower() == "true":
+            raise ShowStopperError(name)
     except OSError:
-        print('Error while executing {}'.format(args))
-        return -1
-    return status
-
+        #print('Error while executing {}'.format(args))
+        raise BluvalError(OSError)
 
 def validate_layer(blueprint, layer):
     """validates a layer by validating all testcases under that layer
@@ -69,20 +84,56 @@ def validate_blueprint(yaml_loc, layer):
     """Parse yaml file and validates given layer. If no layer given all layers
     validated
     """
-    with open(yaml_loc) as yaml_file:
+    with open(str(yaml_loc)) as yaml_file:
         yamldoc = yaml.safe_load(yaml_file)
     blueprint = yamldoc['blueprint']
-    if layer is None:
+    if layer is None or layer == "all":
         for each_layer in blueprint['layers']:
             validate_layer(blueprint, each_layer)
     else:
         validate_layer(blueprint, layer)
 
 
+def invoke_docker(bluprint, layer):
+    """Start docker container for given layer
+    """
+    cmd = ("docker run"
+           " -v $HOME/.ssh:/root/.ssh"
+           " -v $HOME/.kube/config:/root/.kube/config"
+           " -v $VALIDATION_HOME/tests/variables.yaml:"
+           "/opt/akraino/validation/tests/variables.yaml"
+           " -v $AKRAINO_HOME/results:/opt/akraino/results"
+           " akraino/validation:{0}-latest"
+           " bin/sh -c"
+           " 'cd /opt/akraino/validation "
+           "&& python bluval/bluval.py -l {0} {1}'").format(layer, bluprint)
+    args = [cmd]
+    try:
+        print('Invoking {}'.format(args))
+        subprocess.call(args, shell=True)
+    except OSError:
+        #print('Error while executing {}'.format(args))
+        raise BluvalError(OSError)
+
+
+def invoke_dockers(yaml_loc, layer, blueprint_name):
+    """Parses yaml file and starts docker container for one/all layers
+    """
+    with open(str(yaml_loc)) as yaml_file:
+        yamldoc = yaml.safe_load(yaml_file)
+    blueprint = yamldoc['blueprint']
+    if layer is None or layer == "all":
+        for each_layer in blueprint['layers']:
+            invoke_docker(blueprint_name, each_layer)
+    else:
+        invoke_docker(blueprint_name, layer)
+
+
 @click.command()
 @click.argument('blueprint')
 @click.option('--layer', '-l')
-def main(blueprint, layer):
+@click.option('--delegate', '-d', is_flag=True)
+def main(blueprint, layer, delegate):
     """Takes blueprint name and optional layer. Validates inputs and derives
     yaml location from blueprint name. Invokes validate on blue print.
     """
@@ -90,8 +141,22 @@ def main(blueprint, layer):
     yaml_loc = mypath.parents[0].joinpath('bluval-{}.yaml'.format(blueprint))
     if layer is not None:
         layer = layer.lower()
-    validate_blueprint(yaml_loc, layer)
-
+    try:
+        if delegate is not None:
+            invoke_dockers(yaml_loc, layer, blueprint)
+        else:
+            validate_blueprint(yaml_loc, layer)
+    except ShowStopperError as err:
+        print('ShowStopperError:', err)
+    except BluvalError as err:
+        print('Unexpected BluvalError', err)
+        raise
+    except:
+        print("Exception in user code:")
+        print("-"*60)
+        traceback.print_exc(file=sys.stdout)
+        print("-"*60)
+        raise
 
 if __name__ == "__main__":
     # pylint: disable=no-value-for-parameter