skip, show_stopper, all tested

* skip, If you want to skip any testcase you can just give "True"
  for this.

* show_stopper, testcase failed meaning we should not continue.
  If you are expecting too many failures because of this failure,
  you mark it as show_stopper: True

* all is special layer, that means all layers.

* -d/--delegate is an option to delegate validation to docker
  containers. docker containers will never use this.

Change-Id: I92b1fc0ac47a6ca3b75e1e2e75c9aea0d206fdaf
Signed-off-by: Naga Sugguna <ns156u@att.com>

diff --git a/.coafile b/.coafile
index 0514a52..d92f581 100644 (file)
--- a/.coafile
+++ b/.coafile
@@ -1,5 +1,6 @@
 [all]
 ignore = .tox/**,
 [all]
 ignore = .tox/**,

+    .py35/**,

     .git/**,
     .gitignore,
     .gitreview,
     .git/**,
     .gitignore,
     .gitreview,

diff --git a/.gitignore b/.gitignore
index 2d802ce..2e45b8c 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -42,12 +42,12 @@ bin/
 .project
 .idea
 *~
 .project
 .idea
 *~

-pmd-bin-6.15.0*
+pmd-bin-*
+.vscode

 
 .*
 !/.gitignore
 !/.coafile
 !/ui/src/main/webapp/.eslintrc
 
 .*
 !/.gitignore
 !/.coafile
 !/ui/src/main/webapp/.eslintrc

-

 # git submodule patching mechanism local artifacts
 .submodules_*
 # git submodule patching mechanism local artifacts
 .submodules_*

diff --git a/bluval/bluval-dummy.yaml b/bluval/bluval-dummy.yaml
index 11e8fc2..4b5fa76 100644 (file)
--- a/bluval/bluval-dummy.yaml
+++ b/bluval/bluval-dummy.yaml
@@ -25,14 +25,18 @@ blueprint:
     hardware: &hardware_dummy
         -
             name: bios_version
     hardware: &hardware_dummy
         -
             name: bios_version

-            # something that can be evaluated to true or false
-            condition: true
+            # To skip this test case keep it as True
+            skip: "True"

             what: bios_version
             what: bios_version

-            # true or false, default is false
-            show_stopper: false
+            # True or False, default is False
+            show_stopper: "False"

         -
             name: hp_baremetal
             what: hp_baremetal
         -
             name: hp_baremetal
             what: hp_baremetal

+            # To skip this test case keep it as False
+            skip: "False"
+            # True or False, default is False
+            show_stopper: "False"

     os: &os_base
         -
             name: cyclictest
     os: &os_base
         -
             name: cyclictest

diff --git a/bluval/bluval.py b/bluval/bluval.py
index 7855816..1580a40 100644 (file)
--- a/bluval/bluval.py
+++ b/bluval/bluval.py
@@ -21,13 +21,31 @@ testcase
 
 import subprocess
 from pathlib import Path
 
 import subprocess
 from pathlib import Path

+import sys
+import traceback

 import click
 import yaml
 
 import click
 import yaml
 

+class BluvalError(Exception):
+    """Base class for exceptions in this module."""
+    pass
+
+
+class ShowStopperError(Exception):
+    """Showstopper test case failed"""
+    pass
+
+

 def run_testcase(testcase):
     """Runs a single testcase
     """
 def run_testcase(testcase):
     """Runs a single testcase
     """

-    show_stopper = testcase.get('show_stopper', False)
+    name = testcase.get('name')
+    skip = testcase.get('skip', "False")
+    if skip.lower() == "true":
+        # skip is mentioned and true.
+        print('Skipping {}'.format(name))
+        return
+    show_stopper = testcase.get('show_stopper', "False")

     what = testcase.get('what')
     mypath = Path(__file__).absolute()
     results_path = mypath.parents[2].joinpath("results/"+testcase.get('layer')+"/"+what)
     what = testcase.get('what')
     mypath = Path(__file__).absolute()
     results_path = mypath.parents[2].joinpath("results/"+testcase.get('layer')+"/"+what)


@@ -42,19 +60,16 @@ def run_testcase(testcase):
     # run the test
     args = ["robot", "-V", str(variables_file), "-d", str(results_path), str(test_path)]
 
     # run the test
     args = ["robot", "-V", str(variables_file), "-d", str(results_path), str(test_path)]
 

-    print('Executing testcase {}'.format(testcase['name']))
-    print('          show_stopper {}'.format(show_stopper))
+    print('Executing testcase {}'.format(name))
+    print('show_stopper {}'.format(show_stopper))

     print('Invoking {}'.format(args))
     try:
         status = subprocess.call(args, shell=False)
     print('Invoking {}'.format(args))
     try:
         status = subprocess.call(args, shell=False)

-        if status != 0 and show_stopper:
-            print('Show stopper testcase failed')
-            return status
+        if status != 0 and show_stopper.lower() == "true":
+            raise ShowStopperError(name)

     except OSError:
     except OSError:

-        print('Error while executing {}'.format(args))
-        return -1
-    return status
-
+        #print('Error while executing {}'.format(args))
+        raise BluvalError(OSError)

 
 def validate_layer(blueprint, layer):
     """validates a layer by validating all testcases under that layer
 
 def validate_layer(blueprint, layer):
     """validates a layer by validating all testcases under that layer


@@ -69,20 +84,56 @@ def validate_blueprint(yaml_loc, layer):
     """Parse yaml file and validates given layer. If no layer given all layers
     validated
     """
     """Parse yaml file and validates given layer. If no layer given all layers
     validated
     """

-    with open(yaml_loc) as yaml_file:
+    with open(str(yaml_loc)) as yaml_file:

         yamldoc = yaml.safe_load(yaml_file)
     blueprint = yamldoc['blueprint']
         yamldoc = yaml.safe_load(yaml_file)
     blueprint = yamldoc['blueprint']

-    if layer is None:
+    if layer is None or layer == "all":

         for each_layer in blueprint['layers']:
             validate_layer(blueprint, each_layer)
     else:
         validate_layer(blueprint, layer)
 
 
         for each_layer in blueprint['layers']:
             validate_layer(blueprint, each_layer)
     else:
         validate_layer(blueprint, layer)
 
 

+def invoke_docker(bluprint, layer):
+    """Start docker container for given layer
+    """
+    cmd = ("docker run"
+           " -v $HOME/.ssh:/root/.ssh"
+           " -v $HOME/.kube/config:/root/.kube/config"
+           " -v $VALIDATION_HOME/tests/variables.yaml:"
+           "/opt/akraino/validation/tests/variables.yaml"
+           " -v $AKRAINO_HOME/results:/opt/akraino/results"
+           " akraino/validation:{0}-latest"
+           " bin/sh -c"
+           " 'cd /opt/akraino/validation "
+           "&& python bluval/bluval.py -l {0} {1}'").format(layer, bluprint)
+    args = [cmd]
+    try:
+        print('Invoking {}'.format(args))
+        subprocess.call(args, shell=True)
+    except OSError:
+        #print('Error while executing {}'.format(args))
+        raise BluvalError(OSError)
+
+
+def invoke_dockers(yaml_loc, layer, blueprint_name):
+    """Parses yaml file and starts docker container for one/all layers
+    """
+    with open(str(yaml_loc)) as yaml_file:
+        yamldoc = yaml.safe_load(yaml_file)
+    blueprint = yamldoc['blueprint']
+    if layer is None or layer == "all":
+        for each_layer in blueprint['layers']:
+            invoke_docker(blueprint_name, each_layer)
+    else:
+        invoke_docker(blueprint_name, layer)
+
+

 @click.command()
 @click.argument('blueprint')
 @click.option('--layer', '-l')
 @click.command()
 @click.argument('blueprint')
 @click.option('--layer', '-l')

-def main(blueprint, layer):
+@click.option('--delegate', '-d', is_flag=True)
+def main(blueprint, layer, delegate):

     """Takes blueprint name and optional layer. Validates inputs and derives
     yaml location from blueprint name. Invokes validate on blue print.
     """
     """Takes blueprint name and optional layer. Validates inputs and derives
     yaml location from blueprint name. Invokes validate on blue print.
     """


@@ -90,8 +141,22 @@ def main(blueprint, layer):
     yaml_loc = mypath.parents[0].joinpath('bluval-{}.yaml'.format(blueprint))
     if layer is not None:
         layer = layer.lower()
     yaml_loc = mypath.parents[0].joinpath('bluval-{}.yaml'.format(blueprint))
     if layer is not None:
         layer = layer.lower()

-    validate_blueprint(yaml_loc, layer)
-
+    try:
+        if delegate is not None:
+            invoke_dockers(yaml_loc, layer, blueprint)
+        else:
+            validate_blueprint(yaml_loc, layer)
+    except ShowStopperError as err:
+        print('ShowStopperError:', err)
+    except BluvalError as err:
+        print('Unexpected BluvalError', err)
+        raise
+    except:
+        print("Exception in user code:")
+        print("-"*60)
+        traceback.print_exc(file=sys.stdout)
+        print("-"*60)
+        raise

 
 if __name__ == "__main__":
     # pylint: disable=no-value-for-parameter
 
 if __name__ == "__main__":
     # pylint: disable=no-value-for-parameter