From f3884a45cd16542418f580a433c47f0a09843323 Mon Sep 17 00:00:00 2001 From: davidplunkett Date: Thu, 21 Mar 2019 22:46:33 +0000 Subject: [PATCH] added key checks and logging to jinja2 processing Change-Id: Icca34435bae5367dbd8fa8c16e63d0139e462394 Signed-off-by: davidplunkett --- scripts/jcopy.py | 95 +++++++++++++--------------------- scripts/jcopy3.py | 79 ---------------------------- templates/baremetal/promjoin.j2 | 4 +- templates/baremetal/rack.j2 | 4 +- templates/pki/pki-catalog.j2 | 4 +- templates/profiles/host/compute-r01.j2 | 2 +- templates/profiles/host/cp-r01.j2 | 2 +- 7 files changed, 45 insertions(+), 145 deletions(-) delete mode 100755 scripts/jcopy3.py diff --git a/scripts/jcopy.py b/scripts/jcopy.py index ac89508..d041c26 100755 --- a/scripts/jcopy.py +++ b/scripts/jcopy.py @@ -15,78 +15,57 @@ # limitations under the License. # ############################################################################## -# # jcopy.py - Copy a file or files to a target directory, making # substitutions as needed from the values contained in a YAML file. +# compatibile with python and python3 # -# usage: jcopy.py -# -# Note: jcopy.sh is for Python2, jcopy3.sh is for Python3 -# +# usage: jcopy.py import os.path import jinja2 import sys import yaml -def expand_files(target_dir, dir_name, files): - global total - xlen = len(sys.argv[2]) - targdir = target_dir + dir_name[xlen:] - if not os.path.exists(targdir): - os.makedirs(targdir) - env = jinja2.Environment() - env.trim_blocks = True - env.lstrip_blocks = True +def usage(msg=None): + if not msg is None: + print(msg) + print( 'usage: jcopy.py ' ) + sys.exit(1) - for f in files: - if f.endswith(".j2"): - t = f.replace(".j2", ".yaml") - source_path = dir_name + '/' + f - target_path = targdir + '/' + t - if os.path.isfile(source_path): - with open(source_path) as fd: - template = env.from_string(fd.read()) - data = template.render(yaml=yaml) - fd2 = open(target_path,'w') - fd2.write(data) - fd2.write("\n") - fd2.close() - print '{0} -> {1}'.format(source_path, target_path) - total += 1 +if len(sys.argv) != 4: + usage('incorrect number of arguments') -def expand_file(target_file, file): - global total - if not os.path.exists(os.path.dirname(target_file)): - os.makedirs(os.path.dirname(target_file)) - env = jinja2.Environment() - env.trim_blocks = True - env.lstrip_blocks = True - with open(file) as fd: - template = env.from_string(fd.read()) - data = template.render(yaml=yaml) - fd2 = open(target_file,'w') - fd2.write(data) - fd2.write("\n") - fd2.close() - print '{0} -> {1}'.format(file, target_file) - total += 1 +yaml_input=sys.argv[1] +j2in_name=sys.argv[2] +yaml_outdir=os.path.dirname(sys.argv[3]+'/') -if len(sys.argv) != 4: - print 'usage: jcopy.py ' - sys.exit(1) +if not os.path.isfile(yaml_input): + usage('input yaml file {} does not exist'.format(yaml_input)) -with open(sys.argv[1]) as f: - yaml = yaml.safe_load(f) +with open(yaml_input) as f: + siteyaml = yaml.safe_load(f) -total = 0 -if os.path.isfile(sys.argv[2]): - expand_file(sys.argv[3], sys.argv[2]) +if os.path.isfile(j2in_name): + j2_env = jinja2.Environment(loader=jinja2.FileSystemLoader(os.path.dirname(j2in_name)), trim_blocks=True, lstrip_blocks=True, undefined=jinja2.make_logging_undefined() ) + templates=[ os.path.basename(j2in_name) ] else: - os.path.walk(sys.argv[2], expand_files, sys.argv[3]) -print '%d files processed.' % total + j2_env = jinja2.Environment(loader=jinja2.FileSystemLoader(j2in_name), trim_blocks=True, lstrip_blocks=True, undefined=jinja2.make_logging_undefined() ) + templates=j2_env.list_templates(extensions=('j2')) + +# find length of longest template name to pad output +fill=len(max(templates,key=len)) + +for f in templates: + t=j2_env.get_template(name=f) + fname=yaml_outdir+'/'+f.replace('.j2','.yaml') + print( '### {0: <{fill}} ==> {1}'.format(f, fname, fill=fill) ) + if not os.path.exists(os.path.dirname(fname)): + os.makedirs(os.path.dirname(fname)) + t.stream(yaml=siteyaml).dump(fname) + +print('%d files processed.' % len(templates)) sys.exit(0) -# sudo python -m ensurepip --default-pip -# sudo python -m pip install --upgrade pip setuptools wheel -# pip install --user jinja2 PyYAML +# sudo apt-get install python-jinja2 python-yaml +# sudo apt-get install python3-jinja2 python3-yaml + diff --git a/scripts/jcopy3.py b/scripts/jcopy3.py deleted file mode 100755 index 4ce9249..0000000 --- a/scripts/jcopy3.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/local/bin/python -############################################################################## -# Copyright © 2018 AT&T Intellectual Property. All rights reserved. # -# # -# Licensed under the Apache License, Version 2.0 (the "License"); you may # -# not use this file except in compliance with the License. # -# # -# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -# Unless required by applicable law or agreed to in writing, software # -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -# See the License for the specific language governing permissions and # -# limitations under the License. # -############################################################################## - -# -# jcopy.py - Copy a file or files to a target directory, making -# substitutions as needed from the values contained in a YAML file. -# -# usage: jcopy.py -# -# Note: jcopy.sh is for Python2, jcopy3.sh is for Python3 -# - -import os.path -import jinja2 -import sys -import yaml - -def expand_files(target_dir, dir_name, files): - global total - xlen = len(sys.argv[2]) - targdir = target_dir + dir_name[xlen:] - if not os.path.exists(targdir): - os.makedirs(targdir) - env = jinja2.Environment() - for f in files: - source_path = dir_name + '/' + f - target_path = targdir + '/' + f - if os.path.isfile(source_path): - with open(source_path) as fd: - template = env.from_string(fd.read()) - data = template.render(yaml=yaml) - fd2 = open(target_path,'w') - fd2.write(data) - fd2.close() - total += 1 - -def expand_file(target_dir, file): - global total - if not os.path.exists(target_dir): - os.makedirs(target_dir) - env = jinja2.Environment() - with open(file) as fd: - template = env.from_string(fd.read()) - data = template.render(yaml=yaml) - target_path = target_dir + '/' + os.path.basename(file) - fd2 = open(target_path,'w') - fd2.write(data) - fd2.close() - total += 1 - -if len(sys.argv) != 4: - print('usage: jcopy.py ') - sys.exit(1) - -with open(sys.argv[1]) as f: - yaml = yaml.safe_load(f) - -total = 0 -if os.path.isfile(sys.argv[2]): - expand_file(sys.argv[3], os.path.abspath(sys.argv[2])) -else: - for root, dirs, files in os.walk(sys.argv[2]): - expand_files(sys.argv[3], root, files) -print('%d files processed.' % total) -sys.exit(0) diff --git a/templates/baremetal/promjoin.j2 b/templates/baremetal/promjoin.j2 index c28363d..d16b48e 100644 --- a/templates/baremetal/promjoin.j2 +++ b/templates/baremetal/promjoin.j2 @@ -35,9 +35,9 @@ data: {% for server in yaml.masters %} - '{{server.name}}' {% endfor %} -{% for server in yaml.workers %} +{% if 'workers' in yaml %}{% for server in yaml.workers %} - '{{server.name}}' -{% endfor %} +{% endfor %}{% endif %} {% raw %} # TODO(alanmeadows) move what is global about this document - everything except nodenames to global assets: - path: /opt/promjoin.sh diff --git a/templates/baremetal/rack.j2 b/templates/baremetal/rack.j2 index b6e6620..464a826 100644 --- a/templates/baremetal/rack.j2 +++ b/templates/baremetal/rack.j2 @@ -49,7 +49,7 @@ data: tags: - 'masters' {% endfor %} -{% for server in yaml.workers %} +{% if 'workers' in yaml %}{% for server in yaml.workers %} --- schema: 'drydock/BaremetalNode/v1' metadata: @@ -84,5 +84,5 @@ data: rack: RACK01 tags: - 'workers' -{% endfor %} +{% endfor %}{% endif %} ... diff --git a/templates/pki/pki-catalog.j2 b/templates/pki/pki-catalog.j2 index 17e18f1..ae5ab0b 100644 --- a/templates/pki/pki-catalog.j2 +++ b/templates/pki/pki-catalog.j2 @@ -66,7 +66,7 @@ data: groups: - system:nodes {% endfor %} -{% for server in yaml.workers %} +{% if 'workers' in yaml %}{% for server in yaml.workers %} - document_name: kubelet-{{ server.name }} common_name: system:node:{{ server.name }} hosts: @@ -76,7 +76,7 @@ data: - {{server.pxe}} groups: - system:nodes -{% endfor %} +{% endfor %}{% endif %} - document_name: scheduler description: Service certificate for Kubernetes scheduler common_name: system:kube-scheduler diff --git a/templates/profiles/host/compute-r01.j2 b/templates/profiles/host/compute-r01.j2 index be609de..4e942be 100644 --- a/templates/profiles/host/compute-r01.j2 +++ b/templates/profiles/host/compute-r01.j2 @@ -74,7 +74,7 @@ data: physical_devices: {% for disk in yaml.disks_compute %} {{disk.name}}: - {% if disk.labels %} + {% if 'labels' in disk %} labels: {% for key, value in disk.labels.items() %} {{key}}: '{{value}}' diff --git a/templates/profiles/host/cp-r01.j2 b/templates/profiles/host/cp-r01.j2 index e59df91..e66e85c 100644 --- a/templates/profiles/host/cp-r01.j2 +++ b/templates/profiles/host/cp-r01.j2 @@ -73,7 +73,7 @@ data: physical_devices: {% for disk in yaml.disks %} {{disk.name}}: - {% if disk.labels %} + {% if 'labels' in disk %} labels: {% for key, value in disk.labels.items() %} {{key}}: '{{value}}' -- 2.16.6