From 6004efeec09813ac229b539409cfda8fbf412244 Mon Sep 17 00:00:00 2001 From: Juha Kosonen Date: Thu, 20 Feb 2020 16:21:12 +0200 Subject: [PATCH 01/16] Add Robot test for running kube-hunter Integrated to k8s layer as an optional test. JIRA: VAL-96 Change-Id: I0d11a3aa9438a7fe3dbe52a1358af5ddcaabff61 Signed-off-by: Juha Kosonen --- .coafile | 3 +- bluval/bluval-base.yaml | 3 + bluval/bluval-iec.yaml | 4 + bluval/bluval-kni.yaml | 4 + bluval/bluval-rec.yaml | 4 + bluval/bluval-unicycle.yaml | 4 + docker/k8s/pip-requirements.txt | 1 + docker/kube-hunter/Dockerfile | 40 ++++++++++ docker/kube-hunter/Makefile | 26 ++++++ docker/kube-hunter/pip-requirements.txt | 1 + tests/k8s/kube-hunter/job.yaml | 31 ++++++++ tests/k8s/kube-hunter/kube-hunter.resource | 123 +++++++++++++++++++++++++++++ tests/k8s/kube-hunter/kube-hunter.robot | 39 +++++++++ 13 files changed, 282 insertions(+), 1 deletion(-) create mode 100644 docker/kube-hunter/Dockerfile create mode 100644 docker/kube-hunter/Makefile create mode 100644 docker/kube-hunter/pip-requirements.txt create mode 100644 tests/k8s/kube-hunter/job.yaml create mode 100644 tests/k8s/kube-hunter/kube-hunter.resource create mode 100644 tests/k8s/kube-hunter/kube-hunter.robot diff --git a/.coafile b/.coafile index fd4a0cc..c472b5d 100644 --- a/.coafile +++ b/.coafile @@ -36,7 +36,8 @@ bears = YAMLLintBear files = **.yaml, **.yml use_spaces = true max_line_length = 120 -ignore = tests/k8s/conformance/custom_repos.yaml +ignore = tests/k8s/conformance/custom_repos.yaml, + tests/k8s/kube-hunter/job.yaml [all.Python] bears = PyLintBear diff --git a/bluval/bluval-base.yaml b/bluval/bluval-base.yaml index ab860f9..845a700 100644 --- a/bluval/bluval-base.yaml +++ b/bluval/bluval-base.yaml @@ -54,6 +54,9 @@ blueprint: - name: ha what: ha + - + name: kube-hunter + what: kube-hunter k8s_networking: &k8s_networking - name: node_connectivity diff --git a/bluval/bluval-iec.yaml b/bluval/bluval-iec.yaml index 1762e5e..1da0158 100644 --- a/bluval/bluval-iec.yaml +++ b/bluval/bluval-iec.yaml @@ -25,3 +25,7 @@ blueprint: name: conformance what: conformance optional: "False" + - + name: kube-hunter + what: kube-hunter + optional: "True" diff --git a/bluval/bluval-kni.yaml b/bluval/bluval-kni.yaml index 75f941b..c615ffb 100644 --- a/bluval/bluval-kni.yaml +++ b/bluval/bluval-kni.yaml @@ -37,3 +37,7 @@ blueprint: - name: conformance what: conformance + - + name: kube-hunter + what: kube-hunter + optional: "True" diff --git a/bluval/bluval-rec.yaml b/bluval/bluval-rec.yaml index 8bc3203..07df9da 100644 --- a/bluval/bluval-rec.yaml +++ b/bluval/bluval-rec.yaml @@ -52,3 +52,7 @@ blueprint: name: etcd_ha what: etcd_ha optional: "True" + - + name: kube-hunter + what: kube-hunter + optional: "True" diff --git a/bluval/bluval-unicycle.yaml b/bluval/bluval-unicycle.yaml index 02e6b31..2f0768d 100644 --- a/bluval/bluval-unicycle.yaml +++ b/bluval/bluval-unicycle.yaml @@ -46,6 +46,10 @@ blueprint: name: etcd_ha what: etcd_ha optional: "True" + - + name: kube-hunter + what: kube-hunter + optional: "True" openstack: &openstack_unicycle - name: tempest diff --git a/docker/k8s/pip-requirements.txt b/docker/k8s/pip-requirements.txt index 18b264c..59a75a8 100644 --- a/docker/k8s/pip-requirements.txt +++ b/docker/k8s/pip-requirements.txt @@ -3,3 +3,4 @@ robotframework-httplibrary robotframework-jsonlibrary robotframework-requests robotframework-sshlibrary +kube-hunter diff --git a/docker/kube-hunter/Dockerfile b/docker/kube-hunter/Dockerfile new file mode 100644 index 0000000..7e65f93 --- /dev/null +++ b/docker/kube-hunter/Dockerfile @@ -0,0 +1,40 @@ +############################################################################## +# Copyright (c) 2020 AT&T, ENEA AB, Nokia and others # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you maynot use this file except in compliance with the License. # +# # +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################## + +FROM python:3.6-alpine3.9 as build + +# Install dependencies +COPY pip-requirements.txt /wheels/requirements/pip-requirements.txt + +RUN apk --no-cache add --update \ + linux-headers \ + build-base + +# Build binaries +WORKDIR /wheels +RUN pip3 install wheel +RUN pip3 wheel -r /wheels/requirements/pip-requirements.txt + +# Copy binaries in the final container +FROM python:3.6-alpine3.9 +COPY --from=build /wheels /wheels + +RUN pip3 install -r /wheels/requirements/pip-requirements.txt \ + -f /wheels && \ + rm -rf /wheels && \ + rm -rf /root/.cache/pip/* + +ENTRYPOINT ["kube-hunter"] diff --git a/docker/kube-hunter/Makefile b/docker/kube-hunter/Makefile new file mode 100644 index 0000000..4a059ba --- /dev/null +++ b/docker/kube-hunter/Makefile @@ -0,0 +1,26 @@ +############################################################################## +# Copyright (c) 2020 AT&T, ENEA AB, Nokia and others # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you maynot use this file except in compliance with the License. # +# # +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################## + +.PHONY: all +all: push-image .push_manifest + +.PHONY: build +build: .build + +.PHONY: push-image +push-image: .push_image + +include ../build.mk diff --git a/docker/kube-hunter/pip-requirements.txt b/docker/kube-hunter/pip-requirements.txt new file mode 100644 index 0000000..6052d92 --- /dev/null +++ b/docker/kube-hunter/pip-requirements.txt @@ -0,0 +1 @@ +kube-hunter diff --git a/tests/k8s/kube-hunter/job.yaml b/tests/k8s/kube-hunter/job.yaml new file mode 100644 index 0000000..62079c5 --- /dev/null +++ b/tests/k8s/kube-hunter/job.yaml @@ -0,0 +1,31 @@ +############################################################################## +# Copyright (c) 2020 AT&T Intellectual Property. # +# Copyright (c) 2020 Nokia. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. # +# # +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################## +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: kube-hunter +spec: + template: + spec: + containers: + - name: kube-hunter + image: {{ image }} + command: ["kube-hunter"] + args: ["--pod"] + restartPolicy: Never + backoffLimit: 4 diff --git a/tests/k8s/kube-hunter/kube-hunter.resource b/tests/k8s/kube-hunter/kube-hunter.resource new file mode 100644 index 0000000..561dea2 --- /dev/null +++ b/tests/k8s/kube-hunter/kube-hunter.resource @@ -0,0 +1,123 @@ +############################################################################## +# Copyright (c) 2020 AT&T Intellectual Property. # +# Copyright (c) 2020 Nokia. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you maynot use this file except in compliance with the License. # +# # +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################## + + +*** Settings *** +Library Collections +Library OperatingSystem +Library Process +Library SSHLibrary +Library String + + +*** Variables *** +${REPORTDIR} ${LOG_PATH}/${SUITE_NAME.replace(' ','_')} +&{KUBE_HUNTER} path=akraino +... name=validation:kube-hunter-latest + + +*** Keywords *** +Open Connection And Log In + Open Connection ${HOST} + Login With Public Key ${USERNAME} ${SSH_KEYFILE} + +Get Cluster Address + ${result}= Run Process kubectl config view --minify + ... -o jsonpath\={.clusters[0].cluster.server} + Should Be Equal As Integers ${result.rc} 0 + ${addr}= Fetch From Right ${result.stdout} :// + ${addr}= Fetch From Left ${addr} : + Should Not Be Empty ${addr} + [Return] ${addr} + +Get Remote Addresses + ${result}= Run Process kubectl get nodes + ... -o jsonpath\={.items[*].status.addresses[?(@.type\=\="ExternalIP")].address} + Should Be Equal As Integers ${result.rc} 0 + Pass Execution If '${result.stdout}' == '${EMPTY}' No external node IPs exposed + @{addrs}= Split String ${result.stdout} + [Return] ${addrs} + +Upload To Internal Registry + [Arguments] ${path} ${name} + ${rc}= Execute Command + ... docker pull ${path}/${name} + ... return_stdout=False return_rc=True + Should Be Equal As Integers ${rc} 0 + ${rc}= Execute Command + ... docker tag ${path}/${name} ${INT_REG}/bluval/${name} + ... return_stdout=False return_rc=True + Should Be Equal As Integers ${rc} 0 + ${rc}= Execute Command + ... docker push ${INT_REG}/bluval/${name} + ... return_stdout=False return_rc=True + Should Be Equal As Integers ${rc} 0 + +Onboard Image + ${INT_REG}= Get Variable Value ${INTERNAL_REGISTRY} ${EMPTY} + Set Test Variable ${INT_REG} + Return From Keyword If $INT_REG == '${EMPTY}' + Open Connection And Log In + Upload To Internal Registry ${KUBE_HUNTER['path']} ${KUBE_HUNTER['name']} + Set To Dictionary ${KUBE_HUNTER} path=${INT_REG}/bluval + +Prepare Job Manifest + Run Process sed -i s|{{ image }}|${KUBE_HUNTER['path']}/${KUBE_HUNTER['name']}|g + ... ${CURDIR}/job.yaml + +Set Scan Status + [Arguments] ${log} + ${STATUS}= Evaluate "No vulnerabilities were found" in """${log}""" + Set Test Variable ${STATUS} + +Delete Scan Job + ${result}= Run Process kubectl delete job kube-hunter + Should Be Equal As Integers ${result.rc} 0 + +Should Discover No Vulnerabilities + Should Be True ${STATUS} + +Run Scan Within Pod + ${result}= Run Process kubectl apply -f job.yaml + Should Be Equal As Integers ${result.rc} 0 + ${result}= Run Process kubectl wait --for\=condition\=complete + ... --timeout\=15m job/kube-hunter + Should Be Equal As Integers ${result.rc} 0 + ${result}= Run Process kubectl get pods --selector\=job-name\=kube-hunter + ... -o jsonpath\={.items[*].metadata.name} + Should Be Equal As Integers ${result.rc} 0 + ${result}= Run Process kubectl logs ${result.stdout} + ... stdout=pod.log + Copy File pod.log ${REPORTDIR}/ + Should Be Equal As Integers ${result.rc} 0 + Set Scan Status ${result.stdout} + +Run Node Scan + ${addrs}= Get Remote Addresses + ${result}= Run Process kube-hunter --remote @{addrs} + ... stdout=node.log + Copy File node.log ${REPORTDIR}/ + Should Be Equal As Integers ${result.rc} 0 + Set Scan Status ${result.stdout} + +Run Cluster Scan + ${addr}= Get Cluster Address + ${result}= Run Process kube-hunter --remote ${addr} + ... stdout=cluster.log + Copy File cluster.log ${REPORTDIR}/ + Should Be Equal As Integers ${result.rc} 0 + Set Scan Status ${result.stdout} diff --git a/tests/k8s/kube-hunter/kube-hunter.robot b/tests/k8s/kube-hunter/kube-hunter.robot new file mode 100644 index 0000000..4ef9bf6 --- /dev/null +++ b/tests/k8s/kube-hunter/kube-hunter.robot @@ -0,0 +1,39 @@ +############################################################################## +# Copyright (c) 2020 AT&T Intellectual Property. # +# Copyright (c) 2020 Nokia. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you maynot use this file except in compliance with the License. # +# # +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################## + + +*** Settings *** +Documentation Hunt for security weaknesses in Kubernetes cluster +Resource kube-hunter.resource + + +*** Test Cases *** +Cluster Remote Scanning + Run Cluster Scan + Should Discover No Vulnerabilities + +Node Remote Scanning + Run Node Scan + Should Discover No Vulnerabilities + +Inside-a-Pod Scanning + [Setup] Run Keywords Onboard Image + ... Prepare Job Manifest + Run Scan Within Pod + Should Discover No Vulnerabilities + [Teardown] Run Keywords Delete Scan Job + ... Close All Connections -- 2.16.6 From 37a1ec033aa1711ac75b2f89b038684d5de80dba Mon Sep 17 00:00:00 2001 From: Daniel Stoica Date: Tue, 7 Jan 2020 13:33:53 +0200 Subject: [PATCH 02/16] Add Vuls security test for x86_64 JIRA: VAL-79 Signed-off-by: Daniel Stoica Change-Id: I6aa553d50fc33327ee893382aec0eee605a94eb1 --- docker/os/Dockerfile | 42 ++++++++++++++++++++++++++++++++++++-- tests/os/vuls/config | 2 ++ tests/os/vuls/config.toml | 8 ++++++++ tests/os/vuls/vuls.robot | 52 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 tests/os/vuls/config create mode 100644 tests/os/vuls/config.toml create mode 100644 tests/os/vuls/vuls.robot diff --git a/docker/os/Dockerfile b/docker/os/Dockerfile index 58bed23..95d8cfe 100644 --- a/docker/os/Dockerfile +++ b/docker/os/Dockerfile @@ -28,6 +28,9 @@ RUN apt-get update && apt-get -y install \ libffi-dev \ make \ libssl-dev \ + golint \ + sqlite3 \ + debian-goodies \ build-essential \ autoconf automake autotools-dev m4 \ linux-headers-generic \ @@ -60,8 +63,16 @@ COPY --from=build /opt/akraino/ltp.tar.gz /opt/akraino/ltp.tar.gz COPY --from=build /opt/akraino/lynis-remote.tar.gz /opt/akraino/lynis-remote.tar.gz -RUN apt-get update && apt-get -y install \ - python3-pip python3.6 && \ +RUN apt-get update && apt-get -y upgrade && apt-get -y install \ + python3-pip python3.6 \ + sqlite \ + debian-goodies \ + wget \ + curl apt-transport-https ca-certificates software-properties-common && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ + apt-get update && \ + apt-get install -y docker-ce && \ cd /usr/bin && ln -s python3 python && \ pip3 install -r /wheels/requirements/pip-requirements.txt \ -f /wheels && \ @@ -70,6 +81,33 @@ RUN apt-get update && apt-get -y install \ rm -rf /var/cache/apt/* && \ rm -rf /var/lib/apt/lists/* +# Fetches vuls databases +SHELL ["/bin/bash", "-c"] +RUN wget https://dl.google.com/go/go1.12.6.linux-amd64.tar.gz -P /root/ && \ + cd /root/ && \ + tar -xzf go1.12.6.linux-amd64.tar.gz -C /root/ && \ + export GOROOT=/root/go && \ + export GOPATH=/root/go/src && \ + export PATH=$PATH:/root/go/bin:/root/go/src/bin && \ + mkdir -p /root/go/src/github.com/future-architect && \ + cd /root/go/src/github.com/future-architect && \ + git clone https://github.com/future-architect/vuls && \ + cd vuls && \ + make install && \ + mkdir -p /root/go/src/github.com/kotakanbe && \ + git -C /root/go/src/github.com/kotakanbe clone https://github.com/kotakanbe/go-cve-dictionary.git && \ + cd /root/go/src/github.com/kotakanbe/go-cve-dictionary/ && \ + make install && \ + mkdir /opt/akraino/validation/tests/os/vuls/ && \ + for i in $(seq 2002 "$(date +"%Y")"); do go-cve-dictionary fetchnvd -http-proxy=${HTTP_PROXY} -dbpath /opt/akraino/validation/tests/os/vuls/cve.sqlite3 -years "$i"; done && \ + git -C /root/go/src/github.com/kotakanbe clone https://github.com/kotakanbe/goval-dictionary.git && \ + cd /root/go/src/github.com/kotakanbe/goval-dictionary && \ + make install && \ + goval-dictionary fetch-ubuntu -http-proxy=${HTTP_PROXY} -dbpath=/opt/akraino/validation/tests/os/vuls/oval.sqlite3 16 && \ + cd /opt/akraino/validation/tests/os/vuls && \ + tar cvzf db.tar.gz *.sqlite3 && \ + rm *.sqlite3 + # Install bluval dependencies RUN pip3 install -r /opt/akraino/validation/bluval/requirements.txt ENV LC_ALL=C.UTF-8 diff --git a/tests/os/vuls/config b/tests/os/vuls/config new file mode 100644 index 0000000..8025e68 --- /dev/null +++ b/tests/os/vuls/config @@ -0,0 +1,2 @@ +Host * + StrictHostKeyChecking no diff --git a/tests/os/vuls/config.toml b/tests/os/vuls/config.toml new file mode 100644 index 0000000..1329cd5 --- /dev/null +++ b/tests/os/vuls/config.toml @@ -0,0 +1,8 @@ +[servers] + +[servers.akraino] +host = "HOST" +port = "22" +user = "USERNAME" +keyPath = "/root/.ssh/id_rsa" +containerType = "docker" diff --git a/tests/os/vuls/vuls.robot b/tests/os/vuls/vuls.robot new file mode 100644 index 0000000..6ddc283 --- /dev/null +++ b/tests/os/vuls/vuls.robot @@ -0,0 +1,52 @@ +############################################################################## +# Copyright (c) 2019 AT&T Intellectual Property. # +# Copyright (c) 2019 Nokia. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you maynot use this file except in compliance with the License. # +# # +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################## + +*** Settings *** +Library SSHLibrary +Library OperatingSystem +Library BuiltIn +Library Process + +*** Variables *** +${LOG_PATH} /opt/akraino/validation/tests/os/vuls + +*** Test Cases *** +Run Vuls test + Set Environment Variable GOROOT /root/go + Set Environment Variable GOPATH /root/go/src + Set Environment Variable PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/root/go/bin:/root/go/src/bin + Set Environment Variable LC_ALL en_US.UTF-8 + Set Environment Variable LANG en_US.UTF-8 + + ${rc} = Run And Return Rc install -D /opt/akraino/validation/tests/os/vuls/config /root/.ssh/ + Should Be Equal As Integers ${rc} 0 + + ${rc} = Run And Return Rc sed -i 's/HOST/${HOST}/g' config.toml + Should Be Equal As Integers ${rc} 0 + + ${rc} = Run And Return Rc sed -i 's/USERNAME/${USERNAME}/g' config.toml + Should Be Equal As Integers ${rc} 0 + + ${rc} = Run And Return Rc tar xvzf db.tar.gz -C /opt/akraino/validation/tests/os/vuls/ + Should Be Equal As Integers ${rc} 0 + + ${rc} = Run And Return Rc vuls scan -config config.toml -ssh-config + Should Be Equal As Integers ${rc} 0 + + ${rc} ${output} = Run And Return Rc And Output vuls report + Should Be Equal As Integers ${rc} 0 + Append To File ${LOG_PATH}/vuls.log ${output}${\n} -- 2.16.6 From fa62d27c82de85dc9c1d703fc8e1026169881fe8 Mon Sep 17 00:00:00 2001 From: Juha Kosonen Date: Mon, 2 Mar 2020 14:21:31 +0200 Subject: [PATCH 03/16] Fix os layer build failures Following changes implemented: - pkg-config is required for building recent version of LTP [1] - do not try to re-create vuls directory [1] https://github.com/linux-test-project/ltp/issues/649 JIRA: VAL-100 Signed-off-by: Juha Kosonen Change-Id: Ib6050dead7e296306ed3f860797fdf6b3944fb43 --- docker/os/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/os/Dockerfile b/docker/os/Dockerfile index 95d8cfe..fc0d033 100644 --- a/docker/os/Dockerfile +++ b/docker/os/Dockerfile @@ -32,6 +32,7 @@ RUN apt-get update && apt-get -y install \ sqlite3 \ debian-goodies \ build-essential \ + pkg-config \ autoconf automake autotools-dev m4 \ linux-headers-generic \ libaio-dev libattr1-dev libcap-dev @@ -98,7 +99,6 @@ RUN wget https://dl.google.com/go/go1.12.6.linux-amd64.tar.gz -P /root/ && \ git -C /root/go/src/github.com/kotakanbe clone https://github.com/kotakanbe/go-cve-dictionary.git && \ cd /root/go/src/github.com/kotakanbe/go-cve-dictionary/ && \ make install && \ - mkdir /opt/akraino/validation/tests/os/vuls/ && \ for i in $(seq 2002 "$(date +"%Y")"); do go-cve-dictionary fetchnvd -http-proxy=${HTTP_PROXY} -dbpath /opt/akraino/validation/tests/os/vuls/cve.sqlite3 -years "$i"; done && \ git -C /root/go/src/github.com/kotakanbe clone https://github.com/kotakanbe/goval-dictionary.git && \ cd /root/go/src/github.com/kotakanbe/goval-dictionary && \ -- 2.16.6 From 2b01f397c6de07f7aec751a5451d349296851db0 Mon Sep 17 00:00:00 2001 From: Daniel Stoica Date: Mon, 24 Feb 2020 18:15:15 +0200 Subject: [PATCH 04/16] Add Vuls security test for ubuntu and centos JIRA: VAL-79 Signed-off-by: Daniel Stoica Change-Id: If81d5ecb68bea40ae179823d655311260e04fc56 --- docker/os/Dockerfile | 65 +++++++++++++++++++++++++----------------------- tests/os/vuls/vuls.robot | 24 +++++++++++++++--- 2 files changed, 55 insertions(+), 34 deletions(-) diff --git a/docker/os/Dockerfile b/docker/os/Dockerfile index 95d8cfe..676062d 100644 --- a/docker/os/Dockerfile +++ b/docker/os/Dockerfile @@ -28,10 +28,12 @@ RUN apt-get update && apt-get -y install \ libffi-dev \ make \ libssl-dev \ + wget \ golint \ sqlite3 \ debian-goodies \ build-essential \ + pkg-config \ autoconf automake autotools-dev m4 \ linux-headers-generic \ libaio-dev libattr1-dev libcap-dev @@ -55,37 +57,13 @@ RUN tar czvf /opt/akraino/ltp.tar.gz /opt/ltp WORKDIR /root/src RUN git clone https://github.com/CISOfy/lynis && tar czvf /opt/akraino/lynis-remote.tar.gz ./lynis -# Copy binaries into the final container and install robot framework -FROM ubuntu:18.04 -COPY --from=build /wheels /wheels -COPY --from=build /opt/akraino/validation /opt/akraino/validation -COPY --from=build /opt/akraino/ltp.tar.gz /opt/akraino/ltp.tar.gz -COPY --from=build /opt/akraino/lynis-remote.tar.gz /opt/akraino/lynis-remote.tar.gz - - -RUN apt-get update && apt-get -y upgrade && apt-get -y install \ - python3-pip python3.6 \ - sqlite \ - debian-goodies \ - wget \ - curl apt-transport-https ca-certificates software-properties-common && \ - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ - apt-get update && \ - apt-get install -y docker-ce && \ - cd /usr/bin && ln -s python3 python && \ - pip3 install -r /wheels/requirements/pip-requirements.txt \ - -f /wheels && \ - rm -rf /wheels && \ - rm -rf /root/.cache/pip/* && \ - rm -rf /var/cache/apt/* && \ - rm -rf /var/lib/apt/lists/* - -# Fetches vuls databases +#Fetches vuls databases SHELL ["/bin/bash", "-c"] -RUN wget https://dl.google.com/go/go1.12.6.linux-amd64.tar.gz -P /root/ && \ +RUN if [ $(uname -m) == 'aarch64' ]; then HOST_ARCH=arm64; else HOST_ARCH=amd64; fi && \ + wget https://dl.google.com/go/go1.12.6.linux-$HOST_ARCH.tar.gz -P /root/ && \ cd /root/ && \ - tar -xzf go1.12.6.linux-amd64.tar.gz -C /root/ && \ + tar -xzf go1.12.6.linux-$HOST_ARCH.tar.gz -C /root/ && \ + rm go1.12.6.linux-$HOST_ARCH.tar.gz && \ export GOROOT=/root/go && \ export GOPATH=/root/go/src && \ export PATH=$PATH:/root/go/bin:/root/go/src/bin && \ @@ -98,16 +76,41 @@ RUN wget https://dl.google.com/go/go1.12.6.linux-amd64.tar.gz -P /root/ && \ git -C /root/go/src/github.com/kotakanbe clone https://github.com/kotakanbe/go-cve-dictionary.git && \ cd /root/go/src/github.com/kotakanbe/go-cve-dictionary/ && \ make install && \ - mkdir /opt/akraino/validation/tests/os/vuls/ && \ for i in $(seq 2002 "$(date +"%Y")"); do go-cve-dictionary fetchnvd -http-proxy=${HTTP_PROXY} -dbpath /opt/akraino/validation/tests/os/vuls/cve.sqlite3 -years "$i"; done && \ git -C /root/go/src/github.com/kotakanbe clone https://github.com/kotakanbe/goval-dictionary.git && \ cd /root/go/src/github.com/kotakanbe/goval-dictionary && \ make install && \ - goval-dictionary fetch-ubuntu -http-proxy=${HTTP_PROXY} -dbpath=/opt/akraino/validation/tests/os/vuls/oval.sqlite3 16 && \ + goval-dictionary fetch-ubuntu -http-proxy=${HTTP_PROXY} -dbpath=/opt/akraino/validation/tests/os/vuls/oval_ubuntu_16.sqlite3 16 && \ + goval-dictionary fetch-ubuntu -http-proxy=${HTTP_PROXY} -dbpath=/opt/akraino/validation/tests/os/vuls/oval_ubuntu_18.sqlite3 18 && \ + goval-dictionary fetch-redhat -http-proxy=${HTTP_PROXY} -dbpath=/opt/akraino/validation/tests/os/vuls/oval_centos.sqlite3 7 && \ + mkdir -p /root/go/src/github.com/knqyf263 && \ + git -C /root/go/src/github.com/knqyf263 clone https://github.com/knqyf263/gost.git && \ + cd /root/go/src/github.com/knqyf263/gost && \ + make install && \ + gost fetch redhat --http-proxy=${HTTP_PROXY} --dbpath=/opt/akraino/validation/tests/os/vuls/gost_centos.sqlite3 && \ cd /opt/akraino/validation/tests/os/vuls && \ tar cvzf db.tar.gz *.sqlite3 && \ rm *.sqlite3 +# Copy binaries into the final container and install robot framework +FROM ubuntu:18.04 +COPY --from=build /wheels /wheels +COPY --from=build /opt/akraino/validation /opt/akraino/validation +COPY --from=build /opt/akraino/ltp.tar.gz /opt/akraino/ltp.tar.gz +COPY --from=build /opt/akraino/lynis-remote.tar.gz /opt/akraino/lynis-remote.tar.gz +COPY --from=build /root/go/bin /root/go/bin +COPY --from=build /root/go/src/bin /root/go/src/bin + +RUN apt-get update && apt-get -y install \ + python3-pip python3.6 && \ + cd /usr/bin && ln -s python3 python && \ + pip3 install -r /wheels/requirements/pip-requirements.txt \ + -f /wheels && \ + rm -rf /wheels && \ + rm -rf /root/.cache/pip/* && \ + rm -rf /var/cache/apt/* && \ + rm -rf /var/lib/apt/lists/* + # Install bluval dependencies RUN pip3 install -r /opt/akraino/validation/bluval/requirements.txt ENV LC_ALL=C.UTF-8 diff --git a/tests/os/vuls/vuls.robot b/tests/os/vuls/vuls.robot index 6ddc283..c630c5a 100644 --- a/tests/os/vuls/vuls.robot +++ b/tests/os/vuls/vuls.robot @@ -19,7 +19,8 @@ Library SSHLibrary Library OperatingSystem Library BuiltIn -Library Process +Suite Setup Open Connection And Log In +Suite Teardown Close All Connections *** Variables *** ${LOG_PATH} /opt/akraino/validation/tests/os/vuls @@ -43,10 +44,27 @@ Run Vuls test ${rc} = Run And Return Rc tar xvzf db.tar.gz -C /opt/akraino/validation/tests/os/vuls/ Should Be Equal As Integers ${rc} 0 - + + ${os} = SSHLibrary.Execute Command source /etc/os-release && echo $ID + ${rc} = Run And Return Rc vuls scan -config config.toml -ssh-config Should Be Equal As Integers ${rc} 0 - ${rc} ${output} = Run And Return Rc And Output vuls report + Run Keyword IF '${os}' == 'ubuntu' Run vuls for ubuntu ELSE IF '${os}' == 'centos' Run vuls for centos ELSE FAIL Distro '${os}' not supported + +*** Keywords *** +Run vuls for ubuntu + ${os_version} = SSHLibrary.Execute Command source /etc/os-release && echo $VERSION_ID | cut -d '.' -f1 + + ${rc} ${output} = Run And Return Rc And Output vuls report -cvedb-sqlite3-path=${LOG_PATH}/cve.sqlite3 -ovaldb-sqlite3-path=${LOG_PATH}/oval_ubuntu_${os_version}.sqlite3 + Should Be Equal As Integers ${rc} 0 + Append To File ${LOG_PATH}/vuls.log ${output}${\n} + +Run vuls for centos + ${rc} ${output} = Run And Return Rc And Output vuls report -cvedb-sqlite3-path=${LOG_PATH}/cve.sqlite3 -ovaldb-sqlite3-path=${LOG_PATH}/oval_centos.sqlite3 -gostdb-sqlite3-path=${LOG_PATH}/gost_centos.sqlite3 Should Be Equal As Integers ${rc} 0 Append To File ${LOG_PATH}/vuls.log ${output}${\n} + +Open Connection And Log In + Open Connection ${HOST} + Login With Public Key ${USERNAME} ${SSH_KEYFILE} -- 2.16.6 From 2fe30c5ad2334067e75347e71f4516416fc94ad3 Mon Sep 17 00:00:00 2001 From: Juha Kosonen Date: Thu, 19 Mar 2020 12:47:44 +0200 Subject: [PATCH 05/16] Update Tempest test case list - RefStack 2019.06 -> 2019.11 Signed-off-by: Juha Kosonen Change-Id: Icec93538cae9dfdb460b069e6918133ce1a593ae --- docker/openstack/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/openstack/Dockerfile b/docker/openstack/Dockerfile index 95bb2e6..6717d4c 100644 --- a/docker/openstack/Dockerfile +++ b/docker/openstack/Dockerfile @@ -17,7 +17,7 @@ # ref: https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#use-multi-stage-builds FROM python:3.6-alpine3.9 as build -ARG REFSTACK_TARGET=2019.06 +ARG REFSTACK_TARGET=2019.11 # Install dependencies COPY pip-requirements.txt /wheels/requirements/pip-requirements.txt -- 2.16.6 From 91bbac212a6deb07ef6d1a5a8b8d55ddf4bab69b Mon Sep 17 00:00:00 2001 From: Juha Kosonen Date: Fri, 27 Mar 2020 10:55:58 +0200 Subject: [PATCH 06/16] Add path to kube-hunter job manifest file Change-Id: I3977be79d96c6122f571efff72d40495e420e767 Signed-off-by: Juha Kosonen --- tests/k8s/kube-hunter/kube-hunter.resource | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/k8s/kube-hunter/kube-hunter.resource b/tests/k8s/kube-hunter/kube-hunter.resource index 561dea2..4611e75 100644 --- a/tests/k8s/kube-hunter/kube-hunter.resource +++ b/tests/k8s/kube-hunter/kube-hunter.resource @@ -92,7 +92,7 @@ Should Discover No Vulnerabilities Should Be True ${STATUS} Run Scan Within Pod - ${result}= Run Process kubectl apply -f job.yaml + ${result}= Run Process kubectl apply -f ${CURDIR}/job.yaml Should Be Equal As Integers ${result.rc} 0 ${result}= Run Process kubectl wait --for\=condition\=complete ... --timeout\=15m job/kube-hunter -- 2.16.6 From 10f8fdddb0d58e4cf15177bc4bb29775244c4cfe Mon Sep 17 00:00:00 2001 From: DANIEL STOICA Date: Tue, 24 Mar 2020 14:46:22 +0200 Subject: [PATCH 07/16] Enable vuls and lynis tests in CI - added the os layer in validation/bluval/bluval-iec.yaml - made change in vuls robot script to sincronize the path complements https://gerrit.akraino.org/r/c/ci-management/+/3307 Signed-off-by: DANIEL STOICA Depends-On: I9e5349a0e3e3a1e7331a5f5b4b4a8d5a1b01aa51 Change-Id: Ibb16f76405f5b43797e0d859db0681759adf5e18 --- bluval/bluval-iec.yaml | 11 +++++++++++ tests/os/vuls/vuls.robot | 22 ++++++++-------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/bluval/bluval-iec.yaml b/bluval/bluval-iec.yaml index 1da0158..86fbef2 100644 --- a/bluval/bluval-iec.yaml +++ b/bluval/bluval-iec.yaml @@ -19,6 +19,7 @@ blueprint: name: demo layers: - k8s + - os k8s: &k8s - @@ -29,3 +30,13 @@ blueprint: name: kube-hunter what: kube-hunter optional: "True" + + os: &os + - + name: lynis + what: lynis + optional: "False" + - + name: vuls + what: vuls + optional: "False" diff --git a/tests/os/vuls/vuls.robot b/tests/os/vuls/vuls.robot index c630c5a..3b3a901 100644 --- a/tests/os/vuls/vuls.robot +++ b/tests/os/vuls/vuls.robot @@ -22,9 +22,6 @@ Library BuiltIn Suite Setup Open Connection And Log In Suite Teardown Close All Connections -*** Variables *** -${LOG_PATH} /opt/akraino/validation/tests/os/vuls - *** Test Cases *** Run Vuls test Set Environment Variable GOROOT /root/go @@ -33,38 +30,35 @@ Run Vuls test Set Environment Variable LC_ALL en_US.UTF-8 Set Environment Variable LANG en_US.UTF-8 - ${rc} = Run And Return Rc install -D /opt/akraino/validation/tests/os/vuls/config /root/.ssh/ - Should Be Equal As Integers ${rc} 0 - - ${rc} = Run And Return Rc sed -i 's/HOST/${HOST}/g' config.toml + ${rc} = Run And Return Rc install -D ${CURDIR}/config /root/.ssh/ Should Be Equal As Integers ${rc} 0 - ${rc} = Run And Return Rc sed -i 's/USERNAME/${USERNAME}/g' config.toml + ${rc} = Run And Return Rc sed -i -e 's/HOST/${HOST}/g' -e 's/USERNAME/${USERNAME}/g' ${CURDIR}/config.toml Should Be Equal As Integers ${rc} 0 - ${rc} = Run And Return Rc tar xvzf db.tar.gz -C /opt/akraino/validation/tests/os/vuls/ + ${rc} = Run And Return Rc tar xvzf ${CURDIR}/db.tar.gz -C ${CURDIR} Should Be Equal As Integers ${rc} 0 ${os} = SSHLibrary.Execute Command source /etc/os-release && echo $ID - ${rc} = Run And Return Rc vuls scan -config config.toml -ssh-config + ${rc} = Run And Return Rc vuls scan -config ${CURDIR}/config.toml -ssh-config Should Be Equal As Integers ${rc} 0 - Run Keyword IF '${os}' == 'ubuntu' Run vuls for ubuntu ELSE IF '${os}' == 'centos' Run vuls for centos ELSE FAIL Distro '${os}' not supported + Run Keyword IF '${os}' == 'ubuntu' Run vuls for ubuntu ELSE IF '${os}' == 'centos' Run vuls for centos ELSE FAIL Distro '${os}' not supported *** Keywords *** Run vuls for ubuntu ${os_version} = SSHLibrary.Execute Command source /etc/os-release && echo $VERSION_ID | cut -d '.' -f1 - ${rc} ${output} = Run And Return Rc And Output vuls report -cvedb-sqlite3-path=${LOG_PATH}/cve.sqlite3 -ovaldb-sqlite3-path=${LOG_PATH}/oval_ubuntu_${os_version}.sqlite3 + ${rc} ${output} = Run And Return Rc And Output vuls report -config ${CURDIR}/config.toml -cvedb-sqlite3-path=${CURDIR}/cve.sqlite3 -ovaldb-sqlite3-path=${CURDIR}/oval_ubuntu_${os_version}.sqlite3 Should Be Equal As Integers ${rc} 0 Append To File ${LOG_PATH}/vuls.log ${output}${\n} Run vuls for centos - ${rc} ${output} = Run And Return Rc And Output vuls report -cvedb-sqlite3-path=${LOG_PATH}/cve.sqlite3 -ovaldb-sqlite3-path=${LOG_PATH}/oval_centos.sqlite3 -gostdb-sqlite3-path=${LOG_PATH}/gost_centos.sqlite3 + ${rc} ${output} = Run And Return Rc And Output vuls report -config ${CURDIR}/config.toml -cvedb-sqlite3-path=${CURDIR}/cve.sqlite3 -ovaldb-sqlite3-path=${CURDIR}/oval_centos.sqlite3 -gostdb-sqlite3-path=${CURDIR}/gost_centos.sqlite3 Should Be Equal As Integers ${rc} 0 Append To File ${LOG_PATH}/vuls.log ${output}${\n} Open Connection And Log In Open Connection ${HOST} - Login With Public Key ${USERNAME} ${SSH_KEYFILE} + Login With Public Key ${USERNAME} ${SSH_KEYFILE} \ No newline at end of file -- 2.16.6 From 5619a6390e3d65acf3638ed23bcabbff3b34b316 Mon Sep 17 00:00:00 2001 From: Juha Kosonen Date: Wed, 1 Apr 2020 10:28:30 +0300 Subject: [PATCH 08/16] Change the criticality of kube-hunter test Vulnerabilities discovered by kube-hunter does not impact to overall result of test suite execution. Change-Id: If3593412e68f397258de0fe7dcf86f102ae4d0ce Signed-off-by: Juha Kosonen --- bluval/bluval.py | 2 +- tests/k8s/kube-hunter/kube-hunter.resource | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bluval/bluval.py b/bluval/bluval.py index c79c830..63b301f 100644 --- a/bluval/bluval.py +++ b/bluval/bluval.py @@ -64,7 +64,7 @@ def run_testcase(testcase): # run the test args = ["robot", "-V", str(variables_updated_file), "-d", str(results_path), - "-b", "debug.log", str(test_path)] + "-n", "non-critical", "-b", "debug.log", str(test_path)] print('Executing testcase {}'.format(name)) print('show_stopper {}'.format(show_stopper)) diff --git a/tests/k8s/kube-hunter/kube-hunter.resource b/tests/k8s/kube-hunter/kube-hunter.resource index 4611e75..6651ddc 100644 --- a/tests/k8s/kube-hunter/kube-hunter.resource +++ b/tests/k8s/kube-hunter/kube-hunter.resource @@ -89,7 +89,9 @@ Delete Scan Job Should Be Equal As Integers ${result.rc} 0 Should Discover No Vulnerabilities - Should Be True ${STATUS} + Return From Keyword If ${STATUS} + Fail Vulnerabilities discovered + ... non-critical Run Scan Within Pod ${result}= Run Process kubectl apply -f ${CURDIR}/job.yaml -- 2.16.6 From d58226eb9ad62b7c489bf19326d4aa834d2a85d1 Mon Sep 17 00:00:00 2001 From: DANIEL STOICA Date: Fri, 3 Apr 2020 16:32:28 +0300 Subject: [PATCH 09/16] Enable kube-hunter tests and fix layer order Signed-off-by: DANIEL STOICA Depends-On: I4d36305d9ffd6fb7688002b6cb72a18baed77803 Change-Id: I5c19b527ec86be0a60b7c31fcd579f0802e9874d --- bluval/bluval-iec.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/bluval/bluval-iec.yaml b/bluval/bluval-iec.yaml index 86fbef2..09889b9 100644 --- a/bluval/bluval-iec.yaml +++ b/bluval/bluval-iec.yaml @@ -18,18 +18,8 @@ blueprint: name: demo layers: - - k8s - os - - k8s: &k8s - - - name: conformance - what: conformance - optional: "False" - - - name: kube-hunter - what: kube-hunter - optional: "True" + - k8s os: &os - @@ -40,3 +30,13 @@ blueprint: name: vuls what: vuls optional: "False" + + k8s: &k8s + - + name: conformance + what: conformance + optional: "False" + - + name: kube-hunter + what: kube-hunter + optional: "False" -- 2.16.6 From 473b026700468523d993927eb5282c8ae962d3b7 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 6 Apr 2020 16:06:43 +0000 Subject: [PATCH 10/16] docker: os: Allow caching vuls db between builds Vuls database files are quite large and take a lot of time to download, occasionally leading to out-of-memory issues on certain Jenkins slave build nodes. To overcome this limitation, allow the build process to cache its data between subsequent runs: - if the build host machine (i.e. build server) already has a file at /opt/akraino/validation/tests/os/vuls/db.tar.gz, that file will be included in the Docker build context and extracted during build (Dockerfile ADD implicitly does that for .tar.gz files), then brought up to date and re-archived; - to enable vuls DB caching, one can create an (initially) empty tar.gz file at the expected location on the host (build) machine using: $ make -C docker/os .init_db_cache This is a one time (manual) job, `make build` or equivalent calls will ensure the db.tar.gz file is kept up to date afterwards; Implementation quirks: - getting the updated db.tar.gz from the os docker image requires us to spawn the image as a running container (since Docker does not allow extracting files from the image directly), so we handle this step as a post-docker-build step via another double-colon `.build` make target; - since we want to perform both a pre-docker-build (copying build server persistent db.tar.gz to the current build dir, i.e. inside the Docker build context) and a post-docker-build step (copying back the updated db.tar.gz from the os docker image to the build server persistent location), we leverage double-colon Makefile targets for `.build`: * pre-docker-build .build (from docker/os/Makefile before the include ../build.mk statement) is executed first; * .build from ../build.mk (common for all Docker builds in validation) is then normally executed, resulting in a tagged docker image; * post-docker-build .build (from docker/os/Makefile after the include statement) is executed last; While at it, fix some related issues: - fix build issue caused by a new (changed upstream) dependency on python 'cryptography' package >= 2.5 (which would require SSL and FFI libs inside the destionation container where pip tries to compile the new requirement) by adding the new requirement to pip-requirements.txt, so we also create a wheel for it in the build container; - reduce verbosity of `go-cve-dictionary fetchnvd` to eliminate some non-printable characters from the job output; - reduce verbosity of wget dot progress; - add timestamp inside os build container to force Docker cache invalidation of database fetch/update step (to avoid missing upstream changes masked by Docker cache); - split build container vuls database fetch into 2 separate steps covering tool download/installation, respectively fetch/update, so we leverage the Docker cache for the first step while always invalidating the cache for the second; - `gost fetch` should use a number of threads comparable with the number of available processors; JIRA: VAL-101 Change-Id: I41d7bec0e72c92da1596abd67e2c1306ef9ffffa Signed-off-by: Alexandru Avadanii --- .gitignore | 1 + docker/os/Dockerfile | 28 +++++++++++++++++----------- docker/os/Makefile | 31 +++++++++++++++++++++++++++++++ docker/os/pip-requirements.txt | 1 + 4 files changed, 50 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 92f8161..30c15b6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ bluval/__pycache__ docker/manifest-tool +docker/os/db.tar.gz* *.sw? # Compiled class files diff --git a/docker/os/Dockerfile b/docker/os/Dockerfile index 676062d..c4d5938 100644 --- a/docker/os/Dockerfile +++ b/docker/os/Dockerfile @@ -57,10 +57,10 @@ RUN tar czvf /opt/akraino/ltp.tar.gz /opt/ltp WORKDIR /root/src RUN git clone https://github.com/CISOfy/lynis && tar czvf /opt/akraino/lynis-remote.tar.gz ./lynis -#Fetches vuls databases +# Fetches vuls databases (invalidate cache using unique timestamp) SHELL ["/bin/bash", "-c"] RUN if [ $(uname -m) == 'aarch64' ]; then HOST_ARCH=arm64; else HOST_ARCH=amd64; fi && \ - wget https://dl.google.com/go/go1.12.6.linux-$HOST_ARCH.tar.gz -P /root/ && \ + wget https://dl.google.com/go/go1.12.6.linux-$HOST_ARCH.tar.gz -P /root/ --progress=dot:giga && \ cd /root/ && \ tar -xzf go1.12.6.linux-$HOST_ARCH.tar.gz -C /root/ && \ rm go1.12.6.linux-$HOST_ARCH.tar.gz && \ @@ -76,23 +76,30 @@ RUN if [ $(uname -m) == 'aarch64' ]; then HOST_ARCH=arm64; else HOST_ARCH=amd64; git -C /root/go/src/github.com/kotakanbe clone https://github.com/kotakanbe/go-cve-dictionary.git && \ cd /root/go/src/github.com/kotakanbe/go-cve-dictionary/ && \ make install && \ - for i in $(seq 2002 "$(date +"%Y")"); do go-cve-dictionary fetchnvd -http-proxy=${HTTP_PROXY} -dbpath /opt/akraino/validation/tests/os/vuls/cve.sqlite3 -years "$i"; done && \ git -C /root/go/src/github.com/kotakanbe clone https://github.com/kotakanbe/goval-dictionary.git && \ cd /root/go/src/github.com/kotakanbe/goval-dictionary && \ make install && \ - goval-dictionary fetch-ubuntu -http-proxy=${HTTP_PROXY} -dbpath=/opt/akraino/validation/tests/os/vuls/oval_ubuntu_16.sqlite3 16 && \ - goval-dictionary fetch-ubuntu -http-proxy=${HTTP_PROXY} -dbpath=/opt/akraino/validation/tests/os/vuls/oval_ubuntu_18.sqlite3 18 && \ - goval-dictionary fetch-redhat -http-proxy=${HTTP_PROXY} -dbpath=/opt/akraino/validation/tests/os/vuls/oval_centos.sqlite3 7 && \ mkdir -p /root/go/src/github.com/knqyf263 && \ git -C /root/go/src/github.com/knqyf263 clone https://github.com/knqyf263/gost.git && \ cd /root/go/src/github.com/knqyf263/gost && \ - make install && \ - gost fetch redhat --http-proxy=${HTTP_PROXY} --dbpath=/opt/akraino/validation/tests/os/vuls/gost_centos.sqlite3 && \ + make install + +ADD db.tar.gz /opt/akraino/validation/tests/os/vuls/ +ADD db.tar.gz.timestamp /root/ +RUN \ + export GOROOT=/root/go && \ + export GOPATH=/root/go/src && \ + export PATH=$PATH:/root/go/bin:/root/go/src/bin && \ + for i in $(seq 2002 "$(date +"%Y")"); do go-cve-dictionary fetchnvd -quiet -http-proxy=${HTTP_PROXY} -dbpath /opt/akraino/validation/tests/os/vuls/cve.sqlite3 -years "$i"; done && \ + goval-dictionary fetch-ubuntu -http-proxy=${HTTP_PROXY} -dbpath=/opt/akraino/validation/tests/os/vuls/oval_ubuntu_16.sqlite3 16 && \ + goval-dictionary fetch-ubuntu -http-proxy=${HTTP_PROXY} -dbpath=/opt/akraino/validation/tests/os/vuls/oval_ubuntu_18.sqlite3 18 && \ + goval-dictionary fetch-redhat -http-proxy=${HTTP_PROXY} -dbpath=/opt/akraino/validation/tests/os/vuls/oval_centos.sqlite3 7 && \ + gost fetch redhat --http-proxy=${HTTP_PROXY} --dbpath=/opt/akraino/validation/tests/os/vuls/gost_centos.sqlite3 --threads=$(nproc) && \ cd /opt/akraino/validation/tests/os/vuls && \ tar cvzf db.tar.gz *.sqlite3 && \ rm *.sqlite3 -# Copy binaries into the final container and install robot framework +# Copy binaries into the final container and install robot framework, bluval dependencies FROM ubuntu:18.04 COPY --from=build /wheels /wheels COPY --from=build /opt/akraino/validation /opt/akraino/validation @@ -106,12 +113,11 @@ RUN apt-get update && apt-get -y install \ cd /usr/bin && ln -s python3 python && \ pip3 install -r /wheels/requirements/pip-requirements.txt \ -f /wheels && \ + pip3 install -r /opt/akraino/validation/bluval/requirements.txt && \ rm -rf /wheels && \ rm -rf /root/.cache/pip/* && \ rm -rf /var/cache/apt/* && \ rm -rf /var/lib/apt/lists/* -# Install bluval dependencies -RUN pip3 install -r /opt/akraino/validation/bluval/requirements.txt ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 diff --git a/docker/os/Makefile b/docker/os/Makefile index 4d43df8..5e5386b 100644 --- a/docker/os/Makefile +++ b/docker/os/Makefile @@ -14,6 +14,26 @@ # limitations under the License. # ############################################################################## +export OS_VULS_DB?=/opt/akraino/validation/tests/os/vuls/db.tar.gz + +.PHONY: .init_db_cache +.init_db_cache: + if [ ! -f $(OS_VULS_DB) ]; then \ + sudo mkdir -p $(dir $(OS_VULS_DB)); \ + sudo chown $(shell id -u):$(shell id -g) $(dir $(OS_VULS_DB)); \ + tar czf $(OS_VULS_DB) -T /dev/null; \ + fi + +.PHONY: .build +.build:: + if [ -f $(OS_VULS_DB) ]; then \ + echo DB cache file found, will be used during the build; \ + cp $(OS_VULS_DB) .; \ + else \ + tar czf $(notdir $(OS_VULS_DB)) -T /dev/null; \ + fi + date +%s > $(notdir $(OS_VULS_DB)).timestamp + .PHONY: all all: push-image .push_manifest @@ -24,3 +44,14 @@ build: .build push-image: .push_image include ../build.mk + +.PHONY: .build +.build:: + if [ -f $(OS_VULS_DB) ]; then \ + echo DB cache file found, will be updated; \ + docker run -v $(OS_VULS_DB):$(OS_VULS_DB).host \ + --rm --entrypoint cp \ + $(REGISTRY)/$(NAME):$(TAG_PRE)-$(HOST_ARCH)-$(TAG_VER) \ + $(OS_VULS_DB) $(OS_VULS_DB).host; \ + fi + rm -f $(notdir $(OS_VULS_DB))* diff --git a/docker/os/pip-requirements.txt b/docker/os/pip-requirements.txt index 6139a45..bd35f83 100644 --- a/docker/os/pip-requirements.txt +++ b/docker/os/pip-requirements.txt @@ -2,3 +2,4 @@ robotframework robotframework-httplibrary robotframework-requests robotframework-sshlibrary +cryptography>=2.5 -- 2.16.6 From b6ade8b653a44400a6c509a726e23c98317e40ab Mon Sep 17 00:00:00 2001 From: DANIEL STOICA Date: Tue, 7 Apr 2020 16:19:27 +0300 Subject: [PATCH 11/16] Change the criticality of OS security test JIRA: VAL-102 Added a checker for the lynis and vuls tests reports, if vulnerabilities are detected the tests fail with non-critical tag Signed-off-by: DANIEL STOICA Change-Id: I8399379aa60696cf6ce4c2aee1c2dcfb5f965fbc --- tests/os/lynis/lynis.robot | 3 +++ tests/os/vuls/vuls.robot | 8 +++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/os/lynis/lynis.robot b/tests/os/lynis/lynis.robot index 7d88c0f..de05d8d 100644 --- a/tests/os/lynis/lynis.robot +++ b/tests/os/lynis/lynis.robot @@ -40,6 +40,9 @@ Run Lynis Audit System Append To File ${log} ${stdout}${\n} Should Be Equal As Integers ${rc} 0 + ${status} = Evaluate "Great, no warnings" in """${stdout}""" + Run Keyword If '${status}' == 'False' FAIL Warnings discovered + ... non-critical *** Keywords *** Open Connection And Log In diff --git a/tests/os/vuls/vuls.robot b/tests/os/vuls/vuls.robot index 3b3a901..d79fab9 100644 --- a/tests/os/vuls/vuls.robot +++ b/tests/os/vuls/vuls.robot @@ -46,6 +46,10 @@ Run Vuls test Run Keyword IF '${os}' == 'ubuntu' Run vuls for ubuntu ELSE IF '${os}' == 'centos' Run vuls for centos ELSE FAIL Distro '${os}' not supported + ${status} = Evaluate "Total: 0" in """${LOG}""" + Run Keyword If '${status}' == 'False' FAIL Vulnerabilities discovered + ... non-critical + *** Keywords *** Run vuls for ubuntu ${os_version} = SSHLibrary.Execute Command source /etc/os-release && echo $VERSION_ID | cut -d '.' -f1 @@ -53,12 +57,14 @@ Run vuls for ubuntu ${rc} ${output} = Run And Return Rc And Output vuls report -config ${CURDIR}/config.toml -cvedb-sqlite3-path=${CURDIR}/cve.sqlite3 -ovaldb-sqlite3-path=${CURDIR}/oval_ubuntu_${os_version}.sqlite3 Should Be Equal As Integers ${rc} 0 Append To File ${LOG_PATH}/vuls.log ${output}${\n} + Set Global Variable ${LOG} ${output} Run vuls for centos ${rc} ${output} = Run And Return Rc And Output vuls report -config ${CURDIR}/config.toml -cvedb-sqlite3-path=${CURDIR}/cve.sqlite3 -ovaldb-sqlite3-path=${CURDIR}/oval_centos.sqlite3 -gostdb-sqlite3-path=${CURDIR}/gost_centos.sqlite3 Should Be Equal As Integers ${rc} 0 Append To File ${LOG_PATH}/vuls.log ${output}${\n} + Set Global Variable ${LOG} ${output} Open Connection And Log In Open Connection ${HOST} - Login With Public Key ${USERNAME} ${SSH_KEYFILE} \ No newline at end of file + Login With Public Key ${USERNAME} ${SSH_KEYFILE} -- 2.16.6 From 058638cc69d8195dfa5d5c182a976b875988df96 Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Thu, 16 Apr 2020 15:04:22 +0200 Subject: [PATCH 12/16] docker: kube-conformance: Enforce upstream tag Stop using environment TAG_VER when tagging kube-conformance images, as they should be tagged with the upstream (fixed) tag, currently 'v1.16' (instead of the validation release tag, e.g. '3.0.0'). JIRA: VAL-105 Change-Id: I1ea7e7395e7a796c7eb22fcdad41ba92c8d555ac Signed-off-by: Alexandru Avadanii --- docker/kube-conformance/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/kube-conformance/Makefile b/docker/kube-conformance/Makefile index ee7b40f..1c0f7f0 100644 --- a/docker/kube-conformance/Makefile +++ b/docker/kube-conformance/Makefile @@ -16,7 +16,7 @@ export WORK_DIR?=$(TAG_PRE) export KUBE_VERSION=1.16 -export TAG_VER?=v$(KUBE_VERSION) +export TAG_VER=v$(KUBE_VERSION) .PHONY: .build .build:: .submodules_patched -- 2.16.6 From 21b8334662643b1877a2be70152a9b631a685af7 Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Fri, 17 Apr 2020 17:50:49 +0200 Subject: [PATCH 13/16] bluval: Add Jenkins log-parser rules Instead of creating the log-parser rules on the fly from the CI job, add the rules to the current repository, allowing the Validation committers to tweak them independently (not through ci-management changes). While at it, extend and improve the previous rules by: - raising a warning instead of an error for robot FAIL messages (which will later be changed via ci-management to _not_ fail the CI job run); - separating the test suites into log parsing sections; - raising an info for PASS robot messages (so they are listed in the log-parser links section for better readability); - fail the CI job only if critical tests failed; For logparser section grouping to work as expected, fix one output buffering issue that used to print the issued commands after their output (and also fix another output buffering related issue to keep things uniform). JIRA: VAL-104 Change-Id: I1506d84a44926e80913562f67f24885a98f94df7 Signed-off-by: Alexandru Avadanii --- bluval/blucon.py | 2 +- bluval/bluval.py | 2 +- bluval/rules.txt | 10 ++++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 bluval/rules.txt diff --git a/bluval/blucon.py b/bluval/blucon.py index 0d5d7ca..2fce9d0 100644 --- a/bluval/blucon.py +++ b/bluval/blucon.py @@ -66,7 +66,7 @@ def invoke_docker(bluprint, layer): args = [cmd] try: - print('\nInvoking {}'.format(args)) + print('\nInvoking {}'.format(args), flush=True) subprocess.call(args, shell=True) except OSError: #print('Error while executing {}'.format(args)) diff --git a/bluval/bluval.py b/bluval/bluval.py index 63b301f..73cc68a 100644 --- a/bluval/bluval.py +++ b/bluval/bluval.py @@ -68,7 +68,7 @@ def run_testcase(testcase): print('Executing testcase {}'.format(name)) print('show_stopper {}'.format(show_stopper)) - print('Invoking {}'.format(args)) + print('Invoking {}'.format(args), flush=True) try: status = subprocess.call(args, shell=False) if status != 0 and show_stopper.lower() == "true": diff --git a/bluval/rules.txt b/bluval/rules.txt new file mode 100644 index 0000000..f2273ff --- /dev/null +++ b/bluval/rules.txt @@ -0,0 +1,10 @@ +# Separate results based on suite (group the messages by suite) +start /(?:Executing testcase ).*$/ + +# Only fail if some critical tests failed +debug /critical test.*passed, 0 failed/ +error /critical test.*passed/ + +# Suite/subsuite test results will be marked as info/warn +info /PASS/ +warn /FAIL/ -- 2.16.6 From 54018ed1ad8b1a85b32f6b6cdea7d36ed70824b5 Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Sat, 18 Apr 2020 18:42:08 +0200 Subject: [PATCH 14/16] blucon.sh: Allow validation/results dir override When running in CI, the validation git repository is cloned in the job workspace, which usually has a custom directory name matching the job name (e.g. bluval-daily-master) instead of the previously hardcoded 'validation' directory name. Also, the results dir should not be a sibling of the validation git repo directory when running in CI, but a subdirectory of the said dir. To accomodate this, allow overriding both previously hardcoded directory paths via environment variables (VALIDATION_DIR, RESULTS_DIR). While at it, add a new '-t' argument for specifying the Docker tag to be used for the validation docker images (previously hardcoded to 'latest') in order to support running docker images associated to a specific Validation release tag (e.g. 3.0.0). JIRA: VAL-95 Change-Id: I4d3c10881de28c64bcca05ff23aa1025f67a1f5e Signed-off-by: Alexandru Avadanii --- bluval/blucon.py | 22 +++++++++++++--------- bluval/blucon.sh | 26 +++++++++++++++----------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/bluval/blucon.py b/bluval/blucon.py index 2fce9d0..ee7f706 100644 --- a/bluval/blucon.py +++ b/bluval/blucon.py @@ -53,16 +53,16 @@ def get_volumes(layer): return volume_list -def invoke_docker(bluprint, layer): +def invoke_docker(bluprint, layer, tag): """Start docker container for given layer """ volume_list = get_volumes('common') + get_volumes(layer) cmd = ("docker run --rm" + volume_list + _SUBNET + - " akraino/validation:{0}-latest" + " akraino/validation:{0}-{3}" " /bin/sh -c" " 'cd /opt/akraino/validation " "&& python -B bluval/bluval.py -l {0} {1} {2}'" - .format(layer, ("-o" if _OPTIONAL_ALSO else ""), bluprint)) + .format(layer, ("-o" if _OPTIONAL_ALSO else ""), bluprint, tag)) args = [cmd] try: @@ -73,7 +73,7 @@ def invoke_docker(bluprint, layer): raise BluvalError(OSError) -def invoke_dockers(yaml_loc, layer, blueprint_name): +def invoke_dockers(yaml_loc, layer, blueprint_name, tag): """Parses yaml file and starts docker container for one/all layers """ with open(str(yaml_loc)) as yaml_file: @@ -81,19 +81,20 @@ def invoke_dockers(yaml_loc, layer, blueprint_name): blueprint = yamldoc['blueprint'] if layer is None or layer == "all": for each_layer in blueprint['layers']: - invoke_docker(blueprint_name, each_layer) + invoke_docker(blueprint_name, each_layer, tag) else: - invoke_docker(blueprint_name, layer) + invoke_docker(blueprint_name, layer, tag) @click.command() @click.argument('blueprint') @click.option('--layer', '-l') @click.option('--network', '-n') +@click.option('--tag', '-t') @click.option('--optional_also', '-o', is_flag=True) -def main(blueprint, layer, network, optional_also): +def main(blueprint, layer, network, tag, optional_also): """Takes blueprint name and optional layer. Validates inputs and derives - yaml location from blueprint name. Invokes validate on blue print. + yaml location from blueprint name. Invokes validate on blueprint. """ global _OPTIONAL_ALSO # pylint: disable=global-statement global _SUBNET # pylint: disable=global-statement @@ -107,8 +108,11 @@ def main(blueprint, layer, network, optional_also): if network is not None: _SUBNET = " --net=" + network print("Using", _SUBNET) + if tag is None: + tag = 'latest' + print("Using tag {}".format(tag)) try: - invoke_dockers(yaml_loc, layer, blueprint) + invoke_dockers(yaml_loc, layer, blueprint, tag) except ShowStopperError as err: print('ShowStopperError:', err) except BluvalError as err: diff --git a/bluval/blucon.sh b/bluval/blucon.sh index afe54db..b882f92 100755 --- a/bluval/blucon.sh +++ b/bluval/blucon.sh @@ -18,14 +18,17 @@ if [ -z "$AKRAINO_HOME" ] then - echo "AKRAINO_HOME not available. Setting..." - this_file="$(readlink -f $0)" - bluval_dir="$(dirname $this_file)" - validation_dir="$(dirname $bluval_dir)" - parent_dir="$(dirname $validation_dir)" - export AKRAINO_HOME="$parent_dir" + echo "AKRAINO_HOME not available. Setting ..." + AKRAINO_HOME="$(readlink -f "$(dirname "$0")/../..")" fi + +# Allow overriding VALIDATION_DIR and/or RESULTS_DIR via env vars +VALIDATION_DIR=${VALIDATION_DIR:-"${AKRAINO_HOME}/validation"} +RESULTS_DIR=${RESULTS_DIR:-"${AKRAINO_HOME}/results"} + echo "AKRAINO_HOME=$AKRAINO_HOME" +echo "VALIDATION_DIR=$VALIDATION_DIR" +echo "RESULTS_DIR=$RESULTS_DIR" if [ "$#" -eq 0 ] then @@ -38,6 +41,7 @@ then Options: -l, --layer TEXT -n, --network TEXT + -t, --tag TEXT -o, --optional_also --help Show this message and exit.' @@ -45,13 +49,13 @@ then fi echo "Building docker image" -image_tag=$( (git branch || echo "* local") | grep "^\*" | awk '{print $2}') -docker build -t akraino/validation:blucon-$image_tag $AKRAINO_HOME/validation/bluval +img="akraino/validation:blucon-$(git rev-parse --abbrev-ref HEAD || echo local)" +docker build -t "$img" "$VALIDATION_DIR/bluval" set -x docker run --rm \ -v /var/run/docker.sock:/var/run/docker.sock \ - -v $AKRAINO_HOME/results:/opt/akraino/results \ - -v $AKRAINO_HOME/validation:/opt/akraino/validation \ - akraino/validation:blucon-$image_tag "$@" + -v "$RESULTS_DIR":/opt/akraino/results \ + -v "$VALIDATION_DIR":/opt/akraino/validation \ + "$img" "$@" -- 2.16.6 From fbde8757f3d54d62cf5ba8adf2789a338540871e Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Wed, 22 Apr 2020 15:14:47 +0200 Subject: [PATCH 15/16] blucon: Add argument '-p' for pulling latest tag If `--pull` or `-p` argument is passed to blucon, it will now run a `docker pull` command prior to launching the testcases with `docker run`. This should prevent the testcases from using stale docker images if already present on the machine. JIRA: VAL-107 Change-Id: Id4b7947a0de7abf07f69812c6deef71919b9041a Signed-off-by: Alexandru Avadanii --- bluval/blucon.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/bluval/blucon.py b/bluval/blucon.py index ee7f706..8072382 100644 --- a/bluval/blucon.py +++ b/bluval/blucon.py @@ -29,6 +29,7 @@ import yaml from bluutil import BluvalError from bluutil import ShowStopperError +_PULL = False _OPTIONAL_ALSO = False _SUBNET = "" @@ -53,9 +54,25 @@ def get_volumes(layer): return volume_list +def pull_docker(layer, tag): + """Pull docker image for given layer + """ + cmd = ("docker pull akraino/validation:{0}-{1}" + .format(layer, tag)) + + args = [cmd] + try: + print('\nPulling image using {}'.format(args), flush=True) + subprocess.call(args, shell=True) + except OSError: + raise BluvalError(OSError) + + def invoke_docker(bluprint, layer, tag): """Start docker container for given layer """ + if _PULL: + pull_docker(layer, tag) volume_list = get_volumes('common') + get_volumes(layer) cmd = ("docker run --rm" + volume_list + _SUBNET + " akraino/validation:{0}-{3}" @@ -92,16 +109,22 @@ def invoke_dockers(yaml_loc, layer, blueprint_name, tag): @click.option('--network', '-n') @click.option('--tag', '-t') @click.option('--optional_also', '-o', is_flag=True) -def main(blueprint, layer, network, tag, optional_also): +@click.option('--pull', '-p', is_flag=True) +# pylint: disable=too-many-arguments +def main(blueprint, layer, network, tag, optional_also, pull): """Takes blueprint name and optional layer. Validates inputs and derives yaml location from blueprint name. Invokes validate on blueprint. """ + global _PULL # pylint: disable=global-statement global _OPTIONAL_ALSO # pylint: disable=global-statement global _SUBNET # pylint: disable=global-statement mypath = Path(__file__).absolute() yaml_loc = mypath.parents[0].joinpath('bluval-{}.yaml'.format(blueprint)) if layer is not None: layer = layer.lower() + if pull: + _PULL = True + print("_PULL {}".format(_PULL)) if optional_also: _OPTIONAL_ALSO = True print("_OPTIONAL_ALSO {}".format(_OPTIONAL_ALSO)) -- 2.16.6 From 528222a0d0fff5208e298548a567f3a63ef4358a Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Fri, 24 Apr 2020 16:49:52 +0200 Subject: [PATCH 16/16] k8s conformance: Sanitize major/minor version str Since we use `kubectl version` to determine K8s client/server version strings, the major/minor semver components might include some custom suffixes, e.g. '1.13+abcdef' would be split into '1' and '13+abcdef'. Avoid including the custom semver suffixes into the kube-conformance image tag name by sanitizating the major/minor version strings. JIRA: VAL-111 Change-Id: Iac4f719e4e982b90b50a23e5bcaeff4bda9fe104 Signed-off-by: Alexandru Avadanii --- tests/k8s/conformance/conformance.robot | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/k8s/conformance/conformance.robot b/tests/k8s/conformance/conformance.robot index edd2271..c51f217 100644 --- a/tests/k8s/conformance/conformance.robot +++ b/tests/k8s/conformance/conformance.robot @@ -148,6 +148,8 @@ Define Images ${versions}= Convert String To JSON ${result.stdout} ${major}= Get Value From Json ${versions} $.serverVersion.major ${minor}= Get Value From Json ${versions} $.serverVersion.minor + ${major}= Get Regexp Matches ${major[0]} \\d+ + ${minor}= Get Regexp Matches ${minor[0]} \\d+ Set To Dictionary ${SONOBUOY_IMGS['e2e']} name=validation:kube-conformance-v${major[0]}.${minor[0]} Onboard Images -- 2.16.6