From: Your Name Date: Mon, 6 Apr 2020 16:06:43 +0000 (+0000) Subject: docker: os: Allow caching vuls db between builds X-Git-Tag: 3.0.0~1 X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F40%2F3340%2F3;p=validation.git docker: os: Allow caching vuls db between builds Vuls database files are quite large and take a lot of time to download, occasionally leading to out-of-memory issues on certain Jenkins slave build nodes. To overcome this limitation, allow the build process to cache its data between subsequent runs: - if the build host machine (i.e. build server) already has a file at /opt/akraino/validation/tests/os/vuls/db.tar.gz, that file will be included in the Docker build context and extracted during build (Dockerfile ADD implicitly does that for .tar.gz files), then brought up to date and re-archived; - to enable vuls DB caching, one can create an (initially) empty tar.gz file at the expected location on the host (build) machine using: $ make -C docker/os .init_db_cache This is a one time (manual) job, `make build` or equivalent calls will ensure the db.tar.gz file is kept up to date afterwards; Implementation quirks: - getting the updated db.tar.gz from the os docker image requires us to spawn the image as a running container (since Docker does not allow extracting files from the image directly), so we handle this step as a post-docker-build step via another double-colon `.build` make target; - since we want to perform both a pre-docker-build (copying build server persistent db.tar.gz to the current build dir, i.e. inside the Docker build context) and a post-docker-build step (copying back the updated db.tar.gz from the os docker image to the build server persistent location), we leverage double-colon Makefile targets for `.build`: * pre-docker-build .build (from docker/os/Makefile before the include ../build.mk statement) is executed first; * .build from ../build.mk (common for all Docker builds in validation) is then normally executed, resulting in a tagged docker image; * post-docker-build .build (from docker/os/Makefile after the include statement) is executed last; While at it, fix some related issues: - fix build issue caused by a new (changed upstream) dependency on python 'cryptography' package >= 2.5 (which would require SSL and FFI libs inside the destionation container where pip tries to compile the new requirement) by adding the new requirement to pip-requirements.txt, so we also create a wheel for it in the build container; - reduce verbosity of `go-cve-dictionary fetchnvd` to eliminate some non-printable characters from the job output; - reduce verbosity of wget dot progress; - add timestamp inside os build container to force Docker cache invalidation of database fetch/update step (to avoid missing upstream changes masked by Docker cache); - split build container vuls database fetch into 2 separate steps covering tool download/installation, respectively fetch/update, so we leverage the Docker cache for the first step while always invalidating the cache for the second; - `gost fetch` should use a number of threads comparable with the number of available processors; JIRA: VAL-101 Change-Id: I41d7bec0e72c92da1596abd67e2c1306ef9ffffa Signed-off-by: Alexandru Avadanii --- diff --git a/.gitignore b/.gitignore index 92f8161..30c15b6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ bluval/__pycache__ docker/manifest-tool +docker/os/db.tar.gz* *.sw? # Compiled class files diff --git a/docker/os/Dockerfile b/docker/os/Dockerfile index 676062d..c4d5938 100644 --- a/docker/os/Dockerfile +++ b/docker/os/Dockerfile @@ -57,10 +57,10 @@ RUN tar czvf /opt/akraino/ltp.tar.gz /opt/ltp WORKDIR /root/src RUN git clone https://github.com/CISOfy/lynis && tar czvf /opt/akraino/lynis-remote.tar.gz ./lynis -#Fetches vuls databases +# Fetches vuls databases (invalidate cache using unique timestamp) SHELL ["/bin/bash", "-c"] RUN if [ $(uname -m) == 'aarch64' ]; then HOST_ARCH=arm64; else HOST_ARCH=amd64; fi && \ - wget https://dl.google.com/go/go1.12.6.linux-$HOST_ARCH.tar.gz -P /root/ && \ + wget https://dl.google.com/go/go1.12.6.linux-$HOST_ARCH.tar.gz -P /root/ --progress=dot:giga && \ cd /root/ && \ tar -xzf go1.12.6.linux-$HOST_ARCH.tar.gz -C /root/ && \ rm go1.12.6.linux-$HOST_ARCH.tar.gz && \ @@ -76,23 +76,30 @@ RUN if [ $(uname -m) == 'aarch64' ]; then HOST_ARCH=arm64; else HOST_ARCH=amd64; git -C /root/go/src/github.com/kotakanbe clone https://github.com/kotakanbe/go-cve-dictionary.git && \ cd /root/go/src/github.com/kotakanbe/go-cve-dictionary/ && \ make install && \ - for i in $(seq 2002 "$(date +"%Y")"); do go-cve-dictionary fetchnvd -http-proxy=${HTTP_PROXY} -dbpath /opt/akraino/validation/tests/os/vuls/cve.sqlite3 -years "$i"; done && \ git -C /root/go/src/github.com/kotakanbe clone https://github.com/kotakanbe/goval-dictionary.git && \ cd /root/go/src/github.com/kotakanbe/goval-dictionary && \ make install && \ - goval-dictionary fetch-ubuntu -http-proxy=${HTTP_PROXY} -dbpath=/opt/akraino/validation/tests/os/vuls/oval_ubuntu_16.sqlite3 16 && \ - goval-dictionary fetch-ubuntu -http-proxy=${HTTP_PROXY} -dbpath=/opt/akraino/validation/tests/os/vuls/oval_ubuntu_18.sqlite3 18 && \ - goval-dictionary fetch-redhat -http-proxy=${HTTP_PROXY} -dbpath=/opt/akraino/validation/tests/os/vuls/oval_centos.sqlite3 7 && \ mkdir -p /root/go/src/github.com/knqyf263 && \ git -C /root/go/src/github.com/knqyf263 clone https://github.com/knqyf263/gost.git && \ cd /root/go/src/github.com/knqyf263/gost && \ - make install && \ - gost fetch redhat --http-proxy=${HTTP_PROXY} --dbpath=/opt/akraino/validation/tests/os/vuls/gost_centos.sqlite3 && \ + make install + +ADD db.tar.gz /opt/akraino/validation/tests/os/vuls/ +ADD db.tar.gz.timestamp /root/ +RUN \ + export GOROOT=/root/go && \ + export GOPATH=/root/go/src && \ + export PATH=$PATH:/root/go/bin:/root/go/src/bin && \ + for i in $(seq 2002 "$(date +"%Y")"); do go-cve-dictionary fetchnvd -quiet -http-proxy=${HTTP_PROXY} -dbpath /opt/akraino/validation/tests/os/vuls/cve.sqlite3 -years "$i"; done && \ + goval-dictionary fetch-ubuntu -http-proxy=${HTTP_PROXY} -dbpath=/opt/akraino/validation/tests/os/vuls/oval_ubuntu_16.sqlite3 16 && \ + goval-dictionary fetch-ubuntu -http-proxy=${HTTP_PROXY} -dbpath=/opt/akraino/validation/tests/os/vuls/oval_ubuntu_18.sqlite3 18 && \ + goval-dictionary fetch-redhat -http-proxy=${HTTP_PROXY} -dbpath=/opt/akraino/validation/tests/os/vuls/oval_centos.sqlite3 7 && \ + gost fetch redhat --http-proxy=${HTTP_PROXY} --dbpath=/opt/akraino/validation/tests/os/vuls/gost_centos.sqlite3 --threads=$(nproc) && \ cd /opt/akraino/validation/tests/os/vuls && \ tar cvzf db.tar.gz *.sqlite3 && \ rm *.sqlite3 -# Copy binaries into the final container and install robot framework +# Copy binaries into the final container and install robot framework, bluval dependencies FROM ubuntu:18.04 COPY --from=build /wheels /wheels COPY --from=build /opt/akraino/validation /opt/akraino/validation @@ -106,12 +113,11 @@ RUN apt-get update && apt-get -y install \ cd /usr/bin && ln -s python3 python && \ pip3 install -r /wheels/requirements/pip-requirements.txt \ -f /wheels && \ + pip3 install -r /opt/akraino/validation/bluval/requirements.txt && \ rm -rf /wheels && \ rm -rf /root/.cache/pip/* && \ rm -rf /var/cache/apt/* && \ rm -rf /var/lib/apt/lists/* -# Install bluval dependencies -RUN pip3 install -r /opt/akraino/validation/bluval/requirements.txt ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 diff --git a/docker/os/Makefile b/docker/os/Makefile index 4d43df8..5e5386b 100644 --- a/docker/os/Makefile +++ b/docker/os/Makefile @@ -14,6 +14,26 @@ # limitations under the License. # ############################################################################## +export OS_VULS_DB?=/opt/akraino/validation/tests/os/vuls/db.tar.gz + +.PHONY: .init_db_cache +.init_db_cache: + if [ ! -f $(OS_VULS_DB) ]; then \ + sudo mkdir -p $(dir $(OS_VULS_DB)); \ + sudo chown $(shell id -u):$(shell id -g) $(dir $(OS_VULS_DB)); \ + tar czf $(OS_VULS_DB) -T /dev/null; \ + fi + +.PHONY: .build +.build:: + if [ -f $(OS_VULS_DB) ]; then \ + echo DB cache file found, will be used during the build; \ + cp $(OS_VULS_DB) .; \ + else \ + tar czf $(notdir $(OS_VULS_DB)) -T /dev/null; \ + fi + date +%s > $(notdir $(OS_VULS_DB)).timestamp + .PHONY: all all: push-image .push_manifest @@ -24,3 +44,14 @@ build: .build push-image: .push_image include ../build.mk + +.PHONY: .build +.build:: + if [ -f $(OS_VULS_DB) ]; then \ + echo DB cache file found, will be updated; \ + docker run -v $(OS_VULS_DB):$(OS_VULS_DB).host \ + --rm --entrypoint cp \ + $(REGISTRY)/$(NAME):$(TAG_PRE)-$(HOST_ARCH)-$(TAG_VER) \ + $(OS_VULS_DB) $(OS_VULS_DB).host; \ + fi + rm -f $(notdir $(OS_VULS_DB))* diff --git a/docker/os/pip-requirements.txt b/docker/os/pip-requirements.txt index 6139a45..bd35f83 100644 --- a/docker/os/pip-requirements.txt +++ b/docker/os/pip-requirements.txt @@ -2,3 +2,4 @@ robotframework robotframework-httplibrary robotframework-requests robotframework-sshlibrary +cryptography>=2.5