From 7ff0f5c599f7246470373fc04d722f43d8dc7eb5 Mon Sep 17 00:00:00 2001
From: Krisztian Lengyel <krisztian.lengyel@nokia.com>
Date: Wed, 25 Sep 2019 14:42:08 +0200
Subject: [PATCH] Disable robots.txt parsing for wget download

Change-Id: Iab84213c2ef657fec22c9dea695fc01bbe4433ef
Signed-off-by: Krisztian Lengyel <krisztian.lengyel@nokia.com>
---
 build_step_create_install_cd.sh | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/build_step_create_install_cd.sh b/build_step_create_install_cd.sh
index 40d3880..abfd91d 100755
--- a/build_step_create_install_cd.sh
+++ b/build_step_create_install_cd.sh
@@ -50,7 +50,14 @@ wget_dir() {
   url=$(remove_extra_slashes_from_url $url)
   # count cut length in case url depth changes
   cut_dirs=$(echo $url | sed -re 's|.*://[^/]+/(.+)|\1|' -e 's|/$||' | grep -o / | wc -l)
-  wget -N -r --no-host-directories --no-verbose --cut-dirs=${cut_dirs} --reject index.html* --no-parent $url
+  wget -N -r \
+    --no-host-directories \
+    --no-verbose \
+    --cut-dirs=${cut_dirs} \
+    --reject index.html* \
+    --no-parent \
+    --execute robots=off \
+    $url
 }
 
 pushd $iso_build_dir
-- 
2.16.6