Disable robots.txt parsing for wget download

Change-Id: Iab84213c2ef657fec22c9dea695fc01bbe4433ef
Signed-off-by: Krisztian Lengyel <krisztian.lengyel@nokia.com>

diff --git a/build_step_create_install_cd.sh b/build_step_create_install_cd.sh
index 40d3880..abfd91d 100755 (executable)
--- a/build_step_create_install_cd.sh
+++ b/build_step_create_install_cd.sh
@@ -50,7 +50,14 @@ wget_dir() {
   url=$(remove_extra_slashes_from_url $url)
   # count cut length in case url depth changes
   cut_dirs=$(echo $url | sed -re 's|.*://[^/]+/(.+)|\1|' -e 's|/$||' | grep -o / | wc -l)
-  wget -N -r --no-host-directories --no-verbose --cut-dirs=${cut_dirs} --reject index.html* --no-parent $url
+  wget -N -r \
+    --no-host-directories \
+    --no-verbose \
+    --cut-dirs=${cut_dirs} \
+    --reject index.html* \
+    --no-parent \
+    --execute robots=off \
+    $url
 }
 
 pushd $iso_build_dir