From: Krisztian Lengyel <krisztian.lengyel@nokia.com>
Date: Wed, 25 Sep 2019 12:42:08 +0000 (+0200)
Subject: Disable robots.txt parsing for wget download
X-Git-Url: https://gerrit.akraino.org/r/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F50%2F1650%2F1;hp=5cb76c5bde04265caaa3241aafd57c67073bb1db;p=ta%2Fbuild-tools.git

Disable robots.txt parsing for wget download

Change-Id: Iab84213c2ef657fec22c9dea695fc01bbe4433ef
Signed-off-by: Krisztian Lengyel <krisztian.lengyel@nokia.com>
---

diff --git a/build_step_create_install_cd.sh b/build_step_create_install_cd.sh
index 40d3880..abfd91d 100755
--- a/build_step_create_install_cd.sh
+++ b/build_step_create_install_cd.sh
@@ -50,7 +50,14 @@ wget_dir() {
   url=$(remove_extra_slashes_from_url $url)
   # count cut length in case url depth changes
   cut_dirs=$(echo $url | sed -re 's|.*://[^/]+/(.+)|\1|' -e 's|/$||' | grep -o / | wc -l)
-  wget -N -r --no-host-directories --no-verbose --cut-dirs=${cut_dirs} --reject index.html* --no-parent $url
+  wget -N -r \
+    --no-host-directories \
+    --no-verbose \
+    --cut-dirs=${cut_dirs} \
+    --reject index.html* \
+    --no-parent \
+    --execute robots=off \
+    $url
 }
 
 pushd $iso_build_dir