Disable robots.txt parsing for wget download

[ta/build-tools.git] / build_step_create_install_cd.sh

diff --git a/build_step_create_install_cd.sh b/build_step_create_install_cd.sh
index 40d3880..abfd91d 100755 (executable)
--- a/build_step_create_install_cd.sh
+++ b/build_step_create_install_cd.sh
@@ -50,7 +50,14 @@ wget_dir() {
   url=$(remove_extra_slashes_from_url $url)
   # count cut length in case url depth changes
   cut_dirs=$(echo $url | sed -re 's|.*://[^/]+/(.+)|\1|' -e 's|/$||' | grep -o / | wc -l)
-  wget -N -r --no-host-directories --no-verbose --cut-dirs=${cut_dirs} --reject index.html* --no-parent $url
+  wget -N -r \
+    --no-host-directories \
+    --no-verbose \
+    --cut-dirs=${cut_dirs} \
+    --reject index.html* \
+    --no-parent \
+    --execute robots=off \
+    $url
 }
 
 pushd $iso_build_dir