Merge remote-tracking branch 'origin/tcwg-staging' into tcwg-llvmprodtcwg-llvmprod

Change-Id: I2f6c8930039c7399f6f58732bafca170d4baa2ad

3 files changed, 92 insertions, 36 deletions

diff --git a/tcwg-base/tcwg-llvmbot/Dockerfile.in b/tcwg-base/tcwg-llvmbot/Dockerfile.in
index 24233ed9..5771509b 100644
--- a/tcwg-base/tcwg-llvmbot/Dockerfile.in
+++ b/tcwg-base/tcwg-llvmbot/Dockerfile.in
@@ -1,8 +1,12 @@
 FROM linaro/ci-#{ARCH}-tcwg-base-ubuntu:#{DISTRO}
 
 RUN apt-get update \
+ && DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common \
+ && DEBIAN_FRONTEND=noninteractive add-apt-repository -y ppa:ubuntu-toolchain-r/test \
+ && apt-get update \
  && DEBIAN_FRONTEND=noninteractive apt-get install -y \
  buildbot \
+ g++-7 \
  && apt-get clean \
  && rm -rf \
  /var/lib/apt/lists/* \
@@ -21,14 +25,6 @@ RUN cd ./llvm-config-buildslave/ \
  /var/tmp/* \
  && sudo -i -u buildslave mkdir /home/buildslave/buildslave
 
-# Add ninja with support for memory-threshold job limitation.
-RUN git clone -b master https://github.com/maxim-kuvyrkov/ninja.git \
- && cd ninja \
- && ./configure.py --bootstrap && ./ninja all && ./ninja_test \
- && mv ninja /usr/local/bin/ninja.bin \
- && cd .. \
- && rm -rf ninja
-
 # Add current clang for libcxx bots.
 RUN \
 #if ARCH_arm64
diff --git a/tcwg-base/tcwg-llvmbot/run.sh b/tcwg-base/tcwg-llvmbot/run.sh
index aec8e31b..04bd6450 100755
--- a/tcwg-base/tcwg-llvmbot/run.sh
+++ b/tcwg-base/tcwg-llvmbot/run.sh
@@ -4,10 +4,10 @@ set -e
 
 bare_metal_bot_p ()
 {
-    case "$1" in
-	"linaro-tk1-"*) return 0 ;;
-	*) return 1 ;;
-    esac
+    if [ -f "/.dockerenv" ]; then
+       return 1
+    fi
+    return 0
 }
 
 use_clang_p ()
@@ -31,6 +31,8 @@ use_clang_p ()
 # Use the oldest maintained clang release (latest - 1).
 setup_clang_release()
 {
+    local bot_name="$1"
+
     # There is a 6.0.1 release but there aren't any AArch64 binaries available
     # so we use 6.0.0 for now.
     local release_num=6.0.0
@@ -44,11 +46,14 @@ setup_clang_release()
     esac
 
     # Download and install clang+llvm into /usr/local
-    (
-	cd /usr/local
-	wget -c --progress=dot:giga http://releases.llvm.org/${release_num}/$clang_ver.tar.xz
-	tar xf $clang_ver.tar.xz
-    )
+    # Docker bots already have clang+llvm downloaded and installed in the image.
+    if bare_metal_bot_p $bot_name; then
+	(
+	    cd /usr/local
+	    wget -c --progress=dot:giga http://releases.llvm.org/${release_num}/$clang_ver.tar.xz
+	    tar xf $clang_ver.tar.xz
+	)
+    fi
     cc=/usr/local/$clang_ver/bin/clang
     cxx=/usr/local/$clang_ver/bin/clang++
 }
@@ -65,10 +70,10 @@ if ! [ -f ~buildslave/buildslave/buildbot.tac ]; then
 fi
 
 if use_clang_p $2 ; then
-    setup_clang_release
+    setup_clang_release $2
 else
-    cc=gcc
-    cxx=g++
+    cc=gcc-7
+    cxx=g++-7
 fi
 
 # With default PATH /usr/local/bin/cc and /usr/local/bin/c++ are detected as
@@ -122,24 +127,39 @@ Linker: $(ld --version | head -n 1)
 C Library: $(ldd --version | head -n 1)
 EOF
 
-if bare_metal_bot_p "$2"; then
-    # TK1s have CPU hot-plug, so ninja might detect smaller number of cores
-    # available for parallelism.  Explicitly set "default" parallelism.
-    cat > /usr/local/bin/ninja <<EOF
+case "$2" in
+    linaro-tk1-*)
+	# TK1s have CPU hot-plug, so ninja might detect smaller number of cores
+	# available for parallelism.  Explicitly set "default" parallelism.
+	cat > /usr/local/bin/ninja <<EOF
 #!/bin/sh
 exec /usr/bin/ninja -j$n_cores "\$@"
 EOF
-else
-    # Throttle ninja on system load, system memory and container memory limit.
-    # When running with "-l 2*N_CORES -m 50 -M 50" ninja will not start new jobs
-    # if system or container memory utilization is beyond 50% or when load is
-    # above double the core count.  Ninja will also stall up to 5 seconds (-D 5000)
-    # before starting a new job to avoid rapid increase of resource usage.
-    cat > /usr/local/bin/ninja <<EOF
+	;;
+    *)
+	# Throttle ninja on system load, system memory and container memory
+	# limits.
+	case "$1" in
+	    lab.llvm.org:9994)
+		# Run silent bots with single-threaded ninja when average load
+		# is beyond twice the number of cores.
+		avg_load_opt="-l $((2*$n_cores))"
+		;;
+	    *)
+		avg_load_opt=""
+		;;
+	esac
+	# Make ninja run single-threaded if system or container memory
+	# utilization is beyond 50% (-m 50 -M 50).
+	# Make ninja stall for up to 5 seconds (-D 5000) before starting
+	# a new job when usage decreases under threshold (to avoid rapid
+	# increase of resource usage from N_CORES-1 new processes).
+	cat > /usr/local/bin/ninja <<EOF
 #!/bin/sh
-exec /usr/local/bin/ninja.bin -j$n_cores -l $((2*$n_cores)) -m 50 -M 50 -D 5000 "\$@"
+exec /usr/local/bin/ninja.bin -j$n_cores $avg_load_opt -m 50 -M 50 -D 5000 "\$@"
 EOF
-fi
+	;;
+esac
 chmod +x /usr/local/bin/ninja
 
 sudo -i -u buildslave buildslave restart ~buildslave/buildslave
diff --git a/tcwg-base/tcwg-llvmbot/start.sh b/tcwg-base/tcwg-llvmbot/start.sh
index 38ee277b..f337f967 100755
--- a/tcwg-base/tcwg-llvmbot/start.sh
+++ b/tcwg-base/tcwg-llvmbot/start.sh
@@ -25,6 +25,37 @@ else
     DOCKER="sudo docker"
 fi
 
+case "$slavename" in
+  linaro-armv8-01-aarch64-full) ;;
+  linaro-armv8-01-aarch64-global-isel) ;;
+  linaro-armv8-01-aarch64-libcxx) ;;
+  linaro-armv8-01-aarch64-libcxx-noeh) ;;
+  linaro-armv8-01-aarch64-lld) ;;
+  linaro-armv8-01-aarch64-quick) ;;
+  linaro-armv8-01-arm-full) ;;
+  linaro-armv8-01-arm-full-selfhost) ;;
+  linaro-armv8-01-arm-global-isel) ;;
+  linaro-armv8-01-arm-libcxx) ;;
+  linaro-armv8-01-arm-libcxx-noeh) ;;
+  linaro-armv8-01-arm-lld) ;;
+  linaro-armv8-01-arm-lnt) ;;
+  linaro-armv8-01-arm-quick) ;;
+  linaro-armv8-01-arm-selfhost-neon) ;;
+  linaro-tk1-*) ;;
+  *)
+    echo "WARNING: Unknown slavename $slavename"
+esac
+
+case "$slavename:$image" in
+  *-aarch64-*:*-arm64-*) ;;
+  *-arm-*:*-armhf-*) ;;
+  linaro-tk1-*:*-armhf-*) ;;
+  *)
+    echo "ERROR: $slavename should not run on $image."
+    echo "Make sure you're running an AArch64 bot on an arm64 image or an ARM bot on an armhf image."
+    exit 1
+esac
+
 case "$buildmaster" in
     "normal")
 	mastername="normal"
@@ -47,8 +78,17 @@ case "$mastername" in
     *) cpu_shares=1000 ;;
 esac
 
-# Use 64G out of 128G.
-memlimit="64"
+memlimit=$(free -m | awk '/^Mem/ { print $2 }')
+case "$slavename" in
+    linaro-tk1-*)
+	# Use at most 90% of RAM on TK1s
+	memlimit=$(($memlimit * 9 / 10))m
+	;;
+    *)
+	# Use at most half of all available RAM.
+	memlimit=$(($memlimit / 2))m
+	;;
+esac
 
 case "$slavename" in
     *-lld) pids_limit="15000" ;;
@@ -60,4 +100,4 @@ esac
 # seccomp:unconfined is required to disable ASLR for sanitizer tests.
 caps="--cap-add=IPC_LOCK --cap-add=SYS_PTRACE --security-opt seccomp:unconfined"
 
-$DOCKER run --name=$mastername-$slavename --hostname=$mastername-$slavename --restart=unless-stopped -dt -p 22 --cpu-shares=$cpu_shares --memory=${memlimit}G --pids-limit=$pids_limit $caps "$image" "$masterurl" "$slavename" "$password"
+$DOCKER run --name=$mastername-$slavename --hostname=$mastername-$slavename --restart=unless-stopped -dt -p 22 --cpu-shares=$cpu_shares --memory=$memlimit --pids-limit=$pids_limit $caps "$image" "$masterurl" "$slavename" "$password"