#!/bin/bash # Improve debug logs PRGNAME=`basename $0` PS4='+ $PRGNAME: ${FUNCNAME+"$FUNCNAME : "}$LINENO: ' set -e arch="native" begin_session=false shared_dir="" board_exp="" finish_session=false sysroot="" ssh_master=false target_ssh_opts="" host_ssh_opts="" profile="tcwg-test" multilib_path="lib" uname="" while getopts "a:bd:e:fh:l:mo:p:P:qu:v" OPTION; do case $OPTION in a) arch=$OPTARG ;; b) begin_session=true ;; d) shared_dir=$OPTARG ;; e) board_exp="$OPTARG" ;; f) finish_session=true ;; h) multilib_path="$OPTARG" ;; l) sysroot=$OPTARG ;; m) ssh_master=true ;; o) target_ssh_opts="$OPTARG" ;; p) host_ssh_opts="$OPTARG" ;; P) profile="$OPTARG" ;; q) exec > /dev/null ;; u) uname="$OPTARG" ;; v) set -x ;; esac done shift $((OPTIND-1)) target="${1%:*}" port="$(echo $1 | grep ":" | sed -e "s/.*://")" triplet_to_deb_arch() { set -e case "$1" in aarch64-*linux-gnu) echo arm64 ;; arm*-*linux-gnueabi*) echo armhf ;; i686-*linux-gnu) echo i386 ;; x86_64-*linux-gnu) echo amd64 ;; *) return 1 ;; esac } triplet_to_deb_dist() { set -e case "$arch" in aarch64-*linux-gnu) echo trusty ;; arm*-*linux-gnueabi*) echo trusty ;; i686-*linux-gnu) echo trusty ;; x86_64-*linux-gnu) echo trusty ;; *) return 1 ;; esac } if [ -z "$target" ]; then echo ERROR: no target specified exit 1 fi if [ -z "$port" ]; then echo "ERROR: no custom [ssh] port specified" exit 1 fi use_qemu=false if ! triplet_to_deb_arch "$arch" >/dev/null 2>&1; then use_qemu=true arch="native" fi # Use '|| true' to avoid early exit if $target does not answer. cpu="$(ssh $target uname -m)" || true case "$cpu" in aarch64) native_arch=aarch64-linux-gnu ;; armv7l) native_arch=arm-linux-gnueabihf ;; armv7*) native_arch=arm-linux-gnueabi ;; i686) native_arch=i686-linux-gnu ;; x86_64) native_arch=x86_64-linux-gnu ;; "") echo "ERROR: target $target returned no cpu type." exit 1 ;; *) echo "ERROR: unrecognized native target $cpu" exit 1 ;; esac if [ x"$arch" = x"native" ]; then arch="$native_arch" fi deb_arch="$(triplet_to_deb_arch $arch)" case "$cpu:$deb_arch" in "x86_64:amd64"|"x86_64:i386") ;; "x86_64:"*) use_qemu=true arch=$native_arch ;; esac deb_arch="$(triplet_to_deb_arch $arch)" deb_dist="$(triplet_to_deb_dist $arch)" if [ "x$board_exp" != "x" ] ; then lava_json="$(grep "^set_board_info lava_json " $board_exp | sed -e "s/^set_board_info lava_json //")" if [ "x$lava_json" != "x" ] && $begin_session; then job_id="$(lava-tool submit-job http://maxim-kuvyrkov@validation.linaro.org/RPC2/ "$lava_json" | sed -e "s/submitted as job id: //")" while sleep 60; do if lava-tool job-output -o - http://maxim-kuvyrkov@validation.linaro.org/RPC2/ $job_id | grep "^Hacking session active" >/dev/null; then lava_ssh_opts="$(lava-tool job-output -o - http://maxim-kuvyrkov@validation.linaro.org/RPC2/ $job_id | grep -a "^Please connect to" | sed -e "s/.* ssh \(.*\) \([^ ]*\) (.*$/\1/")" lava_target="$(lava-tool job-output -o - http://maxim-kuvyrkov@validation.linaro.org/RPC2/ $job_id | grep -a "^Please connect to" | sed -e "s/.* ssh \(.*\) \([^ ]*\) (.*$/\2/")" sed -i -e "s/^set_board_info hostname .*/set_board_info hostname $lava_target/" "$board_exp" echo "set_board_info lava_ssh_opts \"$lava_ssh_opts\"" >> "$board_exp" echo "set_board_info lava_job_id $job_id" >> "$board_exp" target="$lava_target" target_ssh_opts="$target_ssh_opts $lava_ssh_opts" break fi done fi fi orig_target_ssh_opts="$target_ssh_opts" target_ssh_opts="$target_ssh_opts -o ControlMaster=auto -o ControlPersist=1m -o ControlPath=/tmp/ssh-$profile-$port-%u-%r@%h:%p" schroot_id=$profile-$deb_arch-$deb_dist schroot="ssh $target_ssh_opts $target schroot -r -c session:$profile-$port -d / -u root --" rsh_opts="$target_ssh_opts -o Port=$port -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" rsh="ssh $rsh_opts" user="$(ssh $target_ssh_opts $target echo \$USER)" home="$(ssh $target_ssh_opts $target pwd)" if $begin_session; then ssh $target_ssh_opts $target schroot -b -c chroot:$schroot_id -n $profile-$port -d / # Start ssh server on custom port $schroot sed -i -e "\"s/^Port 22/Port $port/\"" /etc/ssh/sshd_config # Run as root $schroot sed -i -e "\"s/^UsePrivilegeSeparation yes/UsePrivilegeSeparation no/\"" /etc/ssh/sshd_config # Increase number of incoming connections from 10 to 256 $schroot sed -i -e "\"/.*MaxStartups.*/d\"" -e "\"/.*MaxSesssions.*/d\"" /etc/ssh/sshd_config $schroot bash -c "\"echo \\\"MaxStartups 256\\\" >> /etc/ssh/sshd_config\"" $schroot bash -c "\"echo \\\"MaxSessions 256\\\" >> /etc/ssh/sshd_config\"" $schroot /etc/init.d/ssh start # Crouton needs firewall rule. $schroot iptables -I INPUT -p tcp --dport $port -j ACCEPT >dev/null 2>&1 || true $schroot mkdir -p /root/.ssh ssh $target_ssh_opts $target cat .ssh/authorized_keys | $schroot bash -c "'cat > /root/.ssh/authorized_keys'" $schroot chmod 0600 /root/.ssh/authorized_keys $rsh root@$target rsync -a /root/ $home/ $rsh root@$target chown -R $user $home/ $rsh root@$target "echo 1 > /dont_keep_session" $rsh root@$target chmod 0666 /dont_keep_session echo $target:$port started schroot: $rsh $target fi if ! [ -z "$shared_dir" ]; then # Generate a one-time key to allow ssh back to host. ssh-keygen -t rsa -N "" -C "test-schroot.$$" -f ~/.ssh/id_rsa-test-schroot.$$ echo >> ~/.ssh/authorized_keys cat ~/.ssh/id_rsa-test-schroot.$$.pub >> ~/.ssh/authorized_keys scp $rsh_opts ~/.ssh/id_rsa-test-schroot.$$ $target:.ssh/ $rsh root@$target mkdir -p "$shared_dir" $rsh root@$target chown -R $user "$shared_dir" $rsh root@$target rm -f /etc/mtab $rsh root@$target ln -s /proc/mounts /etc/mtab tmp_ssh_port="$(($port-10000))" host_ssh_port="$(grep "^Port" /etc/ssh/sshd_config | sed -e "s/^Port //")" test -z "$host_ssh_port" && host_ssh_port="22" # Establish port forwarding $rsh -fN -S none -R $tmp_ssh_port:127.0.0.1:$host_ssh_port $target # Recent versions of sshfs fail if ssh_command has more than a single # white spaces between options or ends with a space; filter ssh_command. ssh_command="$(echo "ssh -o Port=$tmp_ssh_port -o IdentityFile=$home/.ssh/id_rsa-test-schroot.$$ -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $host_ssh_opts" | sed -e "s/ \+/ /g" -e "s/ \$//")" try="0" while [ "$try" -lt "3" ]; do $rsh $target sshfs -C -o ssh_command="\"$ssh_command\"" "$USER@127.0.0.1:$shared_dir" "$shared_dir" | true if [ x"${PIPESTATUS[0]}" != x"0" ]; then try=$(($try + 1)) sleep 1 continue fi break done # Remove temporary key and delete extra empty lines at the end of file. sed -i -e "/.*test-schroot\.$$\$/d" -e '/^$/N;/\n$/D' ~/.ssh/authorized_keys rm ~/.ssh/id_rsa-test-schroot.$$* if [ "$try" != "3" ]; then echo "$target:$port shared directory $shared_dir: SUCCESS" else echo "$target:$port shared directory $shared_dir: FAIL" fi fi if ! [ -z "$sysroot" ]; then rsync -az -e "$rsh" $sysroot/ root@$target:/sysroot/ if [ -e $sysroot/lib64/ld-linux-aarch64.so.1 ]; then # Our aarch64 sysroot has everything in /lib64, but executables # still expect to find dynamic linker under /lib/ld-linux-aarch64.so.1 if [ -h $sysroot/lib ]; then $rsh root@$target "rm /sysroot/lib" fi if [ -h $sysroot/lib ] \ || ! [ -e $sysroot/lib/ld-linux-aarch64.so.1 ]; then $rsh root@$target "mkdir -p /sysroot/lib/" $rsh root@$target "cd /sysroot/lib; ln -s ../lib64/ld-linux-aarch64.so.1 ." fi fi if ! $use_qemu; then # Make sure that sysroot libraries are searched before any other. $rsh root@$target "cat > /etc/ld.so.conf.new" <> /etc/ld.so.conf.new" $rsh root@$target "mv /etc/ld.so.conf.new /etc/ld.so.conf && rsync -a --exclude=/sysroot /sysroot/ / && ldconfig" else # Remove /etc/ld.so.cache to workaround QEMU problem for targets with # different endianness (i.e., /etc/ld.so.cache is endian-dependent). $rsh root@$target "rm /etc/ld.so.cache" # Cleanup runaway QEMU processes that ran for more than 2 minutes. # Note the "-S none" option -- ssh does not always detach from process # when multiplexing is used. I think this is a bug in ssh. # We calculate delay in this fashion to avoid multi-thread tests # getting through a minute of usertime in 60/#_of_cpus seconds. delay=$((60 / $($rsh $target getconf _NPROCESSORS_ONLN))) $rsh -f -S none $target bash -c "\"while sleep $delay; do ps uxf | sed -e \\\"s/ \+/ /g\\\" | cut -d\\\" \\\" -f 2,10- | grep \\\"^[0-9]\+ [0-9]*2:[0-9]\+ ._ qemu-\\\" | cut -d\\\" \\\" -f 1 | xargs -r kill -9; done\"" fi echo $target:$port installed sysroot $sysroot fi if [ x"$uname" != x"" ]; then old_uname="$($rsh root@$target "uname -m")" $rsh root@$target "mv /bin/uname /bin/uname.real" $rsh root@$target "cat > /bin/uname" </dev/null 2>&1 || true ssh $target_ssh_opts $target schroot -f -e -c session:$profile-$port | true if [ x"${PIPESTATUS[0]}" != x"0" ]; then # tcwgbuildXX machines have a kernel problem that a bind mount will be # forever busy if it had an sshfs under it. Seems like fuse is not # cleaning up somethings. The workaround is to lazy unmount the bind. $schroot umount -l / ssh $target_ssh_opts $target schroot -f -e -c session:$profile-$port fi echo $target:$port finished session fi if [ "x$board_exp" != "x" ] ; then lava_job_id="$(grep "^set_board_info lava_job_id " $board_exp | sed -e "s/^set_board_info lava_job_id //")" if [ "x$lava_job_id" != "x" ] && $finish_session; then lava-tool cancel-job http://maxim-kuvyrkov@validation.linaro.org/RPC2/ $lava_job_id fi fi