#!/bin/bash

set -e

rundir=`pwd -P`
mydir="$(dirname $0)"
cd "${mydir}"
mydir="$(pwd)"
cd ${rundir}

usage()
{
    cat <<EOF
Usage: $0 --bench bench_names --config config --nodename nodename [OPT]

	Options:
	--ccprefix prefix
	--cflags cflags
	--clean_older_than days
	--extension {O2|O2g|O3|O3g|etc}
	--hw_tag {tk1_32|tx1_64|tx1_32|etc}
	--forceinstall true/false
	--ignore_errors true/false
	--input_size {test|ref}
	--iterations N
	--nosudo
	--resultsdest host:/path/
	--run_profile {serial|parallel|parallel_NxN}
	--sysroot sysroot_url
	--toolchain {gnu|llvm}
	--support_fortran
	--verbose true/false
EOF
    exit 1
}


sleep_if_nb_jobs_greater_than()
{
  local max_jobs=$1
  local -a joblist
  mapfile -t joblist < <(jobs -p)
  while (( ${#joblist[*]} >= $max_jobs ))
  do
      sleep 60
      mapfile -t joblist < <(jobs -p)
      echo "waiting for a free slot (max $max_jobs)"
      jobs
  done
}

benches=
ccprefix=
cflags=
ldflags="default"
clean_older_than=
config=
extension="O2g"
hw_tag=
forceinstall=false
ignore_errors=false
input_size="test"
iterations="1"
nodename=
nosudo=false
resultsdest=
run_profile="parallel"
sysroot=
toolchain=
support_fortran=
verbose=false
max_build_jobs=$(nproc --all)
max_run_jobs=$(nproc --all)
max_install_jobs=5

while [ $# -ge 1 ]
do
    case $1 in
	--bench)
	    benches="${benches} $2"
	    shift
	    ;;
	--ccprefix)
	    ccprefix="$2"
	    shift
	    ;;
	--cflags)
	    cflags="$2"
	    shift
	    ;;
	--ldflags)
	    ldflags="$2"
	    shift
	    ;;
	--clean_older_than)
	    clean_older_than="$2"
	    shift
	    ;;
	--config)
	    config="$2"
	    shift
	    ;;
	--extension)
	    extension="$2"
	    shift
	    ;;
	--hw_tag)
	    hw_tag="$2"
	    shift
	    ;;
	--forceinstall)
	    forceinstall=$2
	    shift
	    ;;
	--ignore_errors)
	    ignore_errors="$2"
	    shift
	    ;;
	--input_size)
	    input_size="$2"
	    shift
	    ;;
	--iterations)
	    iterations="$2"
	    shift
	    ;;
	--nodename)
	    nodename="$2"
	    shift
	    ;;
	--nosudo)
	    nosudo=true
	    ;;
	--resultsdest)
	    resultsdest="$2"
	    shift
	    ;;
	--run_profile)
	    run_profile="$2"
	    shift
	    ;;
	--sysroot)
	    sysroot="$2"
	    shift
	    ;;
	--toolchain)
	    toolchain="$2"
	    shift
	    ;;
	--support_fortran)
	    support_fortran="--support_fortran"
	    ;;
	--verbose)
	    verbose="$2"
	    shift
	    ;;
	*)
	    echo "ERROR: Wrong argument: $1"
	    usage
	    ;;
    esac
    shift
done

if $verbose; then
    verbose="set -x"
else
    verbose="set +x"
fi

$verbose

# Sanity checks
[ x"${benches}" = x ] && usage
[ x"${config}" = x ] && usage
[ x"${hw_tag}" = x ] && usage
[ x"${resultsdest}" = x ] && usage
[ x"${nodename}" = x ] && usage

# determine spec_version : either cpu2006 or cpu2017
for bch in ${benches}; do
  case ${bch} in
    4[0-9][0-9].*|99[89].specrand)
	bench_version=cpu2006 ;;
    5[0-9][0-9].*|99[6789].specrand_[fi]r)
	bench_version=cpu2017 ;;
    6[0-9][0-9].*|99[6789].specrand_[fi]s)
	bench_version=cpu2017 ;;
    *)
       echo ERROR: Option --bench $bch not recognized
       exit 1
  esac
  if [ -z "$spec_version" ]; then
     spec_version=$bench_version
  elif [ "$spec_version" != "$bench_version" ]; then
   echo ERROR: Option --bench cannot mix spec2006 and spec2017 tests within a single run.
   exit 1
  fi
done

case ${toolchain} in
    llvm|gnu) ;;
    *) usage ;;
esac
case ${forceinstall} in
    true|false) ;;
    *)
	echo ERROR: Wrong value for --forceinstall: $forceinstall
	usage ;;
esac

if $ignore_errors; then
    ignore_errors_opt="--ignore_errors"
else
    ignore_errors_opt=""
fi

declare -A bmks

# Build list of benchmarks to run per cpu
case "$hw_tag:$run_profile" in
    qc_64:serial)
	# QualComm : cpu 0-3 runs (1.8 GHz), cpu 4-6 runs (2.5GHz), cpu 7 (3GHz)
	# For now, we are using cpu 7 only for benchmarking.
	bmks[7]="$benches"
	spec_cpus="0"
	;;
    fx_64:serial)
	# FX700 : 4 NUMA (0-11; 12-23; 24-35; 36-47)
	# - Run perf process on cpu 0
	# - Run benchmarks on NUMA node 3 (cpu : 36-47).
	#   This cpu range is set in spec2xxx-config var $bind_cpus, based on the data below.
	bmks[36]="$benches"
	spec_cpus="0"
	;;
    *:serial)
	bmks[1]="$benches"
	spec_cpus="0"
	;;
    *:parallel|*:parallel_*)
        if [[ $run_profile =~ parallel_.* ]]; then
	   max_build_jobs=$(echo ${run_profile#parallel_} | cut -dx -f1)
	   max_run_jobs=$(echo ${run_profile#parallel_} | cut -dx -f2)
	   [ $max_build_jobs == 0 ] && max_build_jobs=$(nproc --all)
	   [ $max_run_jobs == 0 ] && max_build_jobs=$(nproc --all)
	fi

	read -a benches_arr < <(echo $benches)
	n_cores=$(nproc --all)
	spec_cpus="0-$(($n_cores - 1))"

	cpu=0
	while [ ${#benches_arr[@]} -gt 0 ]; do
	    # Number of benchmarks to run serially per core
	    N=$(((${#benches_arr[@]} + $n_cores - 1) / $n_cores))
	    bmks[$cpu]="${benches_arr[*]:0:$N}"
	    benches_arr=("${benches_arr[@]:$N}")
	    n_cores=$((n_cores-1))
	    cpu=$(($cpu+1))
	done
	;;
esac

cpus=("${!bmks[@]}")

installdir=$HOME/$nodename
mkdir -p ${installdir}

# Clean old results if requested (to make some free space)
for spec in $installdir/$spec_version-*; do
    if [ x"$clean_older_than" != x"" ] && [ -d $spec/result ]; then
	find $spec/result -depth "(" -type f -o -type l ")" -mtime +$clean_older_than -exec rm {} \;
	find $spec/result -depth -empty -delete # removes empty dirs
    fi

    if ! [ -d $spec/result ]; then
	# Remove SPEC installs with no results (i.e., stale unused versions).
	needed=false
	for cpu in "${cpus[@]}"; do
	    if [ x"$spec" = x"$installdir/$spec_version-$hw_tag-$cpu" ]; then
		needed=true
	    fi
	done
	if ! $needed; then
	    rm -rf $spec
	fi
    else
	# Cleanup build, run and exe directories.  Using "runcpu -a scrub"
	# does not work because it needs a config.  Below method is from
	# official SPEC documentation.
	(
	    cd "$spec"
	    set +f
	    rm -rf benchspec/C*/*/run
	    rm -rf benchspec/C*/*/build
	    rm -rf benchspec/C*/*/exe
	)
    fi
done

case ${spec_version} in
    cpu2006) sha1=$(git ls-remote ssh://dev-private-git.linaro.org/restricted-benchmarks/CPU2006 refs/heads/master | cut -f 1) ;;
    cpu2017) sha1=$(git ls-remote ssh://dev-private-git.linaro.org/restricted-benchmarks/CPU2017 refs/heads/master | cut -f 1) ;;
esac

pids=()
for cpu in "${cpus[@]}"; do

    # A fresh machine can install many copies of SPECS.
    # The dev-private-git.linaro.org server limits the number
    # of simultaneous accesses. Thus, we are limiting the
    # number of parallel installation.
    sleep_if_nb_jobs_greater_than $max_install_jobs

    delete=$forceinstall
    if [ -d $installdir/$spec_version-$hw_tag-$cpu ] \
	   && [ x"$(git -C $installdir/$spec_version-$hw_tag-$cpu rev-parse HEAD)" \
		 != x"$sha1" ]; then
	delete=true
    fi
    if $delete; then
	rm -rf ${installdir}/$spec_version-$hw_tag-$cpu
    fi
    if [ ! -d ${installdir}/$spec_version-$hw_tag-$cpu ]; then


	(
	    ${mydir}/spec2xxx-install -v \
		    -d ${installdir}/$spec_version-$hw_tag-$cpu \
		    -s $spec_version
	    # It can happen that on a fresh machine we try to install
	    # 20 copies of SPEC, and, say, 5 of those installs fails due to
	    # git server error.  This leaves 15 good installs, which
	    # the next build deletes as they are empty.  And we stay in
	    # this install/delete loop until all 20 installs manage to
	    # complete in the same run.
	    # Create a fresh file in result/ directory to avoid the next
	    # build deleting this installation as old/empty (see logic for
	    # $clean_older_than above).
	    mkdir -p ${installdir}/$spec_version-$hw_tag-$cpu/result
	    touch ${installdir}/$spec_version-$hw_tag-$cpu/result/installed
	) &
	pids+=($!)
	# Avoid bashing on the git server too much
	sleep 20
    fi
done

status=0

echo "Note: Waiting for $spec_version-$hw_tag-* to install"
for pid in "${pids[@]}"; do
    result=0; wait $pid || result=$?
    if [ $result != 0 ]; then
	status=1
    fi
done

if [ $status != 0 ]; then
    echo "ERROR: Failed installing $spec_version-$hw_tag-*"
    exit $status
fi

if [ x"$sysroot" != x"" ]; then
    sysroot_remote=${sysroot%:*}
    sysroot_dir=${sysroot##*:}
    sysroot_host="$(echo $sysroot_remote | cut -d: -f 1)"
    sysroot_port="$(echo $sysroot_remote | cut -s -d: -f 2)"

    rsync -a --del -e "ssh ${sysroot_port:+-p$sysroot_port}" \
	  "$sysroot_host:$sysroot_dir/" "$HOME/sysroot/"
    sysroot_opt="--sysroot $HOME/sysroot"
else
    sysroot_opt=""
fi

$nosudo || sudo ${mydir}/prepare-board.sh --action start_docker \
		--hw_tag "$hw_tag" --verbose

case "$hw_tag" in
    apm_*)
	# APMs run at 2.4GHz, but they have no cpufreq driver, so we
	# can't query core frequency.
	perf_cycles_period="240000000"
	;;
    tk1_*|tx1_*)
	# We sample at 10HZ, but, due to perf_hack workaround we need
	# double that.
	perf_cycles_period="20HZ"
	;;
    sq_*)
	# SynQuacers run at 1GHz, but they have no cpufreq driver, so we
	# can't query core frequency.
	perf_cycles_period="100000000"
	;;
    fx_*)
	# FX700 runs at 1.8GHz
	perf_cycles_period="180000000"
	;;
    qc_*)
	# QualComm : cpu 0-3 runs (1.8 GHz), cpu 4-6 runs (2.5GHz), cpu 7 (3GHz)
	# For now, we are using cpu 7 only for benchmarking.
	# For perf_cycles_period, we align to cpu 7 frequency set in prepare-board.sh
	perf_cycles_period="251520000"
	;;
esac

# Configure the benchmarks
for cpu in "${cpus[@]}"
do
    ${mydir}/spec2xxx-config --config $config --ccprefix $ccprefix \
	    --ccflags "$cflags" --ldflags "$ldflags" --bind $cpu \
	    --iterations "$iterations" --run_profile $run_profile-$hw_tag \
	    --perf-bin "/usr/lib/linux-tools/$hw_tag/perf" --profiler perf \
	    -e cycles/period=$perf_cycles_period/u \
	    --perf-buildid-dir local --save-temps $sysroot_opt \
	    --toolchain $toolchain $support_fortran \
	    -v ${installdir}/$spec_version-$hw_tag-$cpu
done

# Build the benchmarks
pids=()
for cpu in "${cpus[@]}"
do
    sleep_if_nb_jobs_greater_than $max_build_jobs

    # If nothing to run on this cpu, continue
    [ x"${bmks[${cpu}]}" = x ] && continue
    (
	cd ${installdir}/$spec_version-$hw_tag-$cpu
	set +x
	. shrc
	$verbose

        case ${spec_version} in
          cpu2006) runspec -c $config -e $extension -a build ${ignore_errors_opt} ${bmks[$cpu]} & ;;
          cpu2017) runcpu -c $config -L $extension -a build ${ignore_errors_opt} ${bmks[$cpu]} & ;;
        esac
	
	result=0 && wait $! || result=$?
	${mydir}/spec2xxx-result -v --dest $resultsdest --tag $config --part .build.$cpu --extension $extension ./
	exit $result
    ) &
    pids+=($!)
done

echo "Note: Waiting for builds to finish"
for pid in "${pids[@]}"; do
    result=0; wait $pid || result=$?
    if [ $result != 0 ]; then
	status=1
    fi
done

# Close ssh connection used for building
for cpu in "${cpus[@]}"; do
    cleanup=${installdir}/$spec_version-$hw_tag-$cpu/bin/ssh-$config-finish_build
    if [ -e $cleanup ]; then
	$cleanup
    fi
done

if $ignore_errors; then
    # Remove failed-to-build benchmarks from run lists.
    # Otherwise we will try to build them again.
    for cpu in "${cpus[@]}"; do
	failed_csv="${installdir}/$spec_version-$hw_tag-$cpu/result/failed.$config.csv.build.$cpu"
	if [ ! -f "$failed_csv" ]; then
	    continue
	fi
	read -a build_bmks < <(echo "${bmks[$cpu]}")
	bmks[$cpu]=""
	for bmk in "${build_bmks[@]}"; do
	    if ! grep -q "^$bmk," "$failed_csv"; then
		bmks[$cpu]="${bmks[$cpu]} $bmk"
	    fi
	done
    done
elif [ $status != 0 ]; then
    # Skip running benchmarks, and execute prepare-board.sh cleanups.
    cpus=()
fi

# Run the benchmarks
pids=()
for cpu in "${cpus[@]}"; do

    sleep_if_nb_jobs_greater_than $max_run_jobs

    # If nothing to run on this cpu, continue
    [ x"${bmks[${cpu}]}" = x ] && continue
    (
	cd ${installdir}/$spec_version-$hw_tag-$cpu
	set +x
	. shrc
	$verbose

        case ${spec_version} in
          cpu2006) taskset -c $spec_cpus runspec -c $config -e $extension -a run -i $input_size ${ignore_errors_opt} ${bmks[$cpu]} & ;;
          cpu2017) taskset -c $spec_cpus runcpu -c $config -L $extension -a run -i $input_size ${ignore_errors_opt} ${bmks[$cpu]} & ;;
        esac

	result=0 && wait $! || result=$?
	${mydir}/spec2xxx-result -v --dest ${resultsdest} --tag $config --part .run.$cpu --extension $extension ./
	exit $result
    ) &
    pids+=($!)
done

echo "Note: Waiting for runs to finish"
for pid in "${pids[@]}"; do
    result=0; wait $pid || result=$?
    if [ $result != 0 ]; then
	status=1
    fi
done

echo "Note: Runs finished"
$nosudo || sudo ${mydir}/prepare-board.sh --action stop_docker --verbose &
result=0 && wait $! || result=$?
if [ $result != 0 ]; then
    echo "Warning: prepare-board.sh did not finish cleanly"
fi

if $ignore_errors && [ $status != 0 ]; then
    echo "Ignoring build and/or run failures with status $status and exiting with 0"
    exit 0
fi

exit $status