Scripts to isolate CPUs/start task on isolated CPUS

 README
 isolate-cpu.sh  Isolates desired CPUs.
 isolate-task.sh Uses isolate-cpu.sh to isolate CPUs before running
                 the desired task. It also provides the possibility
                 to trace kernel disturbance on the isolated CPUs.

Change-Id: I7312b695f821c845e14c51d4f1ba4d5731c52fa6
Signed-off-by: Ravineet Singh <ravineet.singh@linaro.org>

3 files changed, 470 insertions, 0 deletions

diff --git a/README b/README
new file mode 100644
index 0000000..cb02056
--- /dev/null
+++ b/README
@@ -0,0 +1,22 @@
+Helper scripts to check and set CPU isolation and execution of application in
+isolated CPU(s)
+
+Files:
+isolate-cpu.sh       isolates desired CPUs
+isolate-task.sh      uses isolate-cpu.sh to isolate CPUs before running
+                     the desired task. It also provides the possibility
+                     to trace kernel disturbance on the isolated CPUs.
+
+isolate-cpu.sh checks the kernel configuration and the kernel cmdline to
+ determine if one or several CPUs are isolated, i.e. it checks for;
+ CONFIG_NO_HZ_FULL_ALL=y", and CONFIG_RCU_NOCB_CPU_ALL=y" in the kernel config
+ and rcu_nocbs,nohz_full in the kernel cmdline.
+ If the desired CPU(s) are not inte the above configuration, it warns but continues
+ to isolate the CPU(s) as much as possibe. The isolation is accomplished by
+ - Redirecting all IRQ's away from desired isolated cores
+ - Using cset (cpuset) to move all running processes and kernel threads
+   away from desired isolated cores
+
+isolate-task.sh uses isolate-cpu.sh to isolate desired CPU(s). In addition
+ it starts the supplied application on isolated CPU(s) and optionally traces
+ the isolated CPU(s) for kernel interaction.
diff --git a/isolate-cpu.sh b/isolate-cpu.sh
new file mode 100755
index 0000000..1de02ae
--- /dev/null
+++ b/isolate-cpu.sh
@@ -0,0 +1,266 @@
+#!/bin/bash
+#
+# This script isolates desired CPUS, i.e.
+# - Checks kernel cmdline and kernel config to determine
+#   if the environment is optimised for isolated task execution;
+# - Moves CPU interrupts, kernel threads, tasks etc. away from the targeted CPU.
+# Note: CPU 0 cannot be isolated, i.e minumum 2 CPU's are required.
+#
+#Todo:
+# - How to define/verify if a CPU is isolated?
+
+print_usage() {
+    echo "$0 [-h] [-a] [-c <cpu list>] [-l] [-r] [-d]"
+    echo
+    echo " Isolate CPU(s) from other tasks, kernel threads and IRQs"
+    echo " Args:"
+    echo "  -h       Print this message"
+    echo "  -a       Isolate all CPUs (except CPU 0)"
+    echo "  -c       List of CPUs to be isolated."
+    echo "  -l       Show isolation proporties"
+    echo "  -r       Reset isolation"
+    echo "  -d       Show debug printouts"
+    echo ""
+    echo " Examples:"
+    echo "  Isolate all CPU(s) (except 0) "
+    echo "  $0 -a"
+    echo
+    echo "  Isolate CPUs 1-3 "
+    echo "  $0 -c 1-3"
+    echo
+    echo "  Isolate CPUs 1 and 4 "
+    echo "  $0 -c 1,4 "
+}
+
+dlog() {
+    [ $DEBUG ] && echo "$*"
+}
+
+warn() {
+    printf "Warning:\n " >&2
+    echo "$*" >&2
+}
+
+die() {
+    printf "Error:\n " >&2
+    echo "$*" >&2
+    shield_reset
+    exit 1
+}
+
+get_cpu_array() {
+    [ $1 ] || die "$FUNCNAME internal error!"
+
+    local cpus=""
+    IFS=. a=$1; IFS=, a=$a;
+    for str in $a; do
+        if [[ $str == *[\-]* ]]; then
+            str=$(echo $str| sed 's/-/../g')
+            str=$(eval echo {$str})
+        fi
+
+        if [ "$cpus" != "" ]; then
+            cpus="$cpus $str"
+        else
+            cpus=$str
+        fi
+    done
+
+    echo $cpus
+}
+
+##
+# Check kernel config and kernel cmfline for rcu callbacs and no_hz
+# *Note* isolcpu= kernel cmdline option isolates CPUs from SMP balancing
+#        If needed, this can be done via cpusets/user/cpuset.sched_load_balance
+##
+check_kernel_config() {
+
+    eval $(cat /proc/cmdline | grep -o 'nohz_full=[^ ]*')
+
+    local configs="/proc/config.gz /boot/config-$(uname -r) /boot/config "
+    dlog "Looking for Kernel configs; $configs "
+    for config in $configs; do
+        if [ -e $config ]; then
+            dlog "Kenel configuration found:$config"
+            break
+        fi
+    done
+
+    local all_except_0="1-$(($(getconf _NPROCESSORS_ONLN) - 1))"
+    if [ $config ]; then
+        nohz_full=$(zgrep "CONFIG_NO_HZ_FULL_ALL=y" $config  2>/dev/null) && nohz_full=$all_except_0
+        rcu_nocbs=$(zgrep "CONFIG_RCU_NOCB_CPU_ALL=y" $config  2>/dev/null) && rcu_nocbs=$all_except_0
+
+    else
+        warn "Kernel config not found, only checking /proc/cmdline for isolation features."
+    fi
+
+    #rcu_nocbs and nohz_full kernel config superseeds cmdline option
+    if ! [ "$rcu_nocbs" ]; then
+        eval $(cat /proc/cmdline | grep -o 'rcu_nocbs=[^ ]*')
+    fi
+
+    if ! [ "$nohz_full" ]; then
+        eval $(cat /proc/cmdline | grep -o 'nohz_full=[^ ]*')
+    fi
+
+    if [ -z "$nohz_full" ]; then
+        warn "No CPU is isolated from kernel ticks, CONFIG_NO_HZ_FULL_ALL=y not set in kernel, nor nohz_full= set in kernel cmdline."
+    else
+        dlog "nohz_full:$nohz_full"
+        gbl_isolated_cpus=$nohz_full
+        export gbl_isolated_cpus
+    fi
+
+    [ -z "$rcu_nocbs" ] &&
+        warn "No CPU is set to block RCU threads, CONFIG_RCU_NOCB_CPU_ALL=y not set in kernel, nor rcu_nocbs= set in kernel cmdline."
+
+    [ "$nohz_full" = "$rcu_nocbs" ]  || warn "Configuration mismatch: nohz_full=$nohz_full," \
+                                             "rcu_nocbs=$rcu_nocbs. CPU isolation will not be efficient"
+
+    if [ $rcu_nocbs ]; then
+        dlog "rcu_nocbs=$rcu_nocbs"
+    fi
+
+    return 0
+}
+
+cpus_valid() {
+    local cpus="$1"
+    local isolated=$2
+    local iarray=$(get_cpu_array $isolated)
+    local carray=$(get_cpu_array $cpus)
+
+    for c in $carray; do
+        for i in $iarray; do
+            if [ $i = $c ]; then
+                yah=$i
+            fi
+        done
+        [ -z "$yah" ] && return 1
+    done
+
+    return 0
+}
+check_prequesties() {
+    dlog "Checking prequesties; user is root, kernel has cpuset support, and commads; cset, zgrep, getconf are available"
+    [ $UID -eq 0 ] || die "You need to be root!"
+    grep -q -s cpuset /proc/filesystems || die "Kernel does not support cpuset!"
+    which getconf > /dev/null 2>&1 || die "getconf command not found, please install getconf"
+    which cset > /dev/null 2>&1 || die "cset command not found, please install cpuset"
+    which zgrep > /dev/null 2>&1 || die "zgrep command not found, please install gzip"
+}
+
+shield_reset() {
+    cset shield -r >/dev/null 2>&1
+}
+
+shield_list() {
+    sets="/cpusets/*/"
+    for i in $sets ; do
+        if ! [ -e $i ]; then
+            continue
+        fi
+        printf "Domain %s cpus %s, running %d tasks\n" $(basename $i) $(cat $i/cpuset.cpus) $(cat $i/tasks | wc -l)
+    done
+}
+
+shield_cpus() {
+    local cpus="$1"
+
+    dlog "shielding CPU:s $cpus"
+    #Reset and create new shield
+    cset shield -r >/dev/null 2>&1
+    out=$(cset shield -c $cpus kthread=on 2>&1)  || die "cset failed; $out"
+
+    # Delay the annoying vmstat timer far away
+    sysctl vm.stat_interval=120 >/dev/null
+
+    # Shutdown nmi watchdog as it uses perf events
+    sysctl -w kernel.watchdog=0 >/dev/null
+
+    # Pin the writeback workqueue to CPU0
+    #Fixme, check that /sys/bus is mounted?
+    echo 1 > /sys/bus/workqueue/devices/writeback/cpumask
+
+    #Fixme disable load balanser? How do we schedule our threads?
+    #echo 0 > /cpusets/user/cpuset.sched_load_balance
+
+    #Fixme, for now just send all irqs to core 0
+    for affinity in /proc/irq/*/smp_affinity; do
+        dlog "redirecting $affinity"
+        echo 1 > $affinity 2>/dev/null || dlog "$affinity redirection failed."
+    done
+
+
+    #Fixme, not implemented.
+    if [ $false ];  then
+        for affinity in /proc/irq/*/smp_affinity; do
+            local old_mask=$(cat $affinity)
+            local new_mask=$((oldmask ^ cpus ))
+            echo $new_mask > $affinity
+        done
+    fi
+}
+
+isolate_cpus() {
+
+    local cpus="$1"
+
+    check_kernel_config
+
+    if [ "$gbl_isolated_cpus" ]; then
+        cpus_valid $cpus $gbl_isolated_cpus ||
+            warn "Selected CPU '$cpus' is not inside isolated cpus array:$gbl_isolated_cpus"
+    fi
+
+    dlog "Isolating CPUs $cpus"
+
+    shield_cpus $cpus
+    return 0
+}
+
+##
+# Script entry point
+##
+while getopts hdarlc: arguments
+do
+    case $arguments in
+        h)
+            print_usage
+            exit 0
+            ;;
+        d)
+            DEBUG=1
+            ;;
+        a)
+            ISOL_CPUS="1-$(($(getconf _NPROCESSORS_ONLN) - 1))"
+            ;;
+        r)
+            shield_reset
+            exit 0
+            ;;
+        l)
+            shield_list
+            exit 0
+            ;;
+        c)
+            [ "$ISOL_CPUS" ] || ISOL_CPUS=$OPTARG
+            ;;
+        *)
+            print_usage
+            exit 1
+            ;;
+    esac
+done
+#Remove all flags
+shift $((OPTIND-1))
+
+if ! [ $ISOL_CPUS ]; then
+    print_usage
+    exit 1
+fi
+
+check_prequesties
+isolate_cpus $ISOL_CPUS || die "isolate_cpus failed."
diff --git a/isolate-task.sh b/isolate-task.sh
new file mode 100755
index 0000000..c754db6
--- /dev/null
+++ b/isolate-task.sh
@@ -0,0 +1,182 @@
+#!/bin/bash
+#
+# This script isolates a task on desired CPUs.
+# If desired, kernel calls are traced to see kernel interference on isolated CPUs
+
+print_usage() {
+    echo "$0 [-h] [-t <time>] [-c <cpu list>] <application arg1, arg2, ...>"
+    echo
+    echo " Isolate CPU(s) from other tasks, kernel threads and IRQs"
+    echo " and run an application on isolated CPUs"
+    echo " Args:"
+    echo "  -h       Print this message"
+    echo "  -t       Trace kernel calls for 'time' seconds on chosen CPUs"
+    echo "  -c       List of CPUs to be isolated"
+    echo "  -d       Show debug printouts"
+    echo ""
+    echo "All CPU's, except CPU 0, are isolated unless '-c' specified"
+    echo " Examples:"
+    echo "  Isolate CPU 1,2 and run application in the same."
+    echo "  $0 -c 1,2 /some/path/application"
+    echo
+    echo "  Isolate all possible CPUs and run applicatipon"
+    echo "  $0 /path/application"
+    echo
+    echo "  Isolate all possible CPUs, run applicatipon and trace kervel calls for 10 sec"
+    echo "  $0 -t 10 /path/application"
+}
+
+dlog() {
+    [ $DEBUG ] && echo "$*"
+}
+
+warn() {
+    printf "Warning:\n " >&2
+    echo "$*" >&2
+}
+
+die() {
+    printf "Error:\n " >&2
+    echo "$*" >&2
+    exit 1
+}
+
+get_cpu_array() {
+    [ $1 ] || die "$FUNCNAME internal error!"
+
+    IFS=. a=$1; IFS=, a=$a;
+    for str in $a; do
+        if [[ $str == *[\-]* ]]; then
+            str=$(echo $str| sed 's/-/../g')
+            str=$(eval echo {$str})
+        fi
+
+        if [ $cpus ]; then
+            cpus="$cpus $str"
+        else
+            cpus=$str
+        fi
+    done
+
+    echo $cpus
+}
+
+enable_tracing() {
+    [ $1 -eq 0 ] && return
+
+    DIR=/sys/kernel/debug/tracing
+    echo > $DIR/trace
+    echo 0 > $DIR/tracing_on
+    echo 0 > $DIR/events/irq/enable
+    echo 1 > $DIR/events/sched/sched_switch/enable
+    echo 1 > $DIR/events/workqueue/workqueue_queue_work/enable
+    echo 1 > $DIR/events/workqueue/workqueue_execute_start/enable
+    echo 1 > $DIR/events/timer/hrtimer_expire_entry/enable
+    echo 1 > $DIR/events/timer/tick_stop/enable
+    echo nop > $DIR/current_tracer
+    #To see the complete call grapf, enable the line below
+    #echo function_graph > $DIR/current_tracer
+    echo 1 > $DIR/tracing_on
+
+    dlog "Tracing enabled."
+}
+disable_tracing() {
+    [ $1 -eq 0 ] && return
+
+    echo 0 > $DIR/tracing_on
+    dlog "Tracing disabled."
+}
+
+dump_trace_data(){
+    [ $1 -eq 0 ] && return
+
+    local base=/sys/kernel/debug/tracing/per_cpu
+
+    for cpu in $ISOL_CPUS_ARRAY; do
+        cat $base/cpu$cpu/trace > trace.cpu$cpu
+    done
+    dlog "Trace data dumped for CPUs: $ISOL_CPUS_ARRAY"
+}
+
+isolate_and_run() {
+    local tracing=$1
+    local cpus=$2
+    local app="$3"
+    local base=$(dirname $0)
+
+    $base/isolate-cpu.sh -c $cpus || die "$0 failed"
+
+    enable_tracing $tracing
+    dlog "Starting application: $app"
+    $app&
+    child=$!
+
+    echo $child >> /cpusets/user/tasks
+    if [ $? -ne 0 ]; then
+        kill -9 $child
+        die "Failed to isolate task..."
+    fi
+
+    local counter=$tracing
+    while [  "$counter" -gt "0" ]; do
+        kill -0 $child 2>/dev/null || break
+        sleep 1
+        counter=$(expr $counter - 1)
+    done
+
+    disable_tracing $tracing
+    dump_trace_data $tracing
+    return 0
+}
+
+check_prequesties() {
+    local base=$(dirname $0)
+    local tracing=$1
+
+    [ -e $base/isolate-cpu.sh ] || die "$base/isolate-cpu.sh not found!"
+    if [ $tracing -ne 0 ]; then
+        [ $UID -eq 0 ] || die "You need to be root!"
+        [ -e /sys/kernel/debug/tracing ] || die "Kernel lacks tracing funtionality (ftrace)"
+    fi
+}
+
+##
+# Script entry point
+##
+TRACING=0
+ISOL_CPUS="1-$(($(getconf _NPROCESSORS_ONLN) - 1))"
+while getopts hdc:t: arguments
+do
+    case $arguments in
+        h)
+            print_usage
+            exit 0
+            ;;
+        d)
+            DEBUG=1
+            ;;
+        c)
+            ISOL_CPUS=$OPTARG
+            ;;
+        t)
+            TRACING=$OPTARG
+            ;;
+        *)
+            print_usage
+            exit 1
+            ;;
+    esac
+done
+#Remove all flags
+shift $((OPTIND-1))
+
+if ! [ "$1" ]; then
+    print_usage
+    exit 1
+fi
+
+ISOL_CPUS_ARRAY=$(get_cpu_array $ISOL_CPUS)
+app=$*
+
+check_prequesties $TRACING
+isolate_and_run $TRACING $ISOL_CPUS "$app"