path: root/24h-stress-test.sh

#!/bin/sh

# Set default values.
tests="stress_ng stress_oom stress_network"
duration=$((60 * 60 * 6))
interface=wlan0
link="http://10.0.0.1/shares/RS500G/testdata/stress-network.img"
md5=e5c834fbdaa6bfd8eac5eb9404eefdd4
processor_number=$(grep processor /proc/cpuinfo | wc -l)

usage()
{
    echo "Usage: $0 [-h] [-t tests] [-d duration] [-i interface] [-l link] [-m md5]"
    echo
    echo "Options"
    echo "    -h, --help        Print this help message"
    echo "    -t, --test        Run only specified test from the following:"
    echo "                          stress_ng"
    echo "                          stress_oom"
    echo "                          stress_network"
    echo "    -d, --duration    Set test duration in seconds for each stress test"
    echo "    -i, --interface   Run network stress on the specified interface."
    echo "    -l, --link        Specify file link for download test."
    echo "    -m, --md5         Set md5 value of the file used for download test."
    echo
    echo "Examples"
    echo "    Run all stress tests with defualt settings:"
    echo "        $0"
    echo "    Set test duration for each test to 1 hour:"
    echo "        $0 -d 3600"
    echo "    Run network stress test on eth0:"
    echo "        $0 -t stress_network -i eth0"
    echo "    Run stress_ng and stress_oom:"
    echo "        $0 -t 'stress_ng stress_oom'"
    echo
}

# Parse command line arguments.
while [ $# -gt 0 ]
do
    case $1 in
        -t|--test)
            tests="$2"
            ;;
        -d|--duration)
            duration="$2"
            ;;
        -i|--interface)
            interface="$2"
            ;;
        -l|--link)
            link="$2"
            ;;
        -m|--md5)
            md5=$2
            ;;
        -h|--help)
            usage
            exit 1
            ;;
        *)
            echo "Unknown option $@"
            usage
            exit 1
            ;;
    esac
    shift 2
done

print_test_result()
{
    local test_case_id=$1
    local exit_code=$2
    if [ $exit_code -eq 0 ]; then
        echo "$test_case_id: pass"
    else
        echo "$test_case_id: fail"
    fi
}

backup_log()
{
    if [ -f $1 ]; then
        mv $1 $1-$(date +"%Y%m%d-%H%M%S")
    fi
}

stress_ng()
{
    workloads="cpu io fork switch vm pipe yield hdd cache sock fallocate flock affinity timer dentry urandom sem open sigq poll"
    workload_number=$(echo $workloads | wc -w)
    sub_duration=$(( $duration / $workload_number ))

    echo "CPU(s): $processor_number"
    echo "Workloads to run: $workloads"
    echo "Total stress_ng test duration: $duration seconds"
    echo "Test duration for each workload: $sub_duration seconds"

    count=1
    for i in $workloads
    do
        echo
        echo "[$count/$workload_number] Running $i workload..."
        if [ $i = "vm" ]; then
            # mmap 64M per vm process to avoid OOM, the default is 256M.
            stress-ng --$i $processor_number --vm-bytes 64m --timeout $sub_duration --metrics-brief --verbose 
        else
            stress-ng --$i $processor_number --timeout $sub_duration --metrics-brief --verbose 
        fi
        print_test_result stress-ng-$i $?
        count=$(( $count + 1 ))
    done
}

stress_oom()
{
    mem=$(free | grep Mem | awk '{print $2}')
    swap=$(free | grep Swap | awk '{print $2}')
    total_mem=$(( $mem + $swap ))
    vm_bytes=$(( $total_mem / $processor_number ))

    echo
    echo "CPU(s): $processor_number"
    echo "Total Memory: $total_mem"
    echo "Stress OOM test duration: $duration seconds"
    echo "About to run $processor_number stress-ng-vm instances"

    # Clear dmesg and save new output continuously to a log file.
    dmesg --clear
    dmesg --follow > stress_oom_kern.log 2>&1 &
    kernel_log=$!
    # Disable oom-killer on the log collecting process.
    echo -17 > /proc/$kernel_log/oom_adj

    # Run stress-ng-vm test to trigger oom-killer.
    # In stress-vm.c file, NO_MEM_RETRIES_MAX has been increased to 1000000000 for OOM stress test.
    echo "mmap ${vm_bytes}KB per vm process to occupy all memory to trigger oom-killer."
    stress-ng --vm $processor_number --vm-bytes ${vm_bytes}k --timeout $duration --metrics-brief --verbose 
    print_test_result stress-ng-oom $?

    # Check if oom-killer triggered.
    kill $kernel_log
    oom_number=$(grep "Out of memory: Kill process" stress_oom_kern.log | wc -l)
    if [ $oom_number -eq 0 ]; then
        echo "Failed to active oom-killer."
        echo "stress-oom-test: fail"
    else
        echo "oom-killer activated $oom_number times within $duration seconds"
        echo "stress-oom-test: pass"
    fi
}

stress_network()
{
    echo "Stress network test duration: $duration"
    echo "Test interface: $interface"
    echo "File link: $link"
    echo "md5: $md5"

    # Check if network set on the interface.
    gateway=$(ip route show default | grep -m 1 default | awk '{print $3}')
    ping -c 10 -I $interface $gateway
    if [ $? -ne 0 ]; then
        echo "Please check network connection and rerun this script"
        exit 1
    fi

    # Run 'stress-ng hdd' stress in the background.
    echo "About to run 'stress-ng --hdd 1' in background"
    stress-ng --hdd 1 > /dev/null 2>&1 &
    stress_ng_hdd=$!
    sleep 5

    end=$(( $(date +%s) + $duration ))
    iteration=0
    while [ $(date +%s) -lt $end ]
    do
        echo
        echo "Running stress_network iteration $iteration"
        if [ -z "$(ps cax | grep "stress-ng-hdd")" ]; then
            echo "'stress-ng --hdd 1' is dead, restarting..."
            stress-ng --hdd 1 > /dev/null 2>&1 &
            stress_ng_hdd=$!
        else
            echo "'stress-ng --hdd 1' is running in background"
        fi

        # Network enable/disable test.
        ip link set $interface down
        sleep 15
        ip link show dev $interface | grep "state DOWN"
        print_test_result network-disable-$iteration $?
        ip link set $interface up
        sleep 15
        ip link show dev $interface | grep "state UP"
        print_test_result network-enable-$iteration $?

        # Check if IP obtained.
        dhclient $interface
        ip=$(ip addr show $interface | grep -w inet | awk '{print $2}' | awk -F'/' '{print $1}')
        test -n "$ip"
        print_test_result network-ip-check-$iteration $?

        # File download test.
        test -e stress-network.img && rm -rf stress-network.img
        curl -O --interface $ip $link
        print_test_result file-download-$iteration $?
        local_md5=$(md5sum stress-network.img | awk '{print $1}')
        test "$local_md5" = "$md5"
        print_test_result file-md5-check-$iteration $?

        iteration=$(( iteration + 1 ))
    done

    kill $stress_ng_hdd
}

## Setup environment and run tests
if [ "$(id -u)" != "0" ]; then
   echo "Please run as root."
   exit 1
fi

# Install stress-ng.
stress-ng -V
if [ $? -ne 0 ]; then
    echo "stress-ng not found, installing..."
    abi=$(uname -m)
    case $abi in
        armv7|armv7l|armv7el|armv7lh)
            cp bin/armeabi/stress-ng /usr/bin/stress-ng
            ;;
        arm64|armv8|arm64-v8a|aarch64)
            cp bin/arm64/stress-ng /usr/bin/stress-ng
            ;;
        *)
            echo "Unknown machine type, exiting..."
            exit 1
            ;;
    esac
    chmod +x /usr/bin/stress-ng
    echo
fi

for i in $tests
do
    # Run 6 hours $i test.
    if [ $i = "stress_oom" ]; then
        backup_log stress_oom_syslog.log
    fi
    backup_log $i.log
    $i 2>&1 | tee $i.log
done