automated: linux: fix OTA upgrade test

The test for OTA upgrade was out of date. This patch brings it to the
usable state with FoundriesFactory OTA and aktualizr-lite.

Signed-off-by: Milosz Wasilewski <milosz.wasilewski@foundries.io>

13 files changed, 574 insertions, 119 deletions

diff --git a/automated/linux/ota-update/README.md b/automated/linux/ota-update/README.md
new file mode 100644
index 00000000..ef283e9d
--- /dev/null
+++ b/automated/linux/ota-update/README.md
@@ -0,0 +1,115 @@
+This test consists of 3 parts:
+ * download-update.yaml
+ * verify-upgrade.yaml
+ * verify-reboot.yaml
+
+To confirm full upgrade all 3 .yaml files need to be used in a test job.
+The DUT needs to reboot twice during the job. Therefore interactive
+shell is used as a part of the job. Example test job (testing part) below:
+
+```
+- test:
+    namespace: before
+    timeout:
+      minutes: 10
+    definitions:
+    - repository: http://github.com/linaro/test-definitions.git
+      from: git
+      path: automated/linux/ota-upgrade/download-update.yaml
+      name: prepare-kernel-upgrade
+- test:
+    namespace: before
+    timeout:
+      minutes: 1
+    interactive:
+    - name: kernel-poweroff
+      prompts: []
+      script:
+      - command: poweroff
+        name: poweroff
+        wait_for_prompt: False
+        successes:
+        - message: "reboot: Power down"
+- deploy:
+    namespace: after
+    connection-namespace: before
+    timeout:
+      minutes: 10
+    to: downloads
+    images:
+      bootloader:
+        url: <example file to download>
+- boot:
+    namespace: after
+    connection-namespace: before
+    prompts:
+     - "Password:"
+     - "root:"
+    timeout:
+      minutes: 10
+    auto_login:
+      login_prompt: 'login:'
+      username: foo
+      password_prompt: "Password:"
+      password: "bar"
+      login_commands:
+      - sudo su
+      - bar
+    method: minimal
+    transfer_overlay:
+      download_command: wget
+      unpack_command: tar -xzf
+- test:
+    namespace: after
+    connection-namespace: before
+    timeout:
+      minutes: 5
+    definitions:
+    - repository: http://github.com/linaro/test-definitions.git
+      from: git
+      path: automated/linux/ota-upgrade/verify-upgrade.yaml
+      name: verify-kernel-upgrade
+      parameters:
+        TARGET_VERSION: "123"
+- deploy:
+    namespace: after2
+    connection-namespace: before
+    timeout:
+      minutes: 10
+    to: downloads
+    images:
+      bootloader:
+        url: <example file to download>
+- boot:
+    namespace: after2
+    connection-namespace: before
+    prompts:
+     - "Password:"
+     - "root:"
+    timeout:
+      minutes: 10
+    auto_login:
+      login_prompt: 'login:'
+      username: foo
+      password_prompt: "Password:"
+      password: "bar"
+      login_commands:
+      - sudo su
+      - bar
+    method: minimal
+    transfer_overlay:
+      download_command: wget
+      unpack_command: tar -xzf
+- test:
+    namespace: after2
+    connection-namespace: before
+    timeout:
+      minutes: 5
+    definitions:
+    - repository: http://github.com/linaro/test-definitions.git
+      from: git
+      path: automated/linux/ota-update/verify-reboot.yaml
+      name: verify-reboot
+      parameters:
+        TARGET_VERSION: "123"
+```
diff --git a/automated/linux/ota-update/aklite-callback.sh b/automated/linux/ota-update/aklite-callback.sh
new file mode 100755
index 00000000..b1fa1df8
--- /dev/null
+++ b/automated/linux/ota-update/aklite-callback.sh
@@ -0,0 +1,4 @@
+#!/bin/bash -e
+
+echo "${MESSAGE}" > /var/sota/ota.signal
+echo "${RESULT}" > /var/sota/ota.result
diff --git a/automated/linux/ota-update/download-update.sh b/automated/linux/ota-update/download-update.sh
new file mode 100755
index 00000000..74ce6e6b
--- /dev/null
+++ b/automated/linux/ota-update/download-update.sh
@@ -0,0 +1,169 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0-only
+# Copyright (C) 2021 Foundries.io Ltd.
+
+# shellcheck disable=SC1091
+. ./sh-lib
+OUTPUT="$(pwd)/output"
+RESULT_FILE="${OUTPUT}/result.txt"
+export RESULT_FILE
+TYPE="kernel"
+UBOOT_VAR_TOOL=fw_printenv
+export UBOOT_VAR_TOOL
+UBOOT_VAR_SET_TOOL=fw_setenv
+export UBOOT_VAR_SET_TOOL
+
+usage() {
+	echo "\
+	Usage: $0 [-t <kernel|uboot>] [-u <u-boot var read>] [-s <u-boot var set>]
+
+    -t <kernel|uboot>
+        This determines type of upgrade test performed:
+        kernel: perform OTA upgrade without forcing firmware upgrade
+        uboot: force firmware upgrade
+    -u <u-boot variable read tool>
+        Set the name of the tool to read u-boot variables
+        On the unsecured systems it will usually be
+        fw_printenv. On secured systems it might be
+        fiovb_printenv
+    -s <u-boot variable set tool>
+        Set the name of the tool to set u-boot variables
+        On the unsecured systems it will usually be
+        fw_setenv. On secured systems it might be
+        fiovb_setenv
+	"
+}
+
+while getopts "t:u:s:h" opts; do
+	case "$opts" in
+        t) TYPE="${OPTARG}";;
+        u) UBOOT_VAR_TOOL="${OPTARG}";;
+        s) UBOOT_VAR_SET_TOOL="${OPTARG}";;
+        h|*) usage ; exit 1 ;;
+	esac
+done
+
+# the script works only on builds with aktualizr-lite
+# and lmp-device-auto-register
+
+! check_root && error_msg "You need to be root to run this script."
+create_out_dir "${OUTPUT}"
+
+ref_bootcount_before_download=0
+ref_rollback_before_download=0
+ref_bootupgrade_available_before_download=0
+ref_upgrade_available_before_download=0
+ref_fiovb_is_secondary_boot_before_download=0
+ref_bootcount_after_download=0
+ref_rollback_after_download=0
+ref_bootupgrade_available_after_download=0
+ref_upgrade_available_after_download=1
+ref_fiovb_is_secondary_boot_after_download=0
+if [ "${TYPE}" = "uboot" ]; then
+    ref_bootupgrade_available_after_download=1
+fi
+
+# configure aklite callback
+cp aklite-callback.sh /var/sota/
+chmod 755 /var/sota/aklite-callback.sh
+
+mkdir -p /etc/sota/conf.d
+cp z-99-aklite-callback.toml /etc/sota/conf.d/
+report_pass "${TYPE}-create-aklite-callback"
+# create signal files
+touch /var/sota/ota.signal
+touch /var/sota/ota.result
+report_pass "${TYPE}-create-signal-files"
+
+#systemctl mask aktualizr-lite
+# enabling lmp-device-auto-register will fail because aklite is masked
+systemctl enable --now lmp-device-auto-register || error_fatal "Unable to register device"
+# aktualizr-lite update
+# TODO: check if there is an update to download
+# if there isn't, terminate the job
+# use "${upgrade_available_after_download}" for now. Find a better solution later
+
+while ! systemctl is-active aktualizr-lite; do
+    echo "Waiting for aktualizr-lite to start"
+    sleep 1
+done
+# add some delay so aklite can setup variables
+sleep 5
+
+# u-boot variables change when aklite starts (at least on some devices)
+# check u-boot variables to ensure we're on freshly flashed device
+bootcount_before_download=$(uboot_variable_value bootcount)
+compare_test_value "${TYPE}_bootcount_before_download" "${ref_bootcount_before_download}" "${bootcount_before_download}"
+rollback_before_download=$(uboot_variable_value rollback)
+compare_test_value "${TYPE}_rollback_before_download" "${ref_rollback_before_download}" "${rollback_before_download}"
+upgrade_available_before_download=$(uboot_variable_value upgrade_available)
+compare_test_value "${TYPE}_upgrade_available_before_download" "${ref_upgrade_available_before_download}" "${upgrade_available_before_download}"
+
+if [ -f /usr/lib/firmware/version.txt ]; then
+    # boot firmware is upgreadable
+    . /usr/lib/firmware/version.txt
+    bootupgrade_available_before_download=$(uboot_variable_value bootupgrade_available)
+    compare_test_value "${TYPE}_bootupgrade_available_before_download" "${ref_bootupgrade_available_before_download}" "${bootupgrade_available_before_download}"
+    bootfirmware_version_before_download=$(uboot_variable_value bootfirmware_version)
+    # shellcheck disable=SC2154
+    compare_test_value "${TYPE}_bootfirmware_version_before_download" "${bootfirmware_version}" "${bootfirmware_version_before_download}"
+    fiovb_is_secondary_boot_before_download=$(uboot_variable_value fiovb.is_secondary_boot)
+    compare_test_value "${TYPE}_fiovb_is_secondary_boot_before_download" "${ref_fiovb_is_secondary_boot_before_download}" "${fiovb_is_secondary_boot_before_download}"
+else
+    report_skip "${TYPE}_bootupgrade_available_before_download"
+    report_skip "${TYPE}_bootfirmware_version_before_download"
+    report_skip "${TYPE}_fiovb_is_secondary_boot_before_download"
+fi
+
+if [ "${TYPE}" = "uboot" ]; then
+    # manually set boot firmware version to 0
+    "${UBOOT_VAR_SET_TOOL}" bootfirmware_version 0
+fi
+
+# wait for 'install-post' signal
+SIGNAL=$(</var/sota/ota.signal)
+while [ ! "${SIGNAL}" = "install-post" ]
+do
+	echo "Sleeping 1s"
+	sleep 1
+	cat /var/sota/ota.signal
+	SIGNAL=$(</var/sota/ota.signal)
+	echo "SIGNAL: ${SIGNAL}."
+done
+report_pass "${TYPE}-install-post-received"
+
+# check variables after download is completed
+bootcount_after_download=$(uboot_variable_value bootcount)
+compare_test_value "${TYPE}_bootcount_after_download" "${ref_bootcount_after_download}" "${bootcount_after_download}"
+rollback_after_download=$(uboot_variable_value rollback)
+compare_test_value "${TYPE}_rollback_after_download" "${ref_rollback_after_download}" "${rollback_after_download}"
+upgrade_available_after_download=$(uboot_variable_value upgrade_available)
+compare_test_value "${TYPE}_upgrade_available_after_download" "${ref_upgrade_available_after_download}" "${upgrade_available_after_download}"
+if [ -f /usr/lib/firmware/version.txt ]; then
+    . /usr/lib/firmware/version.txt
+    bootupgrade_available_after_download=$(uboot_variable_value bootupgrade_available)
+    compare_test_value "${TYPE}_bootupgrade_available_after_download" "${ref_bootupgrade_available_after_download}" "${bootupgrade_available_after_download}"
+    # shellcheck disable=SC2154
+    ref_bootfirmware_version_after_download="${bootfirmware_version}"
+    if [ "${TYPE}" = "uboot" ]; then
+        ref_bootfirmware_version_after_download=0
+    fi
+    bootfirmware_version_after_download=$(uboot_variable_value bootfirmware_version)
+    # shellcheck disable=SC2154
+    compare_test_value "${TYPE}_bootfirmware_version_after_download" "${ref_bootfirmware_version_after_download}" "${bootfirmware_version_after_download}"
+    fiovb_is_secondary_boot_after_download=$(uboot_variable_value fiovb.is_secondary_boot)
+    compare_test_value "${TYPE}_fiovb_is_secondary_boot_after_download" "${ref_fiovb_is_secondary_boot_after_download}" "${fiovb_is_secondary_boot_after_download}"
+else
+    report_skip "${TYPE}_bootupgrade_available_after_download"
+    report_skip "${TYPE}_bootfirmware_version_after_download"
+    report_skip "${TYPE}_fiovb_is_secondary_boot_after_download"
+fi
+
+UPGRADE_AVAILABLE="${upgrade_available_after_download}"
+if [ "${TYPE}" = "uboot" ]; then
+    UPGRADE_AVAILABLE="${bootupgrade_available_after_download}"
+fi
+
+if [ "${UPGRADE_AVAILABLE}" -ne 1 ]; then
+    lava-test-raise "No-update-available-${TYPE}"
+fi
diff --git a/automated/linux/ota-update/download-update.yaml b/automated/linux/ota-update/download-update.yaml
new file mode 100644
index 00000000..1a788529
--- /dev/null
+++ b/automated/linux/ota-update/download-update.yaml
@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: GPL-2.0-only
+# Copyright (C) 2022 Foundries.io
+metadata:
+    format: Lava-Test Test Definition 1.0
+    name: ota-upgrade-download
+    description: "Download OTA update and (optionally)
+    force u-boot upgrade."
+
+    maintainer:
+        - milosz.wasilewski@foundries.io
+    os:
+        - openembedded
+    scope:
+        - functional
+
+    devices:
+        - imx8mm
+        - imx6ull
+
+params:
+        UBOOT_VAR_TOOL: "fw_printenv"
+        UBOOT_VAR_SET_TOOL: "fw_setenv"
+        TYPE: "kernel"
+run:
+    steps:
+        - cd ./automated/linux/ota-update
+        - ./download-update.sh -t "${TYPE}" -u "${UBOOT_VAR_TOOL}" -s "${UBOOT_VAR_SET_TOOL}"
+        - ../../utils/send-to-lava.sh ./output/result.txt
diff --git a/automated/linux/ota-update/ota-update.py b/automated/linux/ota-update/ota-update.py
deleted file mode 100644
index 33d1932d..00000000
--- a/automated/linux/ota-update/ota-update.py
+++ /dev/null
@@ -1,65 +0,0 @@
-import requests
-import json
-import yaml
-import time
-import sys
-import os
-from argparse import ArgumentParser
-
-sys.path.insert(0, "../../lib/")
-import py_test_lib  # nopep8
-
-OUTPUT = "%s/output" % os.getcwd()
-RESULT_FILE = "%s/result.txt" % OUTPUT
-
-parser = ArgumentParser()
-parser.add_argument(
-    "-d",
-    "--device",
-    dest="devicename",
-    default="hikey-r2-01",
-    help="Device Name to be updated",
-)
-parser.add_argument(
-    "-is", "--installed-sha", dest="installed_sha", default="", help="OTA update sha"
-)
-parser.add_argument(
-    "-us", "--update-sha", dest="update_sha", default="", help="OTA update sha"
-)
-args = parser.parse_args()
-url = "http://api.ota-prototype.linaro.org/devices/%s/" % args.devicename
-headers = {"OTA-TOKEN": "BadT0ken5", "Content-type": "application/json"}
-data = json.dumps({"image": {"hash": args.update_sha}})
-
-
-def match_sha_on_server(sha):
-    loop = 0
-    while loop < 20:
-        r = requests.get(url, headers=headers)
-        resp = yaml.load(r.text)
-        currentsha_on_server = (
-            resp.get("deviceImage").get("image").get("hash").get("sha256")
-        )
-        if currentsha_on_server == sha:
-            return 0
-        loop = loop + 1
-        time.sleep(30)
-        if loop == 10:
-            print("FAIL: Installed sha on device did not match")
-            return -1
-
-
-if match_sha_on_server(args.installed_sha) == 0:
-    py_test_lib.add_result(RESULT_FILE, "installed-device-sha-match-server pass")
-    r = requests.put(url, data=data, headers=headers)
-    if match_sha_on_server(args.update_sha) == 0:
-        py_test_lib.add_result(RESULT_FILE, "ota-update-to-%s pass" % args.update_sha)
-        print(
-            "PASS: %s updated to %s successfully" % (args.devicename, args.update_sha)
-        )
-    else:
-        py_test_lib.add_result(RESULT_FILE, "ota-update-to-%s fail" % args.update_sha)
-        print("FAIL: %s update to %s failed" % (args.devicename, args.update_sha))
-else:
-    py_test_lib.add_result(RESULT_FILE, "installed-device-sha-match-server fail")
-    print("FAIL: Insalled device sha to %s mismatched on the server" % args.devicename)
diff --git a/automated/linux/ota-update/ota-update.sh b/automated/linux/ota-update/ota-update.sh
deleted file mode 100755
index c7c7448e..00000000
--- a/automated/linux/ota-update/ota-update.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-RESULT_FILE="$(pwd)/output/result.txt"
-export RESULT_FILE
-mkdir output
-
-DUT_IPADDRESS=$(lava-target-ip)
-DUT=$(case "${DUT_IPADDRESS}" in
-    ("10.7.0.68") echo "hikey-r2-01";;
-    ("10.7.0.69") echo "hikey-r2-02";;
-    ("10.7.0.66") echo "hikey-r2-03";;
-    ("10.7.0.73") echo "hikey-r2-04";;
-    (*) echo "invalid";;
-    esac)
-
-wget http://testdata.linaro.org/apks/osf/"${DUT}"-sota.tar -O sota.tar
-sshpass -p 'osf' scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no sota.tar osf@"${DUT_IPADDRESS}":~/
-sshpass -p 'osf' ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no osf@"${DUT_IPADDRESS}" "echo osf | sudo -S tar -xvf sota.tar -C /var/"
-sshpass -p 'osf' ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no osf@"${DUT_IPADDRESS}" "echo osf | sudo -S systemctl restart aktualizr"
-
-python ota-update.py -d "${DUT}" -is "${BASELINE_SHA}" -us "${UPDATE_SHA}"
-
-# Restore device image back to baseline for future testing. The server information
-# about images on device should match the actual image on the device.
-python ota-update.py -d "${DUT}" -us "${BASELINE_SHA}" -is "${UPDATE_SHA}"
diff --git a/automated/linux/ota-update/ota-update.yaml b/automated/linux/ota-update/ota-update.yaml
deleted file mode 100644
index 1502fe41..00000000
--- a/automated/linux/ota-update/ota-update.yaml
+++ /dev/null
@@ -1,30 +0,0 @@
-metadata:
-    name: ota-update
-    format: "Lava-Test Test Definition 1.0"
-    description: "Tests OTA update on device"
-    maintainer:
-        - vishal.bhoj@linaro.org
-    os:
-        - ubuntu
-        - debian
-        - centos
-        - fedora
-        - openembedded
-    devices:
-        - hi6220-hikey
-
-    scope:
-        - functional
-        - performance
-
-params:
-        # TODO: Need ostree package info to be trigger deployment
-        SKIP_INSTALL: "false"
-        UPDATE_SHA: ""
-        BASELINE_SHA: ""
-
-run:
-    steps:
-        - cd automated/linux/ota-update
-        - . ./ota-update.sh
-        - ../../utils/send-to-lava.sh ./output/result.txt
diff --git a/automated/linux/ota-update/sh-lib b/automated/linux/ota-update/sh-lib
new file mode 100644
index 00000000..91d9cbae
--- /dev/null
+++ b/automated/linux/ota-update/sh-lib
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+. ../../lib/sh-test-lib
+
+uboot_variable_value() {
+    # shellcheck disable=SC2039
+    local var="$1"
+    result=$("${UBOOT_VAR_TOOL}" "${var}")
+    if [ -n "${result}" ]; then
+        echo "${result#*=}"
+    else
+        echo ""
+    fi
+}
+
+compare_test_value() {
+    # shellcheck disable=SC2039
+    local test_name="$1"
+    # shellcheck disable=SC2039
+    local expected_value="$2"
+    # shellcheck disable=SC2039
+    local tested_value="$3"
+
+    echo "Testing: #${test_name}#"
+    echo "Expected: #${expected_value}#"
+    echo "Actual: #${tested_value}#"
+
+    if [ "${expected_value}" = "${tested_value}" ]; then
+        report_pass "${test_name}"
+    else
+        report_fail "${test_name}"
+    fi
+}
diff --git a/automated/linux/ota-update/verify-reboot.sh b/automated/linux/ota-update/verify-reboot.sh
new file mode 100755
index 00000000..12cf4d2c
--- /dev/null
+++ b/automated/linux/ota-update/verify-reboot.sh
@@ -0,0 +1,75 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0-only
+# Copyright (C) 2021 Foundries.io Ltd.
+
+# shellcheck disable=SC1091
+. ./sh-lib
+OUTPUT="$(pwd)/output"
+RESULT_FILE="${OUTPUT}/result.txt"
+export RESULT_FILE
+REF_TARGET_VERSION=1
+UBOOT_VAR_TOOL=fw_printenv
+export UBOOT_VAR_TOOL
+UBOOT_VAR_SET_TOOL=fw_setenv
+export UBOOT_VAR_SET_TOOL
+
+usage() {
+    echo "\
+    Usage: $0 [-u <u-boot variable read>] [-s <u-boot variable set>] [-v <expected version>]
+
+    -v <target version>
+        Version of the target expected after reboot.
+        Defaults to 1. Should be set to avoid bad results
+    -u <u-boot variable read tool>
+        Set the name of the tool to read u-boot variables
+        On the unsecured systems it will usually be
+        fw_printenv. On secured systems it might be
+        fiovb_printenv
+    -s <u-boot variable set tool>
+        Set the name of the tool to set u-boot variables
+        On the unsecured systems it will usually be
+        fw_setenv. On secured systems it might be
+        fiovb_setenv
+    "
+}
+
+while getopts "u:s:v:h" opts; do
+    case "$opts" in
+        u) UBOOT_VAR_TOOL="${OPTARG}";;
+        s) UBOOT_VAR_SET_TOOL="${OPTARG}";;
+        v) REF_TARGET_VERSION="${OPTARG}";;
+        h|*) usage ; exit 1 ;;
+    esac
+done
+
+! check_root && error_msg "You need to be root to run this script."
+create_out_dir "${OUTPUT}"
+
+ref_bootcount_after_upgrade=0
+ref_rollback_after_upgrade=0
+ref_bootupgrade_available_after_upgrade=0
+ref_upgrade_available_after_upgrade=0
+ref_fiovb_is_secondary_boot_after_upgrade=0
+. /usr/lib/firmware/version.txt
+# shellcheck disable=SC2154
+ref_bootfirmware_version_after_upgrade="${bootfirmware_version}"
+
+# check u-boot variables to ensure upgrade happend
+bootcount_after_upgrade=$(uboot_variable_value bootcount)
+compare_test_value "bootcount_after_upgrade" "${ref_bootcount_after_upgrade}" "${bootcount_after_upgrade}"
+rollback_after_upgrade=$(uboot_variable_value rollback)
+compare_test_value "rollback_after_upgrade" "${ref_rollback_after_upgrade}" "${rollback_after_upgrade}"
+upgrade_available_after_upgrade=$(uboot_variable_value upgrade_available)
+compare_test_value "upgrade_available_after_upgrade" "${ref_upgrade_available_after_upgrade}" "${upgrade_available_after_upgrade}"
+bootupgrade_available_after_upgrade=$(uboot_variable_value bootupgrade_available)
+compare_test_value "bootupgrade_available_after_upgrade" "${ref_bootupgrade_available_after_upgrade}" "${bootupgrade_available_after_upgrade}"
+bootfirmware_version_after_upgrade=$(uboot_variable_value bootfirmware_version)
+compare_test_value "bootfirmware_version_after_upgrade" "${ref_bootfirmware_version_after_upgrade}" "${bootfirmware_version_after_upgrade}"
+fiovb_is_secondary_boot_after_upgrade=$(uboot_variable_value fiovb.is_secondary_boot)
+compare_test_value "fiovb_is_secondary_boot_after_upgrade" "${ref_fiovb_is_secondary_boot_after_upgrade}" "${fiovb_is_secondary_boot_after_upgrade}"
+
+. /etc/os-release
+# shellcheck disable=SC2154
+compare_test_value "target_version_after_upgrade" "${REF_TARGET_VERSION}" "${IMAGE_VERSION}"
+cat /etc/os-release
+cat /boot/loader/uEnv.txt
diff --git a/automated/linux/ota-update/verify-reboot.yaml b/automated/linux/ota-update/verify-reboot.yaml
new file mode 100644
index 00000000..5d627eee
--- /dev/null
+++ b/automated/linux/ota-update/verify-reboot.yaml
@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: GPL-2.0-only
+# Copyright (C) 2022 Foundries.io
+metadata:
+    format: Lava-Test Test Definition 1.0
+    name: ota-update-reboot
+    description: "Verify system booted with upgrade. In case of u-boot
+        upgrade it should boot to primary partition with proper
+        firmware version."
+
+    maintainer:
+        - milosz.wasilewski@foundries.io
+    os:
+        - openembedded
+    scope:
+        - functional
+
+    devices:
+        - imx8mm
+        - imx6ull
+
+params:
+        UBOOT_VAR_TOOL: "fw_printenv"
+        UBOOT_VAR_SET_TOOL: "fw_setenv"
+        TARGET_VERSION: "1"
+run:
+    steps:
+        - cd ./automated/linux/ota-update
+        - ./verify-reboot.sh  -u "${UBOOT_VAR_TOOL}" -s "${UBOOT_VAR_SET_TOOL}" -v "${TARGET_VERSION}"
+        - ../../utils/send-to-lava.sh ./output/result.txt
diff --git a/automated/linux/ota-update/verify-update.sh b/automated/linux/ota-update/verify-update.sh
new file mode 100755
index 00000000..2febe3c7
--- /dev/null
+++ b/automated/linux/ota-update/verify-update.sh
@@ -0,0 +1,88 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0-only
+# Copyright (C) 2021 Foundries.io Ltd.
+
+# shellcheck disable=SC1091
+. ./sh-lib
+OUTPUT="$(pwd)/output"
+RESULT_FILE="${OUTPUT}/result.txt"
+export RESULT_FILE
+TYPE="kernel"
+REF_TARGET_VERSION=1
+UBOOT_VAR_TOOL=fw_printenv
+export UBOOT_VAR_TOOL
+UBOOT_VAR_SET_TOOL=fw_setenv
+export UBOOT_VAR_SET_TOOL
+
+usage() {
+    echo "\
+    Usage: $0 [-t <kernel|uboot>] [-u <u-boot variable read>] [-s <u-boot variable set>] [-v <expected version>]
+
+    -t <kernel|uboot>
+        Defauts to 'kernel'. It either enables or disables
+        checking the upgrade status of u-boot firmware
+    -v <target version>
+        Version of the target expected after reboot.
+        Defaults to 1. Should be set to avoid bad results
+    -u <u-boot variable read tool>
+        Set the name of the tool to read u-boot variables
+        On the unsecured systems it will usually be
+        fw_printenv. On secured systems it might be
+        fiovb_printenv
+    -s <u-boot variable set tool>
+        Set the name of the tool to set u-boot variables
+        On the unsecured systems it will usually be
+        fw_setenv. On secured systems it might be
+        fiovb_setenv
+    "
+}
+
+while getopts "t:u:s:v:h" opts; do
+    case "$opts" in
+        t) TYPE="${OPTARG}";;
+        u) UBOOT_VAR_TOOL="${OPTARG}";;
+        s) UBOOT_VAR_SET_TOOL="${OPTARG}";;
+        v) REF_TARGET_VERSION="${OPTARG}";;
+        h|*) usage ; exit 1 ;;
+    esac
+done
+
+! check_root && error_msg "You need to be root to run this script."
+create_out_dir "${OUTPUT}"
+
+ref_bootcount_after_upgrade=0
+ref_rollback_after_upgrade=0
+ref_bootupgrade_available_after_upgrade=0
+ref_upgrade_available_after_upgrade=0
+ref_fiovb_is_secondary_boot_after_upgrade=0
+
+if [ "${TYPE}" = "uboot" ]; then
+    ref_bootupgrade_available_after_upgrade=1
+    # boots to secondary slot. Set to 0 on a subsequent reboot
+    ref_fiovb_is_secondary_boot_after_upgrade=1
+    # set to 0 forcibly. It stays this way on the 1st reboot
+    ref_bootfirmware_version_after_upgrade=0
+else
+    . /usr/lib/firmware/version.txt
+    # shellcheck disable=SC2154
+    ref_bootfirmware_version_after_upgrade="${bootfirmware_version}"
+fi
+# check u-boot variables to ensure upgrade happend
+bootcount_after_upgrade=$(uboot_variable_value bootcount)
+compare_test_value "bootcount_after_upgrade" "${ref_bootcount_after_upgrade}" "${bootcount_after_upgrade}"
+rollback_after_upgrade=$(uboot_variable_value rollback)
+compare_test_value "rollback_after_upgrade" "${ref_rollback_after_upgrade}" "${rollback_after_upgrade}"
+upgrade_available_after_upgrade=$(uboot_variable_value upgrade_available)
+compare_test_value "upgrade_available_after_upgrade" "${ref_upgrade_available_after_upgrade}" "${upgrade_available_after_upgrade}"
+bootupgrade_available_after_upgrade=$(uboot_variable_value bootupgrade_available)
+compare_test_value "bootupgrade_available_after_upgrade" "${ref_bootupgrade_available_after_upgrade}" "${bootupgrade_available_after_upgrade}"
+bootfirmware_version_after_upgrade=$(uboot_variable_value bootfirmware_version)
+compare_test_value "bootfirmware_version_after_upgrade" "${ref_bootfirmware_version_after_upgrade}" "${bootfirmware_version_after_upgrade}"
+fiovb_is_secondary_boot_after_upgrade=$(uboot_variable_value fiovb.is_secondary_boot)
+compare_test_value "fiovb_is_secondary_boot_after_upgrade" "${ref_fiovb_is_secondary_boot_after_upgrade}" "${fiovb_is_secondary_boot_after_upgrade}"
+
+. /etc/os-release
+# shellcheck disable=SC2154
+compare_test_value "target_version_after_upgrade" "${REF_TARGET_VERSION}" "${IMAGE_VERSION}"
+cat /etc/os-release
+cat /boot/loader/uEnv.txt
diff --git a/automated/linux/ota-update/verify-update.yaml b/automated/linux/ota-update/verify-update.yaml
new file mode 100644
index 00000000..58128c19
--- /dev/null
+++ b/automated/linux/ota-update/verify-update.yaml
@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: GPL-2.0-only
+# Copyright (C) 2022 Foundries.io
+metadata:
+    format: Lava-Test Test Definition 1.0
+    name: ota-update-verify
+    description: "Verify system booted with upgrade"
+
+    maintainer:
+        - milosz.wasilewski@foundries.io
+    os:
+        - openembedded
+    scope:
+        - functional
+
+    devices:
+        - imx8mm
+        - imx6ull
+
+params:
+        UBOOT_VAR_TOOL: "fw_printenv"
+        UBOOT_VAR_SET_TOOL: "fw_setenv"
+        TYPE: "kernel"
+        TARGET_VERSION: "1"
+run:
+    steps:
+        - cd ./automated/linux/ota-update
+        - ./verify-update.sh  -t "${TYPE}" -u "${UBOOT_VAR_TOOL}" -s "${UBOOT_VAR_SET_TOOL}" -v "${TARGET_VERSION}"
+        - ../../utils/send-to-lava.sh ./output/result.txt
diff --git a/automated/linux/ota-update/z-99-aklite-callback.toml b/automated/linux/ota-update/z-99-aklite-callback.toml
new file mode 100644
index 00000000..7a8ae68c
--- /dev/null
+++ b/automated/linux/ota-update/z-99-aklite-callback.toml
@@ -0,0 +1,5 @@
+[pacman]
+callback_program = "/var/sota/aklite-callback.sh"
+[bootloader]
+reboot_command = "/bin/true"
+