diff options
-rw-r--r-- | doc/v2/actions-deploy-to-recovery.rsti | 154 | ||||
-rw-r--r-- | doc/v2/actions-deploy.rst | 1 | ||||
-rw-r--r-- | lava_dispatcher/actions/boot/recovery.py | 108 | ||||
-rw-r--r-- | lava_dispatcher/actions/boot/strategies.py | 1 | ||||
-rw-r--r-- | lava_dispatcher/actions/deploy/recovery.py | 74 | ||||
-rw-r--r-- | lava_dispatcher/actions/deploy/strategies.py | 1 | ||||
-rw-r--r-- | lava_dispatcher/power.py | 2 | ||||
-rw-r--r-- | lava_dispatcher/test/pipeline_refs/hi6220-recovery.yaml | 210 | ||||
-rw-r--r-- | lava_dispatcher/test/sample_jobs/hi6220-recovery.yaml | 235 | ||||
-rw-r--r-- | lava_dispatcher/test/test_recovery.py | 101 | ||||
-rw-r--r-- | lava_dispatcher/utils/udev.py | 2 | ||||
-rw-r--r-- | lava_scheduler_app/tests/device-types/base-fastboot.jinja2 | 11 | ||||
-rw-r--r-- | lava_scheduler_app/tests/device-types/hi6220-hikey-bl.jinja2 | 29 | ||||
-rw-r--r-- | lava_scheduler_app/tests/devices/hi6220-hikey-bl-01.jinja2 | 31 | ||||
-rw-r--r-- | lava_scheduler_app/tests/test_templates.py | 23 | ||||
-rw-r--r-- | lava_scheduler_app/utils.py | 2 | ||||
-rwxr-xr-x | share/lava_lxc_device_add.py | 5 |
17 files changed, 988 insertions, 2 deletions
diff --git a/doc/v2/actions-deploy-to-recovery.rsti b/doc/v2/actions-deploy-to-recovery.rsti new file mode 100644 index 000000000..7757b2b77 --- /dev/null +++ b/doc/v2/actions-deploy-to-recovery.rsti @@ -0,0 +1,154 @@ +.. index:: deploy to recovery + +.. _deploy_to_recovery: + +to: recovery +************ + +Deployment to ``recovery`` allows the use of device dictionary commands +and an LXC test shell to automate recovery mode operations on some +DUTs. + +Successful use of recovery deployments require support by the admins +and by the test writers. + +.. note:: In recovery mode, the device may have different identifiers + and might no longer be unique. This can result in requiring a new + device-type template and only creating one device of this type on + any one worker. Not all devices can support automated recovery + mode. + + Additionally, recovery deployments are **blind** - there is ``udev`` + support to add the device to the LXC but no serial connection, so no + output will be read from the DUT. All tools and libraries required + to execute the recovery test shell need to be added to the LXC. For + example, using an earlier test shell inside the LXC. + +#. Download scripts and binaries to transfer to the device +#. Copy the downloaded artifacts into the LXC. +#. Ensure that power to the device is OFF +#. Execute the ``recovery_mode_command`` to use relays or similar to + put the device into recovery mode, in a dedicated :term:`namespace`. + + .. code-block:: jinja + + {% set recovery_mode_command = [ + '/home/neil/lava-lab/shared/lab-scripts/eth008_control -a 10.15.0.171 -r 1 -s off', + '/home/neil/lava-lab/shared/lab-scripts/eth008_control -a 10.15.0.171 -r 2 -s on'] %} + +#. Apply power. + + .. code-block:: jinja + + - boot: + namespace: recovery + timeout: + minutes: 5 + method: recovery + commands: recovery + +The test job would then define a test action which executes the scripts +using the downloaded files and completes recovery. This script may have +to wait for the device to appear and as the device may then have an +unpredictable device node name, an action to create a symlink with a +known name is likely to be required. The use of LXC ensures that only +one suitable device exists, as long as the device configuration and +recovery mode operations only require a single device matching the +check in the recovery script. + +Example: for the HiKey 6220, the `recovery mode operations +<https://github.com/96boards/documentation/wiki/HiKeyUEFI#flash-binaries-to-emmc->`_ +could be executed as steps in the test shell as follows: + +.. code-block:: yaml + + run: + steps: + - find /dev/ -name 'ttyUSB*' -xdev -type c -quit -exec ln -s {} /dev/recovery ';' + - python /lava-lxc/hisi-idt.py --img1=/lava-lxc/l-loader.bin -d /dev/recovery + # fastboot should wait for the device to reset here + # udev rule copes with adding it to the LXC once it appears + - fastboot flash ptable /lava-lxc/ptable-linux.img + - fastboot flash ptable /lava-lxc/fip.bin + - fastboot flash ptable /lava-lxc/nvme.img + # next boot action takes care of exiting from recovery mode + +.. important:: Make these commands **portable** so that the same script + can be used to deploy new firmware to the device outside of LAVA. + When using a test shell to handle firmware deployments, make sure + that a failure of any test shell command fails the job by using + ``lava-test-raise``. + + .. code-block:: shell + + command(){ + if [ -n "$(which lava-test-case || true)" ]; then + echo $2 + $2 && lava-test-case "$1" --result pass || lava-test-raise "$1" + else + echo $2 + $2 + fi + } + + Then call the function with two arguments, the test case name (with + no spaces) and the command to execute (with substitutions for the + parameterized variables for the files which were downloaded by the + test job): + + .. code-block:: shell + + command 'hisi-idt-l-loader' "python ${SCRIPT} --img1=${LOADER} -d /dev/recovery" + + Take note of the quoting in this shell example. The first parameter + can use single quotes but the second parameter **must** use double + quotes ``"`` so that the values of ``$SCRIPT`` and ``$LOADER`` are + substituted. Portable scripts are free to use whatever language you + prefer. + +.. seealso:: :ref:`test_definition_portability` + +Examples for hikey 6220: + +* https://git.linaro.org/lava-team/refactoring.git/plain/testdefs/hikey-6220-recovery.yaml +* https://git.linaro.org/lava-team/refactoring.git/tree/scripts/hikey-6220-recovery.sh + +When the test shell exits, the device is reset using a second boot ``recovery`` +operation. + +.. code-block:: yaml + + - boot: + namespace: recovery + timeout: + minutes: 5 + method: recovery + commands: exit + +A ``recovery_exit_command`` must be specified in the device dictionary. + +.. code-block:: jinja + + {% set recovery_exit_command = [ + '/home/neil/lava-lab/shared/lab-scripts/eth008_control -a 10.15.0.171 -r 1 -s on', + '/home/neil/lava-lab/shared/lab-scripts/eth008_control -a 10.15.0.171 -r 2 -s off'] %} + +Test jobs can terminate early (either through bugs or cancellation), so +it is important to include the ``recovery_exit`` support in the +``power_off_command`` so that the device is left in a suitable state +for the next test job in the queue. + +.. code-block:: jinja + + {% set power_off_command = ['/usr/bin/pduclient --daemon calvin --hostname pdu --command off --port 04', + 'sleep 30', + '/home/neil/lava-lab/shared/lab-scripts/eth008_control -a 10.15.0.171 -r 1 -s on', + '/home/neil/lava-lab/shared/lab-scripts/eth008_control -a 10.15.0.171 -r 2 -s off'] %} + +The additional command may take some time to complete, so the timeout +of the power_off action may also need extending in the device-type +template. + +.. code-block:: jinja + + {% set action_timeout_power_off = 60 %} diff --git a/doc/v2/actions-deploy.rst b/doc/v2/actions-deploy.rst index e5e1620e6..bb485e303 100644 --- a/doc/v2/actions-deploy.rst +++ b/doc/v2/actions-deploy.rst @@ -64,6 +64,7 @@ Parameter List .. include:: actions-deploy-to-nbd.rsti .. include:: actions-deploy-to-usb.rsti .. include:: actions-deploy-to-download.rsti +.. include:: actions-deploy-to-recovery.rsti .. index:: deploy os diff --git a/lava_dispatcher/actions/boot/recovery.py b/lava_dispatcher/actions/boot/recovery.py new file mode 100644 index 000000000..a813e4010 --- /dev/null +++ b/lava_dispatcher/actions/boot/recovery.py @@ -0,0 +1,108 @@ +# Copyright (C) 2018 Linaro Limited +# +# Author: Neil Williams <neil.williams@linaro.org> +# +# This file is part of LAVA. +# +# LAVA is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# LAVA is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along +# with this program; if not, see <http://www.gnu.org/licenses>. + + +from lava_dispatcher.logical import Boot +from lava_dispatcher.action import ( + Action, + InfrastructureError, + Pipeline, +) +from lava_dispatcher.power import PowerOn, PowerOff + + +class RecoveryBoot(Boot): + + compatibility = 4 + + def __init__(self, parent, parameters): + super().__init__(parent) + self.action = RecoveryBootAction() + self.action.section = self.action_type + self.action.job = self.job + parent.add_action(self.action, parameters) + + @classmethod + def accepts(cls, device, parameters): + if 'method' in parameters: + if parameters['method'] == 'recovery': + return True, 'accepted' + return False, 'boot "method" was not "recovery"' + + +class RecoveryBootAction(Action): + + name = "recovery-boot" + description = "handle entering and leaving recovery mode" + summary = "boot into or out of recovery mode" + + def populate(self, parameters): + """ + PowerOff commands will include recovery mode switching commands + so that when jobs end, the device is available. + Use PowerOn instead of ResetDevice so that the effect of the + switching is preserved until the recovery boot action which + specifies the 'exit' command. + """ + super().populate(parameters) + self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters) + if parameters['commands'] == 'recovery': + # only switch into recovery mode with power off. + self.internal_pipeline.add_action(PowerOff()) + self.internal_pipeline.add_action(SwitchRecoveryCommand(mode='recovery_mode')) + self.internal_pipeline.add_action(PowerOn()) + elif parameters['commands'] == 'exit': + self.internal_pipeline.add_action(PowerOff()) + self.internal_pipeline.add_action(SwitchRecoveryCommand(mode='recovery_exit')) + self.internal_pipeline.add_action(PowerOn()) + else: + self.errors = "Invalid recovery command" + + +class SwitchRecoveryCommand(Action): + + name = 'switch-recovery' + description = 'call commands to switch device into and out of recovery' + summary = 'execute recovery mode commands' + + def __init__(self, mode): + super().__init__() + self.recovery = [] + self.mode = mode + + def validate(self): + super().validate() + self.recovery = self.job.device['actions']['deploy']['methods']['recovery'] + if 'commands' not in self.recovery: + self.errors = "Missing commands to enter recovery mode" + command = self.recovery['commands'].get(self.mode, None) + if not command: + self.errors = "Unable to find %s recovery command" % self.mode + + def run(self, connection, max_end_time, args=None): + connection = super().run(connection, max_end_time, args) + command = self.recovery['commands'][self.mode] + self.logger.info("Switching using '%s' recovery command", self.mode) + if not isinstance(command, list): + command = [command] + for cmd in command: + if not self.run_command(cmd.split(' '), allow_silent=True): + raise InfrastructureError("[recovery] %s failed for %s" % (cmd, self.mode)) + return connection diff --git a/lava_dispatcher/actions/boot/strategies.py b/lava_dispatcher/actions/boot/strategies.py index 161a31d43..20a6aac34 100644 --- a/lava_dispatcher/actions/boot/strategies.py +++ b/lava_dispatcher/actions/boot/strategies.py @@ -41,3 +41,4 @@ from lava_dispatcher.actions.boot.ssh import SshLogin, Schroot from lava_dispatcher.actions.boot.u_boot import UBoot from lava_dispatcher.actions.boot.uefi import UefiShell from lava_dispatcher.actions.boot.uefi_menu import UefiMenu +from lava_dispatcher.actions.boot.recovery import RecoveryBoot diff --git a/lava_dispatcher/actions/deploy/recovery.py b/lava_dispatcher/actions/deploy/recovery.py new file mode 100644 index 000000000..f6de71ba0 --- /dev/null +++ b/lava_dispatcher/actions/deploy/recovery.py @@ -0,0 +1,74 @@ +# Copyright (C) 2018 Linaro Limited +# +# Author: Neil Williams <neil.williams@linaro.org> +# +# This file is part of LAVA. +# +# LAVA is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# LAVA is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along +# with this program; if not, see <http://www.gnu.org/licenses>. + +from lava_dispatcher.action import Pipeline +from lava_dispatcher.connections.serial import ConnectDevice +from lava_dispatcher.actions.deploy.download import DownloaderAction, CopyToLxcAction +from lava_dispatcher.actions.deploy import DeployAction +from lava_dispatcher.logical import Deployment + + +class RecoveryModeAction(DeployAction): + + name = "deploy-recovery-mode" + description = "deploy firmware by switching to recovery mode" + summary = "deploy firmware in recovery mode" + + def populate(self, parameters): + super().populate(parameters) + self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters) + recovery = self.job.device['actions']['deploy']['methods']['recovery'] + recovery_dir = self.mkdtemp() + image_keys = sorted(parameters['images'].keys()) + for image in image_keys: + if image != 'yaml_line': + self.internal_pipeline.add_action(DownloaderAction(image, recovery_dir)) + self.internal_pipeline.add_action(CopyToLxcAction()) + + tags = [] + if 'tags' in recovery: + tags = recovery['tags'] + if 'serial' in tags: + # might not be a usable shell here, just power on. + # FIXME: if used, FastbootAction must not try to reconnect + self.internal_pipeline.add_action(ConnectDevice()) + + +class RecoveryMode(Deployment): + + compatibility = 4 + name = 'recovery-mode' + + def __init__(self, parent, parameters): + super().__init__(parent) + self.action = RecoveryModeAction() + self.action.section = self.action_type + self.action.job = self.job + parent.add_action(self.action, parameters) + + @classmethod + def accepts(cls, device, parameters): + if 'recovery' not in device['actions']['deploy']['methods']: + return False, "'recovery' not in the device configuration deploy methods" + if parameters['to'] != 'recovery': + return False, '"to" parameter is not "recovery"' + if 'images' not in parameters: + return False, '"images" is not in the deployment parameters' + return True, 'accepted' diff --git a/lava_dispatcher/actions/deploy/strategies.py b/lava_dispatcher/actions/deploy/strategies.py index 9a9c073d5..55367d54d 100644 --- a/lava_dispatcher/actions/deploy/strategies.py +++ b/lava_dispatcher/actions/deploy/strategies.py @@ -38,3 +38,4 @@ from lava_dispatcher.actions.deploy.ssh import Ssh from lava_dispatcher.actions.deploy.tftp import Tftp from lava_dispatcher.actions.deploy.uboot_ums import UBootUMS from lava_dispatcher.actions.deploy.vemsd import VExpressMsd +from lava_dispatcher.actions.deploy.recovery import RecoveryMode diff --git a/lava_dispatcher/power.py b/lava_dispatcher/power.py index 55a27b940..cf6cf2636 100644 --- a/lava_dispatcher/power.py +++ b/lava_dispatcher/power.py @@ -265,7 +265,7 @@ class ReadFeedback(Action): if feedback_connection: feedbacks.append((feedback_ns, feedback_connection)) else: - self.logger.warning("No connection for namespace %s", feedback_ns) + self.logger.debug("No connection for namespace %s", feedback_ns) for feedback in feedbacks: bytes_read = feedback[1].listen_feedback(timeout=self.duration) # ignore empty or single newline-only content diff --git a/lava_dispatcher/test/pipeline_refs/hi6220-recovery.yaml b/lava_dispatcher/test/pipeline_refs/hi6220-recovery.yaml new file mode 100644 index 000000000..db32441b8 --- /dev/null +++ b/lava_dispatcher/test/pipeline_refs/hi6220-recovery.yaml @@ -0,0 +1,210 @@ +- class: actions.deploy.lxc.LxcAction + name: lxc-deploy + pipeline: + - {class: actions.deploy.lxc.LxcCreateAction, name: lxc-create-action} + - {class: actions.deploy.lxc.LxcCreateUdevRuleAction, name: lxc-create-udev-rule-action} + - {class: actions.boot.lxc.LxcStartAction, name: boot-lxc} + - {class: actions.deploy.lxc.LxcAptUpdateAction, name: lxc-apt-update} + - {class: actions.deploy.lxc.LxcAptInstallAction, name: lxc-apt-install} + - {class: actions.boot.lxc.LxcStopAction, name: lxc-stop} + - {class: actions.deploy.environment.DeployDeviceEnvironment, name: deploy-device-env} + - class: actions.deploy.overlay.OverlayAction + name: lava-overlay + pipeline: + - {class: actions.deploy.overlay.SshAuthorize, name: ssh-authorize} + - {class: actions.deploy.overlay.VlandOverlayAction, name: lava-vland-overlay} + - {class: actions.deploy.overlay.MultinodeOverlayAction, name: lava-multinode-overlay} + - class: actions.deploy.testdef.TestDefinitionAction + name: test-definition + pipeline: + - {class: actions.deploy.testdef.GitRepoAction, name: git-repo-action} + - {class: actions.deploy.testdef.TestOverlayAction, name: test-overlay} + - {class: actions.deploy.testdef.TestInstallAction, name: test-install-overlay} + - {class: actions.deploy.testdef.TestRunnerAction, name: test-runscript-overlay} + - {class: actions.deploy.testdef.GitRepoAction, name: git-repo-action} + - {class: actions.deploy.testdef.TestOverlayAction, name: test-overlay} + - {class: actions.deploy.testdef.TestInstallAction, name: test-install-overlay} + - {class: actions.deploy.testdef.TestRunnerAction, name: test-runscript-overlay} + - {class: actions.deploy.testdef.GitRepoAction, name: git-repo-action} + - {class: actions.deploy.testdef.TestOverlayAction, name: test-overlay} + - {class: actions.deploy.testdef.TestInstallAction, name: test-install-overlay} + - {class: actions.deploy.testdef.TestRunnerAction, name: test-runscript-overlay} + - {class: actions.deploy.testdef.InlineRepoAction, name: inline-repo-action} + - {class: actions.deploy.testdef.TestOverlayAction, name: test-overlay} + - {class: actions.deploy.testdef.TestInstallAction, name: test-install-overlay} + - {class: actions.deploy.testdef.TestRunnerAction, name: test-runscript-overlay} + - {class: actions.deploy.overlay.CompressOverlay, name: compress-overlay} + - {class: actions.deploy.overlay.PersistentNFSOverlay, name: persistent-nfs-overlay} + - {class: actions.deploy.apply_overlay.ApplyLxcOverlay, name: apply-lxc-overlay} +- class: actions.boot.lxc.BootLxcAction + name: lxc-boot + pipeline: + - {class: actions.boot.lxc.LxcStartAction, name: boot-lxc} + - {class: actions.boot.lxc.LxcAddStaticDevices, name: lxc-add-static} + - {class: connections.lxc.ConnectLxc, name: connect-lxc} + - {class: shell.ExpectShellSession, name: expect-shell-connection} + - {class: actions.boot.environment.ExportDeviceEnvironment, name: export-device-env} +- class: actions.deploy.recovery.RecoveryModeAction + name: deploy-recovery-mode + pipeline: + - class: actions.deploy.download.DownloaderAction + name: download-retry + pipeline: + - {class: actions.deploy.download.HttpDownloadAction, name: http-download} + - class: actions.deploy.download.DownloaderAction + name: download-retry + pipeline: + - {class: actions.deploy.download.HttpDownloadAction, name: http-download} + - class: actions.deploy.download.DownloaderAction + name: download-retry + pipeline: + - {class: actions.deploy.download.HttpDownloadAction, name: http-download} + - class: actions.deploy.download.DownloaderAction + name: download-retry + pipeline: + - {class: actions.deploy.download.HttpDownloadAction, name: http-download} + - class: actions.deploy.download.DownloaderAction + name: download-retry + pipeline: + - {class: actions.deploy.download.HttpDownloadAction, name: http-download} + - {class: actions.deploy.download.CopyToLxcAction, name: copy-to-lxc} +- class: actions.boot.recovery.RecoveryBootAction + name: recovery-boot + pipeline: + - {class: power.PowerOff, name: power-off} + - {class: actions.boot.recovery.SwitchRecoveryCommand, name: switch-recovery} + - {class: power.PowerOn, name: power-on} +- class: actions.test.shell.TestShellRetry + name: lava-test-retry + pipeline: + - {class: actions.test.shell.TestShellAction, name: lava-test-shell} +- class: actions.boot.recovery.RecoveryBootAction + name: recovery-boot + pipeline: + - {class: power.PowerOff, name: power-off} + - {class: actions.boot.recovery.SwitchRecoveryCommand, name: switch-recovery} + - {class: power.PowerOn, name: power-on} +- class: actions.deploy.fastboot.FastbootAction + name: fastboot-deploy + pipeline: + - {class: connections.serial.ConnectDevice, name: connect-device} + - class: power.ResetDevice + name: reset-device + pipeline: + - {class: power.PDUReboot, name: pdu-reboot} + - class: actions.deploy.download.DownloaderAction + name: download-retry + pipeline: + - {class: actions.deploy.download.HttpDownloadAction, name: http-download} + - class: actions.deploy.download.DownloaderAction + name: download-retry + pipeline: + - {class: actions.deploy.download.HttpDownloadAction, name: http-download} + - class: actions.deploy.download.DownloaderAction + name: download-retry + pipeline: + - {class: actions.deploy.download.HttpDownloadAction, name: http-download} + - class: actions.deploy.download.DownloaderAction + name: download-retry + pipeline: + - {class: actions.deploy.download.HttpDownloadAction, name: http-download} + - class: actions.deploy.download.DownloaderAction + name: download-retry + pipeline: + - {class: actions.deploy.download.HttpDownloadAction, name: http-download} + - class: actions.deploy.fastboot.FastbootFlashOrderAction + name: fastboot-flash-order-action + pipeline: + - {class: power.ReadFeedback, name: read-feedback} + - {class: actions.deploy.fastboot.FastbootFlashAction, name: fastboot-flash-action} + - {class: power.PDUReboot, name: pdu-reboot} + - {class: power.ReadFeedback, name: read-feedback} + - {class: actions.deploy.fastboot.FastbootFlashAction, name: fastboot-flash-action} + - {class: power.PDUReboot, name: pdu-reboot} + - {class: power.ReadFeedback, name: read-feedback} + - {class: actions.deploy.fastboot.FastbootFlashAction, name: fastboot-flash-action} + - {class: actions.deploy.fastboot.FastbootFlashAction, name: fastboot-flash-action} + - {class: actions.deploy.fastboot.FastbootFlashAction, name: fastboot-flash-action} +- class: actions.boot.grub.GrubSequenceAction + name: grub-sequence-action + pipeline: + - {class: actions.boot.fastboot.WaitFastBootInterrupt, name: wait-fastboot-interrupt} + - {class: actions.boot.AutoLoginAction, name: auto-login-action} +- class: actions.test.shell.TestShellRetry + name: lava-test-retry + pipeline: + - {class: actions.test.shell.TestShellAction, name: lava-test-shell} +- class: actions.deploy.fastboot.FastbootAction + name: fastboot-deploy + pipeline: + - class: actions.deploy.overlay.OverlayAction + name: lava-overlay + pipeline: + - {class: actions.deploy.overlay.SshAuthorize, name: ssh-authorize} + - {class: actions.deploy.overlay.VlandOverlayAction, name: lava-vland-overlay} + - {class: actions.deploy.overlay.MultinodeOverlayAction, name: lava-multinode-overlay} + - class: actions.deploy.testdef.TestDefinitionAction + name: test-definition + pipeline: + - {class: actions.deploy.testdef.GitRepoAction, name: git-repo-action} + - {class: actions.deploy.testdef.TestOverlayAction, name: test-overlay} + - {class: actions.deploy.testdef.TestInstallAction, name: test-install-overlay} + - {class: actions.deploy.testdef.TestRunnerAction, name: test-runscript-overlay} + - {class: actions.deploy.testdef.InlineRepoAction, name: inline-repo-action} + - {class: actions.deploy.testdef.TestOverlayAction, name: test-overlay} + - {class: actions.deploy.testdef.TestInstallAction, name: test-install-overlay} + - {class: actions.deploy.testdef.TestRunnerAction, name: test-runscript-overlay} + - {class: actions.deploy.overlay.CompressOverlay, name: compress-overlay} + - {class: actions.deploy.overlay.PersistentNFSOverlay, name: persistent-nfs-overlay} + - {class: connections.serial.ConnectDevice, name: connect-device} + - class: power.ResetDevice + name: reset-device + pipeline: + - {class: power.PDUReboot, name: pdu-reboot} + - class: actions.deploy.download.DownloaderAction + name: download-retry + pipeline: + - {class: actions.deploy.download.HttpDownloadAction, name: http-download} + - {class: actions.deploy.environment.DeployDeviceEnvironment, name: deploy-device-env} + - class: actions.deploy.download.DownloaderAction + name: download-retry + pipeline: + - {class: actions.deploy.download.HttpDownloadAction, name: http-download} + - {class: actions.deploy.environment.DeployDeviceEnvironment, name: deploy-device-env} + - class: actions.deploy.download.DownloaderAction + name: download-retry + pipeline: + - {class: actions.deploy.download.HttpDownloadAction, name: http-download} + - {class: actions.deploy.apply_overlay.ApplyOverlaySparseImage, name: apply-overlay-sparse-image} + - {class: actions.deploy.environment.DeployDeviceEnvironment, name: deploy-device-env} + - class: actions.deploy.fastboot.FastbootFlashOrderAction + name: fastboot-flash-order-action + pipeline: + - {class: power.ReadFeedback, name: read-feedback} + - {class: actions.deploy.fastboot.FastbootFlashAction, name: fastboot-flash-action} + - {class: power.PDUReboot, name: pdu-reboot} + - {class: power.ReadFeedback, name: read-feedback} + - {class: actions.deploy.fastboot.FastbootFlashAction, name: fastboot-flash-action} + - {class: power.PDUReboot, name: pdu-reboot} + - {class: power.ReadFeedback, name: read-feedback} + - {class: actions.deploy.fastboot.FastbootFlashAction, name: fastboot-flash-action} +- class: actions.boot.grub.GrubSequenceAction + name: grub-sequence-action + pipeline: + - {class: actions.boot.fastboot.WaitFastBootInterrupt, name: wait-fastboot-interrupt} + - {class: actions.boot.AutoLoginAction, name: auto-login-action} + - {class: shell.ExpectShellSession, name: expect-shell-connection} + - {class: actions.boot.environment.ExportDeviceEnvironment, name: export-device-env} +- class: actions.test.shell.TestShellRetry + name: lava-test-retry + pipeline: + - {class: actions.test.shell.TestShellAction, name: lava-test-shell} +- class: actions.test.shell.TestShellRetry + name: lava-test-retry + pipeline: + - {class: actions.test.shell.TestShellAction, name: lava-test-shell} +- class: power.FinalizeAction + name: finalize + pipeline: + - {class: power.PowerOff, name: power-off} + - {class: power.ReadFeedback, name: read-feedback} diff --git a/lava_dispatcher/test/sample_jobs/hi6220-recovery.yaml b/lava_dispatcher/test/sample_jobs/hi6220-recovery.yaml new file mode 100644 index 000000000..970973915 --- /dev/null +++ b/lava_dispatcher/test/sample_jobs/hi6220-recovery.yaml @@ -0,0 +1,235 @@ +device_type: hi6220-hikey-bl +job_name: HiKey 6220 write to eMMC +timeouts: + job: + minutes: 60 + action: + minutes: 15 + connection: + minutes: 2 +priority: medium +visibility: public + +metadata: + source: https://git.linaro.org/lava-team/refactoring.git + path: hi6220-recovery.yaml + recovery-build: '55' + +protocols: + lava-lxc: + name: lxc-hikey-test + template: debian + distribution: debian + release: stretch + +actions: +- deploy: + namespace: tlxc + timeout: + minutes: 5 + to: lxc + packages: + - adb + - fastboot + - python + - python-serial + os: debian + +- boot: + namespace: tlxc + prompts: + - 'root@(.*):/#' + timeout: + minutes: 5 + method: lxc + +- deploy: + timeout: + minutes: 10 + to: recovery + namespace: recovery + connection: lxc + images: + script: + url: http://images.validation.linaro.org/snapshots.linaro.org/96boards/reference-platform/components/uefi-staging/55/hikey/release/hisi-idt.py + loader: + url: http://images.validation.linaro.org/snapshots.linaro.org/96boards/reference-platform/components/uefi-staging/55/hikey/release/l-loader.bin + ptable: + url: http://images.validation.linaro.org/snapshots.linaro.org/96boards/reference-platform/components/uefi-staging/55/hikey/release/ptable-linux-8g.img + fastboot: + url: http://images.validation.linaro.org/snapshots.linaro.org/96boards/reference-platform/components/uefi-staging/55/hikey/release/fip.bin + nvme: + url: http://images.validation.linaro.org/snapshots.linaro.org/96boards/reference-platform/components/uefi-staging/55/hikey/release/nvme.img + os: debian + +- boot: + namespace: recovery + timeout: + minutes: 5 + method: recovery + commands: recovery + +- test: + namespace: tlxc + connection: lxc + timeout: + minutes: 10 + definitions: + - repository: https://git.linaro.org/lava-team/refactoring.git/ + from: git + path: testdefs/hikey-6220-recovery.yaml + name: execute-recovery + +- boot: + namespace: recovery + timeout: + minutes: 5 + method: recovery + commands: exit + +- deploy: + timeout: + minutes: 15 + to: fastboot + namespace: droid + connection: lxc + images: + ptable: + url: http://images.validation.linaro.org/snapshots.linaro.org/96boards/reference-platform/components/uefi-staging/59/hikey/release/ptable-aosp-8g.img + reboot: hard-reset + boot: + url: http://images.validation.linaro.org/builds.96boards.org/snapshots/hikey/linaro/aosp-master/357/boot.img.xz + compression: xz + reboot: hard-reset + cache: + url: http://images.validation.linaro.org/builds.96boards.org/snapshots/hikey/linaro/aosp-master/357/cache.img.xz + compression: xz + userdata: + url: http://images.validation.linaro.org/builds.96boards.org/snapshots/hikey/linaro/aosp-master/357/userdata.img.xz + compression: xz + system: + url: http://images.validation.linaro.org/builds.96boards.org/snapshots/hikey/linaro/aosp-master/357/system.img.xz + compression: xz + os: debian + protocols: + lava-lxc: + - action: fastboot-deploy + request: pre-power-command + timeout: + minutes: 2 + +- boot: + namespace: droid + connection: droid + prompts: + - 'healthd: No battery devices found' + timeout: + minutes: 15 + method: grub + commands: installed + +- test: + namespace: tlxc + connection: lxc + timeout: + minutes: 10 + definitions: + - repository: https://git.linaro.org/lava-team/refactoring.git/ + from: git + path: android/lava-android-basic-lxc.yaml + name: v2-make-adb-connection + +- deploy: + timeout: + minutes: 45 + to: fastboot + # OE deployment + namespace: hikey + connection: lxc + images: + ptable: + url: http://images.validation.linaro.org/builds.96boards.org/snapshots/reference-platform/components/uefi-staging/49/hikey/release/ptable-linux-8g.img + reboot: hard-reset + boot: + url: http://images.validation.linaro.org/snapshots.linaro.org/openembedded/lkft/morty/hikey/rpb/linux-mainline/588/boot-0.0+AUTOINC+06e4def583-fb1158a365-r0-hikey-20180128213254-588.uefi.img + reboot: hard-reset + system: + url: http://images.validation.linaro.org/snapshots.linaro.org/openembedded/lkft/morty/hikey/rpb/linux-mainline/588/rpb-console-image-hikey-20180128213254-588.rootfs.img.gz + compression: gz + apply-overlay: true + os: oe + protocols: + lava-lxc: + - action: fastboot-deploy + request: pre-power-command + timeout: + minutes: 2 +- boot: + namespace: hikey + prompts: + - 'root@hikey:~#' + auto_login: + login_prompt: 'login:' + username: root + timeout: + minutes: 5 + method: grub + commands: installed + protocols: + lava-lxc: + - action: grub-sequence-action + request: pre-os-command + timeout: + minutes: 2 + +- test: + namespace: hikey + timeout: + minutes: 5 + definitions: + - repository: http://git.linaro.org/lava-team/lava-functional-tests.git + from: git + path: lava-test-shell/smoke-tests-basic.yaml + name: smoke-tests-basic-oe + - repository: + metadata: + format: Lava-Test Test Definition 1.0 + name: device-helper + description: "check helpers" + os: + - debian + scope: + - functional + run: + steps: + - lava-target-mac + - lava-target-ip + from: inline + name: device-helpers + path: inline/device-helpers.yaml + +- test: + namespace: tlxc + timeout: + minutes: 5 + definitions: + - repository: http://git.linaro.org/lava-team/lava-functional-tests.git + from: git + path: lava-test-shell/smoke-tests-basic.yaml + name: smoke-tests-basic-lxc + - repository: + metadata: + format: Lava-Test Test Definition 1.0 + name: device-helper + description: "check helpers" + os: + - debian + scope: + - functional + run: + steps: + - lava-target-mac + - lava-target-ip + from: inline + name: lxc-helpers + path: inline/lxc-helpers.yaml diff --git a/lava_dispatcher/test/test_recovery.py b/lava_dispatcher/test/test_recovery.py new file mode 100644 index 000000000..a2c0146f7 --- /dev/null +++ b/lava_dispatcher/test/test_recovery.py @@ -0,0 +1,101 @@ +# Copyright (C) 2018 Linaro Limited +# +# Author: Neil Williams <neil.williams@linaro.org> +# +# This file is part of LAVA. +# +# LAVA is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# LAVA is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along +# with this program; if not, see <http://www.gnu.org/licenses>. + + +import os +import yaml +import unittest +from lava_dispatcher.test.test_basic import Factory, StdoutTestCase +from lava_dispatcher.device import NewDevice +from lava_dispatcher.parser import JobParser +from lava_dispatcher.test.utils import DummyLogger, infrastructure_error_multi_paths + + +class FastBootFactory(Factory): # pylint: disable=too-few-public-methods + """ + Not Model based, this is not a Django factory. + Factory objects are dispatcher based classes, independent + of any database objects. + """ + + def create_hikey_bl_device(self, hostname): + """ + Create a device configuration on-the-fly from in-tree + device-type Jinja2 template. + """ + with open( + os.path.join( + os.path.dirname(__file__), + '..', '..', 'lava_scheduler_app', 'tests', + 'devices', 'hi6220-hikey-bl-01.jinja2')) as hikey: + data = hikey.read() + test_template = self.prepare_jinja_template(hostname, data) + rendered = test_template.render() + return (rendered, data) + + def create_hikey_bl_job(self, filename): + (data, device_dict) = self.create_hikey_bl_device('hi6220-hikey-01') + device = NewDevice(yaml.load(data)) + self.validate_data('hi6220-hikey-01', device_dict) + fastboot_yaml = os.path.join(os.path.dirname(__file__), filename) + with open(fastboot_yaml) as sample_job_data: + parser = JobParser() + job = parser.parse(sample_job_data, device, 4212, None, "") + job.logger = DummyLogger() + return job + + +class TestRecoveryMode(StdoutTestCase): # pylint: disable=too-many-public-methods + + def setUp(self): + super().setUp() + self.factory = FastBootFactory() + self.job = self.factory.create_hikey_bl_job('sample_jobs/hi6220-recovery.yaml') + + @unittest.skipIf(infrastructure_error_multi_paths( + ['lxc-info', 'img2simg', 'simg2img']), + "lxc or img2simg or simg2img not installed") + def test_structure(self): + self.assertIsNotNone(self.job) + self.job.validate() + + description_ref = self.pipeline_reference('hi6220-recovery.yaml', job=self.job) + self.assertEqual(description_ref, self.job.pipeline.describe(False)) + + def test_commands(self): + enter = [action for action in self.job.pipeline.actions if action.name == 'recovery-boot'][0] + mode = [action for action in enter.internal_pipeline.actions if action.name == 'switch-recovery'][0] + recovery = self.job.device['actions']['deploy']['methods']['recovery'] + self.assertIsNotNone(recovery['commands'].get(mode.mode, None)) + self.assertEqual( + [ + '/home/neil/lava-lab/shared/lab-scripts/eth008_control -a 10.15.0.171 -r 1 -s off', + '/home/neil/lava-lab/shared/lab-scripts/eth008_control -a 10.15.0.171 -r 2 -s on'], + recovery['commands'][mode.mode]) + self.assertEqual('recovery_mode', mode.mode) + exit_mode = [action for action in self.job.pipeline.actions if action.name == 'recovery-boot'][1] + mode = [action for action in exit_mode.internal_pipeline.actions if action.name == 'switch-recovery'][0] + self.assertIsNotNone(recovery['commands'].get(mode.mode, None)) + self.assertEqual( + [ + '/home/neil/lava-lab/shared/lab-scripts/eth008_control -a 10.15.0.171 -r 1 -s on', + '/home/neil/lava-lab/shared/lab-scripts/eth008_control -a 10.15.0.171 -r 2 -s off'], + recovery['commands'][mode.mode]) + self.assertEqual('recovery_exit', mode.mode) diff --git a/lava_dispatcher/utils/udev.py b/lava_dispatcher/utils/udev.py index 615b0ea3e..4128e0497 100644 --- a/lava_dispatcher/utils/udev.py +++ b/lava_dispatcher/utils/udev.py @@ -341,7 +341,7 @@ def allow_fs_label(device): # will require a filesystem label to identify a device. # So far, mps devices are supported, but these don't provide a # unique serial, so fs label must be used. - fs_label_methods = ['mps'] + fs_label_methods = ['mps', 'recovery'] # Don't allow using filesystem labels by default as they are # unreliable, and can be changed via a malicious job. diff --git a/lava_scheduler_app/tests/device-types/base-fastboot.jinja2 b/lava_scheduler_app/tests/device-types/base-fastboot.jinja2 index 03e4b5cbd..f4c71154f 100644 --- a/lava_scheduler_app/tests/device-types/base-fastboot.jinja2 +++ b/lava_scheduler_app/tests/device-types/base-fastboot.jinja2 @@ -28,6 +28,17 @@ actions: port: {{ ssh_port|default(22) }} user: {{ ssh_user|default('root') }} identity_file: {{ ssh_identity_file }} +{% if recovery_mode %} +{{ recovery_mode }} + recovery_mode: +{% for url in recovery_mode_command %} + - {{ url }} +{% endfor %} + recovery_exit: +{% for url in recovery_exit_command %} + - {{ url }} +{% endfor %} +{% endif %} fastboot: {{- fastboot_deploy_uboot }} {{- fastboot_interrupt_params }} diff --git a/lava_scheduler_app/tests/device-types/hi6220-hikey-bl.jinja2 b/lava_scheduler_app/tests/device-types/hi6220-hikey-bl.jinja2 new file mode 100644 index 000000000..ddc363462 --- /dev/null +++ b/lava_scheduler_app/tests/device-types/hi6220-hikey-bl.jinja2 @@ -0,0 +1,29 @@ +{% extends 'base-fastboot.jinja2' %} +{% set boot_character_delay = 10 %} +{% set root_device = root_device | default('/dev/mmcblk0p9') %} +{% set base_kernel_args = base_kernel_args|default('') %} +{% set console_device = console_device|default('ttyAMA3') %} +{% set baud_rate = baud_rate|default('115200') %} +{% set fastboot_sequence = ['boot'] %} +{% set recovery_mode_command = recovery_mode_command|default('') %} +{% set fastboot_only_command = fastboot_only_command|default('') %} +{# set device_type = "hi6220-hikey-bl - based on r2 based on 960" #} +{% set fastboot_interrupt_params = " + interrupt_prompt: 'Android Fastboot mode' + interrupt_string: ' '"%} +{% set flash_cmds_order = ['ptable', 'xloader', 'fastboot', 'nvme', 'fw_lpm3', +'trustfirmware', 'boot', 'dts', 'system', 'userdata', 'cache'] %} +{% set fastboot_boot_grub = " + grub: + reset_device: False + sequence: + - wait-fastboot-interrupt + installed: + commands: + - boot +"%} +{# Different device-types will have different types and numbers of commands. #} +{% set recovery_mode = " + recovery: + commands: +"%} diff --git a/lava_scheduler_app/tests/devices/hi6220-hikey-bl-01.jinja2 b/lava_scheduler_app/tests/devices/hi6220-hikey-bl-01.jinja2 new file mode 100644 index 000000000..c86554907 --- /dev/null +++ b/lava_scheduler_app/tests/devices/hi6220-hikey-bl-01.jinja2 @@ -0,0 +1,31 @@ +{% extends 'hi6220-hikey-bl.jinja2' %} +{% set fastboot_options = ['-S', '256M', '-u'] %} +{% set hard_reset_command = ['/usr/bin/pduclient --daemon calvin --hostname pdu --command off --port 04', +'sleep 30', +'/usr/bin/pduclient --daemon calvin --hostname pdu --command on --port 04'] %} +{% set pre_os_command = '/home/neil/lava-lab/shared/lab-scripts/usb_hub_control -u 9 -p 4000 -m off' %} +{% set pre_power_command = '/home/neil/lava-lab/shared/lab-scripts/usb_hub_control -u 9 -p 4000 -m sync' %} +{% set interrupt_prompt = 'Android Fastboot mode' %} +{% set interrupt_string = "' '" %} +{% set soft_reset_command = 'fastboot -u -s 4F7EC2290037CD2B reboot' %} +{% set power_off_command = ['/usr/bin/pduclient --daemon calvin --hostname pdu --command off --port 04', +'sleep 30', +'/home/neil/lava-lab/shared/lab-scripts/eth008_control -a 10.15.0.171 -r 1 -s on', +'/home/neil/lava-lab/shared/lab-scripts/eth008_control -a 10.15.0.171 -r 2 -s off'] %} +{% set fastboot_serial_number = '4F7EC2290037CD2B' %} +{% set device_info = [{'board_id': '4F7EC2290037CD2B'}, {'parent': True, 'usb_vendor_id': '12d1', 'usb_product_id': '3609'}] %} +{# set device_info = [{'board_id': '4F7EC2290037CD2B'}] #} +{% set adb_serial_number = '4F7EC2290037CD2B' %} +{% set power_on_command = '/usr/bin/pduclient --daemon calvin --hostname pdu --command on --port 04' %} +{% set connection_commands = {'uart1': 'telnet azrael 4100'} %} +{% set connection_list = ['uart1'] %} +{% set connection_tags = {'uart1': ['primary', 'telnet']} %} +{% set flash_cmds_order = ['ptable', 'fastboot', 'nvme', 'boot', 'cache', 'system', 'userdata'] %} +{% set device_mac = '8a:ce:4c:ff:aa:bb' %} +{% set recovery_mode_command = [ +'/home/neil/lava-lab/shared/lab-scripts/eth008_control -a 10.15.0.171 -r 1 -s off', +'/home/neil/lava-lab/shared/lab-scripts/eth008_control -a 10.15.0.171 -r 2 -s on'] %} +{% set recovery_exit_command = [ +'/home/neil/lava-lab/shared/lab-scripts/eth008_control -a 10.15.0.171 -r 1 -s on', +'/home/neil/lava-lab/shared/lab-scripts/eth008_control -a 10.15.0.171 -r 2 -s off'] %} +{% set action_timeout_power_off = 60 %} diff --git a/lava_scheduler_app/tests/test_templates.py b/lava_scheduler_app/tests/test_templates.py index e629afa6a..40722defd 100644 --- a/lava_scheduler_app/tests/test_templates.py +++ b/lava_scheduler_app/tests/test_templates.py @@ -1434,6 +1434,29 @@ class TestTemplates(unittest.TestCase): self.assertNotIn('115200n8', command) self.assertNotIn('n8', command) + def test_recovery_mode(self): + with open(os.path.join(os.path.dirname(__file__), 'devices', 'hi6220-hikey-bl-01.jinja2')) as hikey: + data = hikey.read() + self.assertTrue(self.validate_data('hi620-bl-01', data)) + test_template = prepare_jinja_template('hi620-bl-01', data) + rendered = test_template.render() + template_dict = yaml.load(rendered) + recovery = template_dict['actions']['deploy']['methods'] + self.assertIsNotNone('recovery', recovery) + self.assertIn('recovery', recovery) + self.assertIn('commands', recovery['recovery']) + self.assertIsNotNone('recovery', recovery['recovery']['commands']) + self.assertIn('recovery_mode', recovery['recovery']['commands']) + self.assertEqual( + ['/home/neil/lava-lab/shared/lab-scripts/eth008_control -a 10.15.0.171 -r 1 -s off', + '/home/neil/lava-lab/shared/lab-scripts/eth008_control -a 10.15.0.171 -r 2 -s on'], + recovery['recovery']['commands']['recovery_mode']) + self.assertIn('recovery_exit', recovery['recovery']['commands']) + self.assertEqual( + ['/home/neil/lava-lab/shared/lab-scripts/eth008_control -a 10.15.0.171 -r 1 -s on', + '/home/neil/lava-lab/shared/lab-scripts/eth008_control -a 10.15.0.171 -r 2 -s off'], + recovery['recovery']['commands']['recovery_exit']) + def test_flasher(self): data = """{% extends 'b2260.jinja2' %} {% set flasher_deploy_commands = ['flashing', 'something --else'] %} diff --git a/lava_scheduler_app/utils.py b/lava_scheduler_app/utils.py index 3e9b7491e..4ccc9f2bf 100644 --- a/lava_scheduler_app/utils.py +++ b/lava_scheduler_app/utils.py @@ -354,6 +354,8 @@ def device_dictionary_sequence(): 'fastboot_serial_number', 'device_info', 'static_info', + 'recovery_mode_command', + 'recovery_exit_command', ] diff --git a/share/lava_lxc_device_add.py b/share/lava_lxc_device_add.py index ae34b44e7..9aa667565 100755 --- a/share/lava_lxc_device_add.py +++ b/share/lava_lxc_device_add.py @@ -110,6 +110,11 @@ def main(): except subprocess.CalledProcessError as exc: logger.error("[%s] failed to add device %s: '%s'", uniq_str, device, exc) + logger.close(linger=LINGER) # pylint: disable=no-member + return 2 + except: # pylint: disable=bare-except + logger.close(linger=LINGER) # pylint: disable=no-member + return 3 logger.close(linger=LINGER) # pylint: disable=no-member return 0 |