Remove v1 code

Change-Id: I098edd9edfeb968b303eaedc6afb6654e43b4b98

5 files changed, 1152 insertions, 0 deletions

diff --git a/lava_dispatcher/actions/test/__init__.py b/lava_dispatcher/actions/test/__init__.py
new file mode 100644
index 000000000..da14241d2
--- /dev/null
+++ b/lava_dispatcher/actions/test/__init__.py
@@ -0,0 +1,60 @@
+# Copyright (C) 2014 Linaro Limited
+#
+# Author: Neil Williams <neil.williams@linaro.org>
+#
+# This file is part of LAVA Dispatcher.
+#
+# LAVA Dispatcher is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# LAVA Dispatcher is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along
+# with this program; if not, see <http://www.gnu.org/licenses>.
+
+from lava_dispatcher.action import (
+    Action,
+    JobError,
+)
+from lava_dispatcher.logical import (
+    LavaTest,
+    RetryAction,
+)
+
+
+def handle_testcase(params):
+
+    # FIXME: move to utils
+    data = {}
+    for param in params:
+        parts = param.split('=')
+        if len(parts) == 2:
+            key, value = parts
+            key = key.lower()
+            data[key] = value
+        else:
+            raise JobError(
+                "Ignoring malformed parameter for signal: \"%s\". " % param)
+    return data
+
+
+class TestAction(Action):
+    """
+    Base class for all actions which run lava test
+    cases on a device under test.
+    The subclass selected to do the work will be the
+    subclass returning True in the accepts(device, image)
+    function.
+    Each new subclass needs a unit test to ensure it is
+    reliably selected for the correct deployment and not
+    selected for an invalid deployment or a deployment
+    accepted by a different subclass.
+    """
+
+    name = 'test'
diff --git a/lava_dispatcher/actions/test/monitor.py b/lava_dispatcher/actions/test/monitor.py
new file mode 100644
index 000000000..6734cdcd7
--- /dev/null
+++ b/lava_dispatcher/actions/test/monitor.py
@@ -0,0 +1,200 @@
+# Copyright (C) 2014 Linaro Limited
+#
+# Author: Tyler Baker <tyler.baker@linaro.org>
+#
+# This file is part of LAVA Dispatcher.
+#
+# LAVA Dispatcher is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# LAVA Dispatcher is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along
+# with this program; if not, see <http://www.gnu.org/licenses>.
+
+import re
+import pexpect
+
+from collections import OrderedDict
+from lava_dispatcher.action import (
+    InfrastructureError,
+    LAVABug,
+    Pipeline,
+)
+from lava_dispatcher.actions.test import (
+    TestAction,
+)
+from lava_dispatcher.logical import (
+    LavaTest,
+    RetryAction,
+)
+
+
+class TestMonitor(LavaTest):
+    """
+    LavaTestMonitor Strategy object
+    """
+    def __init__(self, parent, parameters):
+        super(TestMonitor, self).__init__(parent)
+        self.action = TestMonitorRetry()
+        self.action.job = self.job
+        self.action.section = self.action_type
+        parent.add_action(self.action, parameters)
+
+    @classmethod
+    def accepts(cls, device, parameters):
+        # TODO: Add configurable timeouts
+        required_parms = ['name', 'start',
+                          'end', 'pattern']
+        if 'monitors' in parameters:
+            for monitor in parameters['monitors']:
+                if all([x for x in required_parms if x in monitor]):
+                    return True, 'accepted'
+            return False, 'missing a required parameter from %s' % required_parms
+        else:
+            return False, '"monitors" not in parameters'
+
+    @classmethod
+    def needs_deployment_data(cls):
+        return False
+
+    @classmethod
+    def needs_overlay(cls):
+        return False
+
+    @classmethod
+    def has_shell(cls):
+        return False
+
+
+class TestMonitorRetry(RetryAction):
+
+    def __init__(self):
+        super(TestMonitorRetry, self).__init__()
+        self.description = "Retry wrapper for lava-test-monitor"
+        self.summary = "Retry support for Lava Test Monitoring"
+        self.name = "lava-test-monitor-retry"
+
+    def populate(self, parameters):
+        self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
+        self.internal_pipeline.add_action(TestMonitorAction())
+
+
+class TestMonitorAction(TestAction):  # pylint: disable=too-many-instance-attributes
+    """
+    Sets up and runs the LAVA Test Shell Definition scripts.
+    Supports a pre-command-list of operations necessary on the
+    booted image before the test shell can be started.
+    """
+
+    def __init__(self):
+        super(TestMonitorAction, self).__init__()
+        self.description = "Executing lava-test-monitor"
+        self.summary = "Lava Test Monitor"
+        self.name = "lava-test-monitor"
+        self.test_suite_name = None
+        self.report = {}
+        self.fixupdict = {}
+        self.patterns = {}
+
+    def run(self, connection, max_end_time, args=None):
+        connection = super(TestMonitorAction, self).run(connection, max_end_time, args)
+
+        if not connection:
+            raise InfrastructureError("Connection closed")
+        for monitor in self.parameters['monitors']:
+            self.test_suite_name = monitor['name']
+
+            self.fixupdict = monitor.get('fixupdict')
+
+            # pattern order is important because we want to match the end before
+            # it can possibly get confused with a test result
+            self.patterns = OrderedDict()
+            self.patterns["eof"] = pexpect.EOF
+            self.patterns["timeout"] = pexpect.TIMEOUT
+            self.patterns["end"] = monitor['end']
+            self.patterns["test_result"] = monitor['pattern']
+
+            # Find the start string before parsing any output.
+            connection.prompt_str = monitor['start']
+            connection.wait()
+            self.logger.info("ok: start string found, lava test monitoring started")
+
+            with connection.test_connection() as test_connection:
+                while self._keep_running(test_connection, timeout=test_connection.timeout):
+                    pass
+
+        return connection
+
+    def _keep_running(self, test_connection, timeout=120):
+        self.logger.debug("test monitoring timeout: %d seconds", timeout)
+        retval = test_connection.expect(list(self.patterns.values()), timeout=timeout)
+        return self.check_patterns(list(self.patterns.keys())[retval], test_connection)
+
+    def check_patterns(self, event, test_connection):  # pylint: disable=too-many-branches
+        """
+        Defines the base set of pattern responses.
+        Stores the results of testcases inside the TestAction
+        Call from subclasses before checking subclass-specific events.
+        """
+        ret_val = False
+        if event == "end":
+            self.logger.info("ok: end string found, lava test monitoring stopped")
+            self.results.update({'status': 'passed'})
+        elif event == "timeout":
+            self.logger.warning("err: lava test monitoring has timed out")
+            self.errors = "lava test monitoring has timed out"
+            self.results.update({'status': 'failed'})
+        elif event == "test_result":
+            self.logger.info("ok: test case found")
+            match = test_connection.match.groupdict()
+            if 'result' in match:
+                if self.fixupdict:
+                    if match['result'] in self.fixupdict:
+                        match['result'] = self.fixupdict[match['result']]
+                if match['result'] not in ('pass', 'fail', 'skip', 'unknown'):
+                    self.logger.error("error: bad test results: %s", match['result'])
+                else:
+                    if 'test_case_id' in match:
+                        case_id = match['test_case_id'].strip().lower()
+                        # remove special characters to form a valid test case id
+                        case_id = re.sub(r'\W+', '_', case_id)
+                        self.logger.debug('test_case_id: %s', case_id)
+                        results = {
+                            'definition': self.test_suite_name.replace(' ', '-').lower(),
+                            'case': case_id,
+                            'level': self.level,
+                            'result': match['result'],
+                            'extra': {'test_case_id': match['test_case_id'].strip()}
+                        }
+                        if 'measurement' in match:
+                            results.update({'measurement': match['measurement']})
+                        if 'units' in match:
+                            results.update({'units': match['units']})
+                        self.logger.results(results)  # pylint: disable=no-member
+            else:
+                if all(x in match for x in ['test_case_id', 'measurement']):
+                    if match['measurement'] and match['test_case_id']:
+                        case_id = match['test_case_id'].strip().lower()
+                        # remove special characters to form a valid test case id
+                        case_id = re.sub(r'\W+', '_', case_id)
+                        self.logger.debug('test_case_id: %s', case_id)
+                        results = {
+                            'definition': self.test_suite_name.replace(' ', '-').lower(),
+                            'case': case_id,
+                            'level': self.level,
+                            'result': 'pass',
+                            'measurement': float(match['measurement']),
+                            'extra': {'test_case_id': match['test_case_id'].strip()}
+                        }
+                        if 'units' in match:
+                            results.update({'units': match['units']})
+                        self.logger.results(results)  # pylint: disable=no-member
+            ret_val = True
+        return ret_val
diff --git a/lava_dispatcher/actions/test/multinode.py b/lava_dispatcher/actions/test/multinode.py
new file mode 100644
index 000000000..3a02cce54
--- /dev/null
+++ b/lava_dispatcher/actions/test/multinode.py
@@ -0,0 +1,218 @@
+# Copyright (C) 2014 Linaro Limited
+#
+# Author: Neil Williams <neil.williams@linaro.org>
+#
+# This file is part of LAVA Dispatcher.
+#
+# LAVA Dispatcher is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# LAVA Dispatcher is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along
+# with this program; if not, see <http://www.gnu.org/licenses>.
+
+import json
+from lava_dispatcher.actions.test.shell import TestShellAction
+from lava_dispatcher.action import (
+    TestError,
+    Timeout,
+    MultinodeProtocolTimeoutError
+)
+from lava_dispatcher.actions.test import LavaTest
+from lava_dispatcher.protocols.multinode import MultinodeProtocol
+
+
+class MultinodeTestShell(LavaTest):
+    """
+    LavaTestShell Strategy object for Multinode
+    """
+    # higher priority than the plain TestShell
+    priority = 2
+
+    def __init__(self, parent, parameters):
+        super(MultinodeTestShell, self).__init__(parent)
+        self.action = MultinodeTestAction()
+        self.action.job = self.job
+        self.action.section = self.action_type
+        parent.add_action(self.action, parameters)
+
+    @classmethod
+    def accepts(cls, device, parameters):  # pylint: disable=unused-argument
+        if 'role' in parameters:
+            if MultinodeProtocol.name in parameters:
+                if 'target_group' in parameters[MultinodeProtocol.name]:
+                    return True, 'accepted'
+                else:
+                    return False, '"target_group" was not in parameters for %s' % MultinodeProtocol.name
+            else:
+                return False, '%s was not in parameters' % MultinodeProtocol.name
+        return False, '"role" not in parameters'
+
+    @classmethod
+    def needs_deployment_data(cls):
+        return True
+
+    @classmethod
+    def needs_overlay(cls):
+        return True
+
+    @classmethod
+    def has_shell(cls):
+        return True
+
+
+class MultinodeTestAction(TestShellAction):
+
+    def __init__(self):
+        super(MultinodeTestAction, self).__init__()
+        self.name = "multinode-test"
+        self.description = "Executing lava-test-runner"
+        self.summary = "Multinode Lava Test Shell"
+        self.multinode_dict = {
+            'multinode': r'<LAVA_MULTI_NODE> <LAVA_(\S+) ([^>]+)>',
+        }
+
+    def validate(self):
+        super(MultinodeTestAction, self).validate()
+        # MultinodeProtocol is required, others can be optional
+        if MultinodeProtocol.name not in [protocol.name for protocol in self.job.protocols]:
+            self.errors = "Invalid job - missing protocol"
+        if MultinodeProtocol.name not in [protocol.name for protocol in self.protocols]:
+            self.errors = "Missing protocol"
+        if not self.valid:
+            self.errors = "Invalid base class TestAction"
+            return
+        self.patterns.update(self.multinode_dict)
+        self.signal_director.setup(self.parameters)
+
+    def _reset_patterns(self):
+        super(MultinodeTestAction, self)._reset_patterns()
+        self.patterns.update(self.multinode_dict)
+
+    def populate(self, parameters):
+        """
+        Select the appropriate protocol supported by this action from the list available from the job
+        """
+        self.protocols = [protocol for protocol in self.job.protocols if protocol.name == MultinodeProtocol.name]
+        self.signal_director = self.SignalDirector(self.protocols[0])
+
+    def check_patterns(self, event, test_connection, check_char):
+        """
+        Calls the parent check_patterns first, then checks for subclass pattern.
+        """
+        ret = super(MultinodeTestAction, self).check_patterns(event, test_connection, check_char)
+        if event == 'multinode':
+            name, params = test_connection.match.groups()
+            self.logger.debug("Received Multi_Node API <LAVA_%s>" % name)
+            params = params.split()
+            test_case_name = "%s-%s" % (name, params[0])  # use the messageID
+            self.logger.debug("messageID: %s", test_case_name)
+
+            test_case_params = "TEST_CASE_ID=multinode-{} RESULT={}"
+            try:
+                ret = self.signal_director.signal(name, params)
+            except MultinodeProtocolTimeoutError as exc:
+                self.logger.warning("Sync error in %s signal: %s %s" % (event, exc, name))
+
+                self.signal_test_case(
+                    test_case_params.format(test_case_name.lower(), "fail").split())
+
+                return False
+
+            self.signal_test_case(
+                test_case_params.format(test_case_name.lower(), "pass").split())
+            return ret
+        return ret
+
+    class SignalDirector(TestShellAction.SignalDirector):
+
+        def __init__(self, protocol):
+            super(MultinodeTestAction.SignalDirector, self).__init__(protocol)
+            self.base_message = {}
+
+        def setup(self, parameters):
+            """
+            Retrieve the poll_timeout from the protocol parameters which are set after init.
+            """
+            if MultinodeProtocol.name not in parameters:
+                return
+            if 'timeout' in parameters[MultinodeProtocol.name]:
+                self.base_message = {
+                    'timeout': Timeout.parse(parameters[MultinodeProtocol.name]['timeout'])
+                }
+
+        def _on_send(self, *args):
+            self.logger.debug("%s lava-send" % MultinodeProtocol.name)
+            arg_length = len(args)
+            if arg_length == 1:
+                msg = {"request": "lava_send", "messageID": args[0], "message": {}}
+            else:
+                message_id = args[0]
+                remainder = args[1:arg_length]
+                self.logger.debug("%d key value pair(s) to be sent." % len(remainder))
+                data = {}
+                for message in remainder:
+                    detail = str.split(message, "=")
+                    if len(detail) == 2:
+                        data[detail[0]] = detail[1]
+                msg = {"request": "lava_send", "messageID": message_id, "message": data}
+
+            msg.update(self.base_message)
+            self.logger.debug(str("Handling signal <LAVA_SEND %s>" % json.dumps(msg)))
+            reply = self.protocol(msg)
+            if reply == "nack":
+                # FIXME: does this deserve an automatic retry? Does it actually happen?
+                raise TestError("Coordinator was unable to accept LAVA_SEND")
+
+        def _on_sync(self, message_id):
+            self.logger.debug("Handling signal <LAVA_SYNC %s>" % message_id)
+            msg = {"request": "lava_sync", "messageID": message_id, "message": None}
+            msg.update(self.base_message)
+            reply = self.protocol(msg)
+            if reply == "nack":
+                message_str = " nack"
+            else:
+                message_str = ""
+            self.connection.sendline("<LAVA_SYNC_COMPLETE%s>" % message_str)
+            self.connection.sendline('\n')
+
+        def _on_wait(self, message_id):
+            self.logger.debug("Handling signal <LAVA_WAIT %s>" % message_id)
+            msg = {"request": "lava_wait", "messageID": message_id, "message": None}
+            msg.update(self.base_message)
+            reply = self.protocol(msg)
+            self.logger.debug("reply=%s" % reply)
+            message_str = ""
+            if reply == "nack":
+                message_str = " nack"
+            else:
+                for target, messages in reply.items():
+                    for key, value in messages.items():
+                        message_str += " %s:%s=%s" % (target, key, value)
+            self.connection.sendline("<LAVA_WAIT_COMPLETE%s>" % message_str)
+            self.connection.sendline('\n')
+
+        def _on_wait_all(self, message_id, role=None):
+            self.logger.debug("Handling signal <LAVA_WAIT_ALL %s>" % message_id)
+            msg = {"request": "lava_wait_all", "messageID": message_id, "role": role}
+            msg.update(self.base_message)
+            reply = self.protocol(msg)
+            message_str = ""
+            if reply == "nack":
+                message_str = " nack"
+            else:
+                # the reply format is like this :
+                # "{target:{key1:value, key2:value2, key3:value3},
+                #  target2:{key1:value, key2:value2, key3:value3}}"
+                for target, messages in reply.items():
+                    for key, value in messages.items():
+                        message_str += " %s:%s=%s" % (target, key, value)
+            self.connection.sendline("<LAVA_WAIT_ALL_COMPLETE%s>" % message_str)
+            self.connection.sendline('\n')
diff --git a/lava_dispatcher/actions/test/shell.py b/lava_dispatcher/actions/test/shell.py
new file mode 100644
index 000000000..ce0c3fef3
--- /dev/null
+++ b/lava_dispatcher/actions/test/shell.py
@@ -0,0 +1,646 @@
+# Copyright (C) 2014 Linaro Limited
+#
+# Author: Neil Williams <neil.williams@linaro.org>
+#
+# This file is part of LAVA Dispatcher.
+#
+# LAVA Dispatcher is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# LAVA Dispatcher is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along
+# with this program; if not, see <http://www.gnu.org/licenses>.
+
+import re
+import sys
+import time
+import yaml
+import decimal
+import logging
+import pexpect
+from nose.tools import nottest
+from collections import OrderedDict
+
+from lava_dispatcher.actions.test import (
+    TestAction,
+    handle_testcase
+)
+from lava_dispatcher.action import (
+    Pipeline,
+    JobError,
+    InfrastructureError,
+    TestError,
+    LAVABug,
+)
+from lava_dispatcher.logical import (
+    LavaTest,
+    RetryAction
+)
+from lava_dispatcher.connection import (
+    BaseSignalHandler,
+    SignalMatch
+)
+from lava_dispatcher.protocols.lxc import LxcProtocol
+from lava_dispatcher.utils.constants import (
+    DEFAULT_V1_PATTERN,
+    DEFAULT_V1_FIXUP,
+)
+if sys.version > '3':
+    from functools import reduce  # pylint: disable=redefined-builtin
+
+# pylint: disable=too-many-branches,too-many-statements,too-many-instance-attributes,logging-not-lazy
+
+
+class TestShell(LavaTest):
+    """
+    LavaTestShell Strategy object
+    """
+    def __init__(self, parent, parameters):
+        super(TestShell, self).__init__(parent)
+        self.action = TestShellRetry()
+        self.action.job = self.job
+        self.action.section = self.action_type
+        parent.add_action(self.action, parameters)
+
+    @classmethod
+    def accepts(cls, device, parameters):  # pylint: disable=unused-argument
+        if ('definition' in parameters) or ('definitions' in parameters):
+            return True, 'accepted'
+        return False, '"definition" or "definitions" not in parameters'
+
+    @classmethod
+    def needs_deployment_data(cls):
+        return True
+
+    @classmethod
+    def needs_overlay(cls):
+        return True
+
+    @classmethod
+    def has_shell(cls):
+        return True
+
+
+class TestShellRetry(RetryAction):
+
+    def __init__(self):
+        super(TestShellRetry, self).__init__()
+        self.description = "Retry wrapper for lava-test-shell"
+        self.summary = "Retry support for Lava Test Shell"
+        self.name = "lava-test-retry"
+
+    def populate(self, parameters):
+        self.internal_pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
+        self.internal_pipeline.add_action(TestShellAction())
+
+
+# FIXME: move to utils and call inside the overlay
+class PatternFixup(object):
+
+    def __init__(self, testdef, count):
+        """
+        Like all good arrays, the count is expected to start at zero.
+        Avoid calling from validate() or populate() - this needs the
+        RepoAction to be running.
+        """
+        super(PatternFixup, self).__init__()
+        self.pat = DEFAULT_V1_PATTERN
+        self.fixup = DEFAULT_V1_FIXUP
+        if isinstance(testdef, dict) and 'metadata' in testdef:
+            self.testdef = testdef
+            self.name = "%d_%s" % (count, reduce(dict.get, ['metadata', 'name'], testdef))
+        else:
+            self.testdef = {}
+            self.name = None
+
+    def valid(self):
+        return self.fixupdict() and self.pattern() and self.name
+
+    def update(self, pattern, fixupdict):
+        if not isinstance(pattern, str):
+            raise TestError("Unrecognised test parse pattern type: %s" % type(pattern))
+        try:
+            self.pat = re.compile(pattern, re.M)
+        except re.error as exc:
+            raise TestError("Error parsing regular expression %r: %s" % (self.pat, exc.message))
+        self.fixup = fixupdict
+
+    def fixupdict(self):
+        if 'parse' in self.testdef and 'fixupdict' in self.testdef['parse']:
+            self.fixup = self.testdef['parse']['fixupdict']
+        return self.fixup
+
+    def pattern(self):
+        if 'parse' in self.testdef and 'pattern' in self.testdef['parse']:
+            self.pat = self.testdef['parse']['pattern']
+            if not isinstance(self.pat, str):
+                raise TestError("Unrecognised test parse pattern type: %s" % type(self.pat))
+            try:
+                self.pat = re.compile(self.pat, re.M)
+            except re.error as exc:
+                raise TestError("Error parsing regular expression %r: %s" % (self.pat, exc.message))
+        return self.pat
+
+
+class TestShellAction(TestAction):
+    """
+    Sets up and runs the LAVA Test Shell Definition scripts.
+    Supports a pre-command-list of operations necessary on the
+    booted image before the test shell can be started.
+    """
+
+    def __init__(self):
+        super(TestShellAction, self).__init__()
+        self.description = "Executing lava-test-runner"
+        self.summary = "Lava Test Shell"
+        self.name = "lava-test-shell"
+        self.signal_director = self.SignalDirector(None)  # no default protocol
+        self.patterns = {}
+        self.signal_match = SignalMatch()
+        self.definition = None
+        self.testset_name = None
+        self.report = {}
+        self.start = None
+        self.testdef_dict = {}
+        # noinspection PyTypeChecker
+        self.pattern = PatternFixup(testdef=None, count=0)
+        self.current_run = None
+
+    def _reset_patterns(self):
+        # Extend the list of patterns when creating subclasses.
+        self.patterns = {
+            "exit": "<LAVA_TEST_RUNNER>: exiting",
+            "error": "<LAVA_TEST_RUNNER>: ([^ ]+) installer failed, skipping",
+            "eof": pexpect.EOF,
+            "timeout": pexpect.TIMEOUT,
+            "signal": r"<LAVA_SIGNAL_(\S+) ([^>]+)>",
+        }
+        # noinspection PyTypeChecker
+        self.pattern = PatternFixup(testdef=None, count=0)
+
+    def validate(self):
+        if "definitions" in self.parameters:
+            for testdef in self.parameters["definitions"]:
+                if "repository" not in testdef:
+                    self.errors = "Repository missing from test definition"
+        self._reset_patterns()
+        super(TestShellAction, self).validate()
+
+    def run(self, connection, max_end_time, args=None):  # pylint: disable=too-many-locals
+        """
+        Common run function for subclasses which define custom patterns
+        """
+        super(TestShellAction, self).run(connection, max_end_time, args)
+
+        # Get the connection, specific to this namespace
+        connection_namespace = self.parameters.get('connection-namespace', None)
+        parameters = None
+        if connection_namespace:
+            self.logger.debug("Using connection namespace: %s", connection_namespace)
+            parameters = {"namespace": connection_namespace}
+        else:
+            parameters = {'namespace': self.parameters.get('namespace', 'common')}
+            self.logger.debug("Using namespace: %s", parameters['namespace'])
+        connection = self.get_namespace_data(
+            action='shared', label='shared', key='connection', deepcopy=False, parameters=parameters)
+
+        if not connection:
+            raise LAVABug("No connection retrieved from namespace data")
+
+        self.signal_director.connection = connection
+
+        pattern_dict = {self.pattern.name: self.pattern}
+        # pattern dictionary is the lookup from the STARTRUN to the parse pattern.
+        self.set_namespace_data(action=self.name, label=self.name, key='pattern_dictionary', value=pattern_dict)
+        if self.character_delay > 0:
+            self.logger.debug("Using a character delay of %i (ms)", self.character_delay)
+
+        if not connection.prompt_str:
+            connection.prompt_str = [self.job.device.get_constant(
+                'default-shell-prompt')]
+            # FIXME: This should be logged whenever prompt_str is changed, by the connection object.
+            self.logger.debug("Setting default test shell prompt %s", connection.prompt_str)
+        connection.timeout = self.connection_timeout
+        # force an initial prompt - not all shells will respond without an excuse.
+        connection.sendline(connection.check_char)
+        self.wait(connection)
+
+        # use the string instead of self.name so that inheriting classes (like multinode)
+        # still pick up the correct command.
+        running = self.parameters['stage']
+        pre_command_list = self.get_namespace_data(action='test', label="lava-test-shell", key='pre-command-list')
+        lava_test_results_dir = self.get_namespace_data(
+            action='test', label='results', key='lava_test_results_dir')
+        lava_test_sh_cmd = self.get_namespace_data(action='test', label='shared', key='lava_test_sh_cmd')
+
+        if pre_command_list and running == 0:
+            for command in pre_command_list:
+                connection.sendline(command, delay=self.character_delay)
+
+        if lava_test_results_dir is None:
+            raise JobError("Nothing to run. Maybe the 'deploy' stage is missing, "
+                           "otherwise this is a bug which should be reported.")
+
+        self.logger.debug("Using %s" % lava_test_results_dir)
+        connection.sendline('ls -l %s/' % lava_test_results_dir, delay=self.character_delay)
+        if lava_test_sh_cmd:
+            connection.sendline('export SHELL=%s' % lava_test_sh_cmd, delay=self.character_delay)
+
+        try:
+            feedbacks = []
+            for feedback_ns in self.data.keys():  # pylint: disable=no-member
+                if feedback_ns == self.parameters.get('namespace'):
+                    continue
+                feedback_connection = self.get_namespace_data(
+                    action='shared', label='shared', key='connection',
+                    deepcopy=False, parameters={"namespace": feedback_ns})
+                if feedback_connection:
+                    self.logger.debug("Will listen to feedbacks from '%s' for 1 second",
+                                      feedback_ns)
+                    feedbacks.append((feedback_ns, feedback_connection))
+
+            with connection.test_connection() as test_connection:
+                # the structure of lava-test-runner means that there is just one TestAction and it must run all definitions
+                test_connection.sendline(
+                    "%s/bin/lava-test-runner %s/%s" % (
+                        lava_test_results_dir,
+                        lava_test_results_dir,
+                        running),
+                    delay=self.character_delay)
+
+                test_connection.timeout = min(self.timeout.duration, self.connection_timeout.duration)
+                self.logger.info("Test shell timeout: %ds (minimum of the action and connection timeout)",
+                                 test_connection.timeout)
+
+                # Because of the feedbacks, we use a small value for the
+                # timeout.  This allows to grab feedback regularly.
+                last_check = time.time()
+                while self._keep_running(test_connection, test_connection.timeout, connection.check_char):
+                    # Only grab the feedbacks every test_connection.timeout
+                    if feedbacks and time.time() - last_check > test_connection.timeout:
+                        for feedback in feedbacks:
+                            self.logger.debug("Listening to namespace '%s'", feedback[0])
+                            # The timeout is really small because the goal is only
+                            # to clean the buffer of the feedback connections:
+                            # the characters are already in the buffer.
+                            # With an higher timeout, this can have a big impact on
+                            # the performances of the overall loop.
+                            feedback[1].listen_feedback(timeout=1)
+                            self.logger.debug("Listening to namespace '%s' done", feedback[0])
+                        last_check = time.time()
+        finally:
+            if self.current_run is not None:
+                self.logger.error("Marking unfinished test run as failed")
+                self.current_run["duration"] = "%.02f" % (time.time() - self.start)
+                self.logger.results(self.current_run)  # pylint: disable=no-member
+                self.current_run = None
+
+        # Only print if the report is not empty
+        if self.report:
+            self.logger.debug(yaml.dump(self.report, default_flow_style=False))
+        if self.errors:
+            raise TestError(self.errors)
+        return connection
+
+    def pattern_error(self, test_connection):
+        (testrun, ) = test_connection.match.groups()
+        self.logger.error("Unable to start testrun %s. "
+                          "Read the log for more details.", testrun)
+        self.errors = "Unable to start testrun %s" % testrun
+        # This is not accurate but required when exiting.
+        self.start = time.time()
+        self.current_run = {
+            "definition": "lava",
+            "case": testrun,
+            "result": "fail"
+        }
+        return True
+
+    def signal_start_run(self, params):
+        self.signal_director.test_uuid = params[1]
+        self.definition = params[0]
+        uuid = params[1]
+        self.start = time.time()
+        self.logger.info("Starting test lava.%s (%s)", self.definition, uuid)
+        # set the pattern for this run from pattern_dict
+        testdef_index = self.get_namespace_data(action='test-definition', label='test-definition',
+                                                key='testdef_index')
+        uuid_list = self.get_namespace_data(action='repo-action', label='repo-action', key='uuid-list')
+        for (key, value) in enumerate(testdef_index):
+            if self.definition == "%s_%s" % (key, value):
+                pattern_dict = self.get_namespace_data(action='test', label=uuid_list[key], key='testdef_pattern')
+                pattern = pattern_dict['testdef_pattern']['pattern']
+                fixup = pattern_dict['testdef_pattern']['fixupdict']
+                self.patterns.update({'test_case_result': re.compile(pattern, re.M)})
+                self.pattern.update(pattern, fixup)
+                self.logger.info("Enabling test definition pattern %r" % pattern)
+                self.logger.info("Enabling test definition fixup %r" % self.pattern.fixup)
+        self.current_run = {
+            "definition": "lava",
+            "case": self.definition,
+            "uuid": uuid,
+            "result": "fail"
+        }
+        testdef_commit = self.get_namespace_data(
+            action='test', label=uuid, key='commit-id')
+        if testdef_commit:
+            self.current_run.update({
+                'commit_id': testdef_commit
+            })
+
+    def signal_end_run(self, params):
+        self.definition = params[0]
+        uuid = params[1]
+        # remove the pattern for this run from pattern_dict
+        self._reset_patterns()
+        # catch error in ENDRUN being handled without STARTRUN
+        if not self.start:
+            self.start = time.time()
+        self.logger.info("Ending use of test pattern.")
+        self.logger.info("Ending test lava.%s (%s), duration %.02f",
+                         self.definition, uuid,
+                         time.time() - self.start)
+        self.current_run = None
+        res = {
+            "definition": "lava",
+            "case": self.definition,
+            "uuid": uuid,
+            'repository': self.get_namespace_data(
+                action='test', label=uuid, key='repository'),
+            'path': self.get_namespace_data(
+                action='test', label=uuid, key='path'),
+            "duration": "%.02f" % (time.time() - self.start),
+            "result": "pass"
+        }
+        revision = self.get_namespace_data(action='test', label=uuid, key='revision')
+        res['revision'] = revision if revision else 'unspecified'
+        res['namespace'] = self.parameters['namespace']
+        connection_namespace = self.parameters.get('connection_namespace', None)
+        if connection_namespace:
+            res['connection-namespace'] = connection_namespace
+        commit_id = self.get_namespace_data(action='test', label=uuid, key='commit-id')
+        if commit_id:
+            res['commit_id'] = commit_id
+
+        self.logger.results(res)  # pylint: disable=no-member
+        self.start = None
+
+    @nottest
+    def signal_test_case(self, params):
+        try:
+            data = handle_testcase(params)
+            # get the fixup from the pattern_dict
+            res = self.signal_match.match(data, fixupdict=self.pattern.fixupdict())
+        except (JobError, TestError) as exc:
+            self.logger.error(str(exc))
+            return True
+
+        p_res = self.get_namespace_data(action='test', label=self.signal_director.test_uuid, key='results')
+        if not p_res:
+            p_res = OrderedDict()
+            self.set_namespace_data(
+                action='test', label=self.signal_director.test_uuid, key='results', value=p_res)
+
+        # prevent losing data in the update
+        # FIXME: support parameters and retries
+        if res["test_case_id"] in p_res:
+            raise JobError(
+                "Duplicate test_case_id in results: %s",
+                res["test_case_id"])
+        # turn the result dict inside out to get the unique
+        # test_case_id/testset_name as key and result as value
+        res_data = {
+            'definition': self.definition,
+            'case': res["test_case_id"],
+            'result': res["result"]
+        }
+        # check for measurements
+        if 'measurement' in res:
+            try:
+                measurement = decimal.Decimal(res['measurement'])
+            except decimal.InvalidOperation:
+                raise TestError("Invalid measurement %s", res['measurement'])
+            res_data['measurement'] = measurement
+            if 'units' in res:
+                res_data['units'] = res['units']
+
+        if self.testset_name:
+            res_data['set'] = self.testset_name
+            self.report[res['test_case_id']] = {
+                'set': self.testset_name,
+                'result': res['result']
+            }
+        else:
+            self.report[res['test_case_id']] = res['result']
+        # Send the results back
+        self.logger.results(res_data)  # pylint: disable=no-member
+
+    @nottest
+    def signal_test_reference(self, params):
+        if len(params) != 3:
+            raise TestError("Invalid use of TESTREFERENCE")
+        res_dict = {
+            'case': params[0],
+            'definition': self.definition,
+            'result': params[1],
+            'reference': params[2],
+        }
+        if self.testset_name:
+            res_dict.update({'set': self.testset_name})
+        self.logger.results(res_dict)  # pylint: disable=no-member
+
+    @nottest
+    def signal_test_set(self, params):
+        name = None
+        action = params.pop(0)
+        if action == "START":
+            name = "testset_" + action.lower()
+            try:
+                self.testset_name = params[0]
+            except IndexError:
+                raise JobError("Test set declared without a name")
+            self.logger.info("Starting test_set %s", self.testset_name)
+        elif action == "STOP":
+            self.logger.info("Closing test_set %s", self.testset_name)
+            self.testset_name = None
+            name = "testset_" + action.lower()
+        return name
+
+    @nottest
+    def pattern_test_case(self, test_connection):
+        match = test_connection.match
+        if match is pexpect.TIMEOUT:
+            self.logger.warning("err: lava_test_shell has timed out (test_case)")
+            return False
+        res = self.signal_match.match(match.groupdict(), fixupdict=self.pattern.fixupdict())
+        self.logger.debug("outer_loop_result: %s" % res)
+        return True
+
+    @nottest
+    def pattern_test_case_result(self, test_connection):
+        res = test_connection.match.groupdict()
+        fixupdict = self.pattern.fixupdict()
+        if res['result'] in fixupdict:
+            res['result'] = fixupdict[res['result']]
+        if res:
+            # disallow whitespace in test_case_id
+            test_case_id = "%s" % res['test_case_id'].replace('/', '_')
+            if ' ' in test_case_id.strip():
+                self.logger.debug("Skipping invalid test_case_id '%s'", test_case_id.strip())
+                return True
+            res_data = {
+                'definition': self.definition,
+                'case': res["test_case_id"],
+                'result': res["result"]
+            }
+            # check for measurements
+            if 'measurement' in res:
+                try:
+                    measurement = decimal.Decimal(res['measurement'])
+                except decimal.InvalidOperation:
+                    raise TestError("Invalid measurement %s", res['measurement'])
+                res_data['measurement'] = measurement
+                if 'units' in res:
+                    res_data['units'] = res['units']
+
+            self.logger.results(res_data)  # pylint: disable=no-member
+            self.report[res["test_case_id"]] = res["result"]
+        return True
+
+    def check_patterns(self, event, test_connection, check_char):  # pylint: disable=unused-argument
+        """
+        Defines the base set of pattern responses.
+        Stores the results of testcases inside the TestAction
+        Call from subclasses before checking subclass-specific events.
+        """
+        ret_val = False
+        if event == "exit":
+            self.logger.info("ok: lava_test_shell seems to have completed")
+            self.testset_name = None
+
+        elif event == "error":
+            # Parsing is not finished
+            ret_val = self.pattern_error(test_connection)
+
+        elif event == "eof":
+            self.testset_name = None
+            raise InfrastructureError("lava_test_shell connection dropped.")
+
+        elif event == "timeout":
+            # allow feedback in long runs
+            ret_val = True
+
+        elif event == "signal":
+            name, params = test_connection.match.groups()
+            self.logger.debug("Received signal: <%s> %s" % (name, params))
+            params = params.split()
+            if name == "STARTRUN":
+                self.signal_start_run(params)
+            elif name == "ENDRUN":
+                self.signal_end_run(params)
+            elif name == "TESTCASE":
+                self.signal_test_case(params)
+            elif name == "TESTREFERENCE":
+                self.signal_test_reference(params)
+            elif name == "TESTSET":
+                ret = self.signal_test_set(params)
+                if ret:
+                    name = ret
+
+            self.signal_director.signal(name, params)
+            ret_val = True
+
+        elif event == "test_case":
+            ret_val = self.pattern_test_case(test_connection)
+        elif event == 'test_case_result':
+            ret_val = self.pattern_test_case_result(test_connection)
+        return ret_val
+
+    def _keep_running(self, test_connection, timeout, check_char):
+        if 'test_case_results' in self.patterns:
+            self.logger.info("Test case result pattern: %r" % self.patterns['test_case_results'])
+        retval = test_connection.expect(list(self.patterns.values()), timeout=timeout)
+        return self.check_patterns(list(self.patterns.keys())[retval], test_connection, check_char)
+
+    class SignalDirector(object):
+
+        # FIXME: create proxy handlers
+        def __init__(self, protocol=None):
+            """
+            Base SignalDirector for singlenode jobs.
+            MultiNode and LMP jobs need to create a suitable derived class as both also require
+            changes equivalent to the old _keep_running functionality.
+
+            SignalDirector is the link between the Action and the Connection. The Action uses
+            the SignalDirector to interact with the I/O over the Connection.
+            """
+            self._cur_handler = BaseSignalHandler(protocol)
+            self.protocol = protocol  # communicate externally over the protocol API
+            self.connection = None  # communicate with the device
+            self.logger = logging.getLogger("dispatcher")
+            self.test_uuid = None
+
+        def setup(self, parameters):
+            """
+            Allows the parent Action to pass extra data to a customised SignalDirector
+            """
+            pass
+
+        def signal(self, name, params):
+            handler = getattr(self, "_on_" + name.lower(), None)
+            if not handler and self._cur_handler:
+                handler = self._cur_handler.custom_signal
+                params = [name] + list(params)
+            if handler:
+                try:
+                    # The alternative here is to drop the getattr and have a long if:elif:elif:else.
+                    # Without python support for switch, this gets harder to read than using
+                    # a getattr lookup for the callable (codehelp). So disable checkers:
+                    # noinspection PyCallingNonCallable
+                    handler(*params)
+                except TypeError as exc:
+                    # handle serial corruption which can overlap kernel messages onto test output.
+                    self.logger.exception(str(exc))
+                    raise TestError("Unable to handle the test shell signal correctly: %s" % str(exc))
+                except JobError as exc:
+                    self.logger.error("job error: handling signal %s failed: %s", name, exc)
+                    return False
+                return True
+
+        def postprocess_bundle(self, bundle):
+            pass
+
+        def _on_testset_start(self, set_name):
+            pass
+
+        def _on_testset_stop(self):
+            pass
+
+        # noinspection PyUnusedLocal
+        def _on_startrun(self, test_run_id, uuid):  # pylint: disable=unused-argument
+            """
+            runsh.write('echo "<LAVA_SIGNAL_STARTRUN $TESTRUN_ID $UUID>"\n')
+            """
+            self._cur_handler = None
+            if self._cur_handler:
+                self._cur_handler.start()
+
+        # noinspection PyUnusedLocal
+        def _on_endrun(self, test_run_id, uuid):  # pylint: disable=unused-argument
+            if self._cur_handler:
+                self._cur_handler.end()
+
+        def _on_starttc(self, test_case_id):
+            if self._cur_handler:
+                self._cur_handler.starttc(test_case_id)
+
+        def _on_endtc(self, test_case_id):
+            if self._cur_handler:
+                self._cur_handler.endtc(test_case_id)
diff --git a/lava_dispatcher/actions/test/strategies.py b/lava_dispatcher/actions/test/strategies.py
new file mode 100644
index 000000000..fd9ecf6e6
--- /dev/null
+++ b/lava_dispatcher/actions/test/strategies.py
@@ -0,0 +1,28 @@
+# Copyright (C) 2014 Linaro Limited
+#
+# Author: Neil Williams <neil.williams@linaro.org>
+#
+# This file is part of LAVA Dispatcher.
+#
+# LAVA Dispatcher is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# LAVA Dispatcher is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along
+# with this program; if not, see <http://www.gnu.org/licenses>.
+
+# List just the subclasses supported for this base strategy
+# imported by the parser to populate the list of subclasses.
+
+# pylint: disable=unused-import
+
+from lava_dispatcher.actions.test.shell import TestShell
+from lava_dispatcher.actions.test.multinode import MultinodeTestShell
+from lava_dispatcher.actions.test.monitor import TestMonitor