BIGTOP-2577: kafka charm test updates (closes #157)

Signed-off-by: Kevin W Monroe <kevin.monroe@canonical.com>

7 files changed, 113 insertions, 118 deletions

diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/README.md b/bigtop-packages/src/charm/kafka/layer-kafka/README.md
index 1db627ee..0d416773 100644
--- a/bigtop-packages/src/charm/kafka/layer-kafka/README.md
+++ b/bigtop-packages/src/charm/kafka/layer-kafka/README.md
@@ -19,17 +19,19 @@
 Apache Kafka is an open-source message broker project developed by the Apache
 Software Foundation written in Scala. The project aims to provide a unified,
 high-throughput, low-latency platform for handling real-time data feeds. Learn
-more at [kafka.apache.org](http://kafka.apache.org/).
+more at [kafka.apache.org][].
 
-This charm deploys the Kafka component of the Apache Bigtop platform.
+This charm deploys the Kafka component of the [Apache Bigtop][] platform.
 
+[kafka.apache.org]: http://kafka.apache.org/
+[Apache Bigtop]: http://bigtop.apache.org/
 
-# Deploying / Using
+
+# Deploying
 
 A working Juju installation is assumed to be present. If Juju is not yet set
-up, please follow the
-[getting-started](https://jujucharms.com/docs/2.0/getting-started)
-instructions prior to deploying this charm.
+up, please follow the [getting-started][] instructions prior to deploying this
+charm.
 
 Kafka requires the Zookeeper distributed coordination service. Deploy and
 relate them as follows:
@@ -38,6 +40,17 @@ relate them as follows:
     juju deploy zookeeper
     juju add-relation kafka zookeeper
 
+## Network-Restricted Environments
+Charms can be deployed in environments with limited network access. To deploy
+in this environment, configure a Juju model with appropriate proxy and/or
+mirror options. See [Configuring Models][] for more information.
+
+[getting-started]: https://jujucharms.com/docs/stable/getting-started
+[Configuring Models]: https://jujucharms.com/docs/stable/models-config
+
+
+# Using
+
 Once deployed, there are a number of actions available in this charm.
 > **Note**: Actions described below assume Juju 2.0 or greater. If using an
 earlier version of Juju, the action syntax is:
@@ -83,7 +96,7 @@ are ready:
 This is particularly useful when combined with `watch` to track the on-going
 progress of the deployment:
 
-    watch -n 0.5 juju status
+    watch -n 2 juju status
 
 The message column will provide information about a given unit's state.
 This charm is ready for use once the status message indicates that it is
@@ -102,7 +115,7 @@ of Juju, the syntax is `juju action do kafka/0 smoke-test`.
 
 Watch the progress of the smoke test actions with:
 
-    watch -n 0.5 juju show-action-status
+    watch -n 2 juju show-action-status
 
 > **Note**: The above assumes Juju 2.0 or greater. If using an earlier version
 of Juju, the syntax is `juju action status`.
@@ -202,15 +215,6 @@ machine, simply pass ``0.0.0.0`` to ``network_interface``.
     juju config kafka network_interface=0.0.0.0
 
 
-# Network-Restricted Environments
-
-Charms can be deployed in environments with limited network access. To deploy
-in this environment, configure a Juju model with appropriate
-proxy and/or mirror options. See
-[Configuring Models](https://jujucharms.com/docs/2.0/models-config) for more
-information.
-
-
 # Contact Information
 
 - <bigdata@lists.ubuntu.com>
diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/actions/read-topic b/bigtop-packages/src/charm/kafka/layer-kafka/actions/read-topic
index b385f668..8b7a27d6 100755
--- a/bigtop-packages/src/charm/kafka/layer-kafka/actions/read-topic
+++ b/bigtop-packages/src/charm/kafka/layer-kafka/actions/read-topic
@@ -15,16 +15,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-sys.path.append('lib')
-
 import kafkautils
 import subprocess
+import sys
+sys.path.append('lib')  # Add our helpers to our path.
 
-from charmhelpers.core import hookenv, host
-from charms.layer.apache_bigtop_base import get_layer_opts
-from charms.reactive import is_state
-from jujubigdata import utils
+from charmhelpers.core import hookenv, host  # noqa: E402
+from charms.layer.apache_bigtop_base import get_layer_opts  # noqa: E402
+from charms.reactive import is_state  # noqa: E402
+from jujubigdata import utils  # noqa: E402
 
 
 if not is_state('kafka.started'):
diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/actions/write-topic b/bigtop-packages/src/charm/kafka/layer-kafka/actions/write-topic
index b879b654..6a56265d 100755
--- a/bigtop-packages/src/charm/kafka/layer-kafka/actions/write-topic
+++ b/bigtop-packages/src/charm/kafka/layer-kafka/actions/write-topic
@@ -15,16 +15,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-sys.path.append('lib')
-
 import kafkautils
 import subprocess
+import sys
+sys.path.append('lib')  # Add our helpers to our path.
 
-from charmhelpers.core import hookenv, host
-from charms.layer.apache_bigtop_base import get_layer_opts
-from charms.reactive import is_state
-from jujubigdata import utils
+from charmhelpers.core import hookenv, host  # noqa: E402
+from charms.layer.apache_bigtop_base import get_layer_opts  # noqa: E402
+from charms.reactive import is_state  # noqa: E402
+from jujubigdata import utils  # noqa: E402
 
 
 if not is_state('kafka.started'):
diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/reactive/kafka.py b/bigtop-packages/src/charm/kafka/layer-kafka/reactive/kafka.py
index 6e7d325e..97f96d67 100644
--- a/bigtop-packages/src/charm/kafka/layer-kafka/reactive/kafka.py
+++ b/bigtop-packages/src/charm/kafka/layer-kafka/reactive/kafka.py
@@ -14,7 +14,7 @@
 # limitations under the License.
 
 from charmhelpers.core import hookenv
-from charms.layer.apache_bigtop_base import get_layer_opts
+from charms.layer.apache_bigtop_base import get_layer_opts, get_package_version
 from charms.layer.bigtop_kafka import Kafka
 from charms.reactive import set_state, remove_state, when, when_not
 from charms.reactive.helpers import data_changed
@@ -44,6 +44,9 @@ def configure_kafka(zk):
     kafka.open_ports()
     set_state('kafka.started')
     hookenv.status_set('active', 'ready')
+    # set app version string for juju status output
+    kafka_version = get_package_version('kafka') or 'unknown'
+    hookenv.application_version_set(kafka_version)
 
 
 @when('kafka.started', 'zookeeper.ready')
diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/tests/01-deploy.py b/bigtop-packages/src/charm/kafka/layer-kafka/tests/01-deploy.py
index af11f9c8..62d6a559 100755
--- a/bigtop-packages/src/charm/kafka/layer-kafka/tests/01-deploy.py
+++ b/bigtop-packages/src/charm/kafka/layer-kafka/tests/01-deploy.py
@@ -15,34 +15,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import unittest
 import amulet
+import unittest
 
 
 class TestDeploy(unittest.TestCase):
     """
-    Trivial deployment test for Apache Kafka.
+    Trivial deployment test for Apache Bigtop Kafka.
     """
     @classmethod
     def setUpClass(cls):
-        cls.d = amulet.Deployment(series='trusty')
-        cls.d.add('kafka', 'kafka')
-        cls.d.add('openjdk', 'openjdk')
-        cls.d.add('zk', 'zookeeper')
-
-        cls.d.configure('openjdk', {'java-type': 'jdk',
-                                    'java-major': '8'})
-
-        cls.d.relate('kafka:zookeeper', 'zk:zookeeper')
-        cls.d.relate('kafka:java', 'openjdk:java')
-        try:
-            cls.d.relate('zk:java', 'openjdk:java')
-        except ValueError:
-            # No need to related older versions of the zookeeper charm
-            # to java.
-            pass
-
-        cls.d.setup(timeout=900)
+        cls.d = amulet.Deployment(series='xenial')
+        cls.d.add('kafka', charm='kafka')
+        cls.d.add('zookeeper', charm='cs:xenial/zookeeper')
+
+        cls.d.relate('kafka:zookeeper', 'zookeeper:zookeeper')
+
+        cls.d.setup(timeout=1800)
         cls.d.sentry.wait_for_messages({'kafka': 'ready'}, timeout=1800)
         cls.kafka = cls.d.sentry['kafka'][0]
 
diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/tests/02-smoke-test.py b/bigtop-packages/src/charm/kafka/layer-kafka/tests/02-smoke-test.py
index 330cde9f..f396bdb3 100755
--- a/bigtop-packages/src/charm/kafka/layer-kafka/tests/02-smoke-test.py
+++ b/bigtop-packages/src/charm/kafka/layer-kafka/tests/02-smoke-test.py
@@ -15,34 +15,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import unittest
 import amulet
+import unittest
 
 
 class TestDeploy(unittest.TestCase):
     """
-    Smoke test of Apache Kafka.
+    Smoke test for Apache Bigtop Kafka.
     """
     @classmethod
     def setUpClass(cls):
-        cls.d = amulet.Deployment(series='trusty')
-        cls.d.add('kafka', 'kafka')
-        cls.d.add('openjdk', 'openjdk')
-        cls.d.add('zk', 'zookeeper')
-
-        cls.d.configure('openjdk', {'java-type': 'jdk',
-                                    'java-major': '8'})
-
-        cls.d.relate('kafka:zookeeper', 'zk:zookeeper')
-        cls.d.relate('kafka:java', 'openjdk:java')
-        try:
-            cls.d.relate('zk:java', 'openjdk:java')
-        except ValueError:
-            # No need to related older versions of the zookeeper charm
-            # to java.
-            pass
-
-        cls.d.setup(timeout=900)
+        cls.d = amulet.Deployment(series='xenial')
+        cls.d.add('kafka', charm='kafka')
+        cls.d.add('zookeeper', charm='cs:xenial/zookeeper')
+
+        cls.d.relate('kafka:zookeeper', 'zookeeper:zookeeper')
+
+        cls.d.setup(timeout=1800)
         cls.d.sentry.wait_for_messages({'kafka': 'ready'}, timeout=1800)
         cls.kafka = cls.d.sentry['kafka'][0]
 
@@ -50,9 +39,11 @@ class TestDeploy(unittest.TestCase):
         """
         Validate Kafka by running the smoke-test action.
         """
-        smk_uuid = self.kafka.action_do("smoke-test")
-        output = self.d.action_fetch(smk_uuid, full_output=True)
-        assert "completed" in output['status']
+        uuid = self.kafka.run_action('smoke-test')
+        result = self.d.action_fetch(uuid, full_output=True)
+        # action status=completed on success
+        if (result['status'] != "completed"):
+            self.fail('Kafka smoke-test failed: %s' % result)
 
 
 if __name__ == '__main__':
diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/tests/10-config-changed.py b/bigtop-packages/src/charm/kafka/layer-kafka/tests/10-config-changed.py
index dd20c53d..4fd44ceb 100755
--- a/bigtop-packages/src/charm/kafka/layer-kafka/tests/10-config-changed.py
+++ b/bigtop-packages/src/charm/kafka/layer-kafka/tests/10-config-changed.py
@@ -15,83 +15,93 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import unittest
 import amulet
 import re
+import time
+import unittest
 
 
 class TestConfigChanged(unittest.TestCase):
     """
-    Test to verify that we update network interface bindings successfully.
-
+    Test to verify that we can bind to listen for client connections
+    on a specific interface.
     """
     @classmethod
     def setUpClass(cls):
-        cls.d = amulet.Deployment(series='trusty')
-        cls.d.log.debug("foo!")
-        cls.d.add('kafka', 'kafka')
-        cls.d.add('openjdk', 'openjdk')
-        cls.d.add('zk', 'zookeeper')
-
-        cls.d.configure('openjdk', {'java-type': 'jdk',
-                                    'java-major': '8'})
-
-        cls.d.relate('kafka:zookeeper', 'zk:zookeeper')
-        cls.d.relate('kafka:java', 'openjdk:java')
-        try:
-            cls.d.relate('zk:java', 'openjdk:java')
-        except ValueError:
-            # No need to related older versions of the zookeeper charm
-            # to java.
-            pass
-
-        cls.d.setup(timeout=900)
+        cls.d = amulet.Deployment(series='xenial')
+        cls.d.add('kafka', charm='kafka')
+        cls.d.add('zookeeper', charm='cs:xenial/zookeeper')
+
+        cls.d.relate('kafka:zookeeper', 'zookeeper:zookeeper')
+
+        cls.d.setup(timeout=1800)
         cls.d.sentry.wait_for_messages({'kafka': 'ready'}, timeout=1800)
-        cls.kafka = cls.d.sentry['kafka'][0]
+        cls.unit = cls.d.sentry['kafka'][0]
 
     def test_bind_network_interface(self):
         """
-        Test to verify that we update network interface bindings successfully.
-
+        Verify that we update client port bindings successfully.
         """
-        self.d.configure('kafka', {'network_interface': 'eth0'})
-        self.d.sentry.wait_for_messages({'kafka': 'updating zookeeper instances'}, timeout=600)
-
-        self.d.sentry.wait_for_messages({'kafka': 'ready'}, timeout=600)
-        ret = self.kafka.run(
+        network_interface = None
+        # Regular expression should handle interfaces in the format
+        # eth[n], and in the format en[foo] (the "predicatble
+        # interface names" in v197+ of systemd).
+        ethernet_interface = re.compile('^e[thn]+.*')
+        interfaces, _ = self.unit.run(
+            "ifconfig -a | sed 's/[ \t].*//;/^$/d'")
+        interfaces = interfaces.split()  # Splits on newlines
+        for interface in interfaces:
+            if ethernet_interface.match(interface):
+                network_interface = interface
+                break
+
+        if network_interface is None:
+            raise Exception(
+                "Could not find any interface on the unit that matched my "
+                "criteria.")
+        self.d.configure('kafka', {'network_interface': network_interface})
+
+        # NB: we used to watch for a maintenance status message, but every now
+        # and then, we'd miss it. Wait 2m to let the config-changed hook settle.
+        time.sleep(120)
+        ret = self.unit.run(
             'grep host.name /etc/kafka/conf/server.properties')[0]
+
         # Correct line should start with host.name (no comment hash
         # mark), followed by an equals sign and something that looks
         # like an IP address (we aren't too strict about it being a
         # valid ip address.)
         matcher = re.compile("^host\.name=\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}.*")
-
-        self.assertTrue('host.name' in ret)
         self.assertTrue(matcher.match(ret))
 
         # Verify that smoke tests still run
-        smk_uuid = self.kafka.action_do("smoke-test")
-        output = self.d.action_fetch(smk_uuid, full_output=True)
-        assert "completed" in output['status']
+        smk_uuid = self.unit.run_action('smoke-test')
+        result = self.d.action_fetch(smk_uuid, full_output=True)
+        # actions set status=completed on success
+        if (result['status'] != "completed"):
+            self.fail('Kafka test failed after setting nic config: %s' % result)
 
     def test_reset_network_interface(self):
         """
-        Verify that we can reset the network interface to 0.
-
+        Verify that we can reset the client port bindings to 0.0.0.0
         """
         self.d.configure('kafka', {'network_interface': '0.0.0.0'})
-        self.d.sentry.wait_for_messages({'kafka': 'updating zookeeper instances'}, timeout=600)
-        self.d.sentry.wait_for_messages({'kafka': 'ready'}, timeout=600)
-        ret = self.kafka.run(
+
+        # NB: we used to watch for a maintenance status message, but every now
+        # and then, we'd miss it. Wait 2m to let the config-changed hook settle.
+        time.sleep(120)
+        ret = self.unit.run(
             'grep host.name /etc/kafka/conf/server.properties')[0]
 
         matcher = re.compile("^host\.name=0\.0\.0\.0.*")
         self.assertTrue(matcher.match(ret))
 
         # Verify that smoke tests still run
-        smk_uuid = self.kafka.action_do("smoke-test")
-        output = self.d.action_fetch(smk_uuid, full_output=True)
-        assert "completed" in output['status']
+        smk_uuid = self.unit.run_action('smoke-test')
+        result = self.d.action_fetch(smk_uuid, full_output=True)
+        # actions set status=completed on success
+        if (result['status'] != "completed"):
+            self.fail('Kafka test failed after resetting nic config: %s' % result)
 
 
 if __name__ == '__main__':