bisect: Implement bisect handler.

  * Implement a new bisect handler that will perform bisect
    operation on the collections. At the moment the only
    supported collection is 'boot'.
  * Add celery task to execute the elaboration.
  * Add tests for the handler.
  * Add necessary variables.

Change-Id: Iacaac5ec3c4c8aaa80ff87064564ac22ffc7e9f3

6 files changed, 445 insertions, 2 deletions

diff --git a/app/handlers/bisect.py b/app/handlers/bisect.py
new file mode 100644
index 0000000..a0e5b97
--- /dev/null
+++ b/app/handlers/bisect.py
@@ -0,0 +1,119 @@
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""The request handler for bisect URLs."""
+
+import tornado
+
+from functools import partial
+from tornado.web import asynchronous
+
+from handlers.base import BaseHandler
+from handlers.common import NOT_VALID_TOKEN
+from handlers.response import HandlerResponse
+from taskqueue.tasks import boot_bisect
+
+from models import (
+    BOOT_COLLECTION,
+)
+
+BISECT_COLLECTIONS = [
+    BOOT_COLLECTION,
+]
+
+
+class BisectHandler(BaseHandler):
+    """Handler used to trigger bisect operations on the data."""
+
+    def __init__(self, application, request, **kwargs):
+        super(BisectHandler, self).__init__(application, request, **kwargs)
+
+    @asynchronous
+    def get(self, *args, **kwargs):
+        self.executor.submit(
+            partial(self.execute_get, *args, **kwargs)
+        ).add_done_callback(
+            lambda future:
+            tornado.ioloop.IOLoop.instance().add_callback(
+                partial(self._create_valid_response, future.result())
+            )
+        )
+
+    def execute_get(self, *args, **kwargs):
+        """This is the actual GET operation.
+
+        It is done in this way so that subclasses can implement a different
+        token authorization if necessary.
+        """
+        response = None
+
+        if self.validate_req_token("GET"):
+            if kwargs:
+                collection = kwargs.get("collection", None)
+                doc_id = kwargs.get("id", None)
+                if all([collection, doc_id]):
+                    response = self._get_bisect(collection, doc_id)
+                else:
+                    response = HandlerResponse(400)
+            else:
+                response = HandlerResponse(400)
+        else:
+            response = HandlerResponse(403)
+            response.reason = NOT_VALID_TOKEN
+
+        return response
+
+    def _get_bisect(self, collection, doc_id):
+        """Retrieve the bisect data.
+
+        :param collection: The name of the collection to operate on.
+        :type collection: str
+        :param doc_id: The ID of the document to execute the bisect on.
+        :type doc_id: str
+        :return A `HandlerResponse` object.
+        """
+        response = None
+
+        if collection in BISECT_COLLECTIONS:
+            if collection == BOOT_COLLECTION:
+                response = self.execute_boot_bisect(
+                    doc_id, self.settings["dboptions"]
+                )
+        else:
+            response = HandlerResponse(400)
+            response.reason = (
+                "Provided bisect collection '%s' is not valid" % collection
+            )
+
+        return response
+
+    @staticmethod
+    def execute_boot_bisect(doc_id, db_options):
+        """Execute the boot bisect operation.
+
+        :param doc_id: The ID of the document to execute the bisect on.
+        :type doc_id: str
+        :param db_options: The mongodb database connection parameters.
+        :type db_options: dict
+        :return A `HandlerResponse` object.
+        """
+        response = HandlerResponse()
+
+        result = boot_bisect.apply_async([doc_id, db_options])
+        while not result.ready():
+            pass
+
+        response.status_code, response.result = result.get()
+        if response.status_code == 404:
+            response.reason = "Boot report not found"
+        return response
diff --git a/app/handlers/tests/test_bisect_handler.py b/app/handlers/tests/test_bisect_handler.py
new file mode 100644
index 0000000..c052b19
--- /dev/null
+++ b/app/handlers/tests/test_bisect_handler.py
@@ -0,0 +1,107 @@
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test module for the BisectHandler handler."""
+
+import json
+import mongomock
+
+from concurrent.futures import ThreadPoolExecutor
+from mock import patch, MagicMock
+from tornado import (
+    ioloop,
+    testing,
+    web,
+)
+
+from handlers.app import AppHandler
+from urls import _BISECT_URL
+
+# Default Content-Type header returned by Tornado.
+DEFAULT_CONTENT_TYPE = 'application/json; charset=UTF-8'
+
+
+class TestBisectHandler(testing.AsyncHTTPTestCase, testing.LogTrapTestCase):
+
+    def setUp(self):
+        self.mongodb_client = mongomock.Connection()
+        super(TestBisectHandler, self).setUp()
+
+        self.task_return_value = MagicMock()
+        self.task_ready = MagicMock()
+        self.task_ready.return_value = True
+        self.task_return_value.ready = self.task_ready
+        self.task_return_value.get = MagicMock()
+        self.task_return_value.get.return_value = 200, []
+
+        patched_boot_bisect_func = patch("handlers.bisect.boot_bisect")
+        self.boot_bisect = patched_boot_bisect_func.start()
+        self.boot_bisect.apply_async = MagicMock()
+        self.boot_bisect.apply_async.return_value = self.task_return_value
+
+        patched_find_token = patch("handlers.base.BaseHandler._find_token")
+        self.find_token = patched_find_token.start()
+        self.find_token.return_value = "token"
+
+        patched_validate_token = patch("handlers.base.validate_token")
+        self.validate_token = patched_validate_token.start()
+        self.validate_token.return_value = True
+
+        self.addCleanup(patched_find_token.stop)
+        self.addCleanup(patched_validate_token.stop)
+        self.addCleanup(patched_boot_bisect_func.stop)
+
+    def get_app(self):
+        dboptions = {
+            'dbpassword': "",
+            'dbuser': ""
+        }
+
+        settings = {
+            'dboptions': dboptions,
+            'client': self.mongodb_client,
+            'executor': ThreadPoolExecutor(max_workers=2),
+            'default_handler_class': AppHandler,
+            'debug': False
+        }
+
+        return web.Application([_BISECT_URL], **settings)
+
+    def get_new_ioloop(self):
+        return ioloop.IOLoop.instance()
+
+    def test_bisect_wrong_collection(self):
+        headers = {'Authorization': 'foo'}
+
+        response = self.fetch('/api/bisect/foo/doc_id', headers=headers)
+        self.assertEqual(response.code, 400)
+
+    def test_boot_bisect_no_token(self):
+        self.find_token.return_value = None
+
+        response = self.fetch('/api/bisect/boot/id')
+        self.assertEqual(response.code, 403)
+
+    def test_boot_bisect_wrong_url(self):
+        headers = {'Authorization': 'foo'}
+
+        response = self.fetch('/api/bisect/boot/', headers=headers)
+        self.assertEqual(response.code, 400)
+
+    def test_boot_bisect_no_id(self):
+        headers = {'Authorization': 'foo'}
+
+        self.task_return_value.get.return_value = 404, []
+
+        response = self.fetch('/api/bisect/boot/foo', headers=headers)
+        self.assertEqual(response.code, 404)
diff --git a/app/taskqueue/tasks.py b/app/taskqueue/tasks.py
index bdf095b..c0b8408 100644
--- a/app/taskqueue/tasks.py
+++ b/app/taskqueue/tasks.py
@@ -26,6 +26,7 @@ from utils.subscription import send
 from utils.batch.common import (
     execute_batch_operation,
 )
+from utils.bisect import execute_boot_bisection
 from utils import LOG
 
 
@@ -83,6 +84,19 @@ def execute_batch(json_obj, db_options):
     return execute_batch_operation(json_obj, db_options)
 
 
+@app.task(name="boot-bisect")
+def boot_bisect(doc_id, db_options):
+    """Run a boot bisect operation on the passed boot document id.
+
+    :param doc_id: The boot document ID.
+    :type doc_id: str
+    :param db_options: The mongodb database connection parameters.
+    :type db_options: dict
+    :return The result of the boot bisect operation.
+    """
+    return execute_boot_bisection(doc_id, db_options)
+
+
 def run_batch_group(batch_op_list, db_options):
     """Execute a list of batch operations.
 
diff --git a/app/tests/__init__.py b/app/tests/__init__.py
index f8f570e..400d11f 100644
--- a/app/tests/__init__.py
+++ b/app/tests/__init__.py
@@ -21,6 +21,7 @@ import unittest
 def test_modules():
     return [
         'handlers.tests.test_batch_handler',
+        'handlers.tests.test_bisect_handler',
         'handlers.tests.test_boot_handler',
         'handlers.tests.test_count_handler',
         'handlers.tests.test_defconf_handler',
diff --git a/app/urls.py b/app/urls.py
index b737c70..f12d927 100644
--- a/app/urls.py
+++ b/app/urls.py
@@ -17,13 +17,14 @@
 
 from tornado.web import url
 
+from handlers.bisect import BisectHandler
+from handlers.batch import BatchHandler
 from handlers.boot import BootHandler
 from handlers.count import CountHandler
 from handlers.defconf import DefConfHandler
 from handlers.job import JobHandler
 from handlers.subscription import SubscriptionHandler
 from handlers.token import TokenHandler
-from handlers.batch import BatchHandler
 
 
 _JOB_URL = url(r'/api/job(?P<sl>/)?(?P<id>.*)', JobHandler, name='job')
@@ -45,13 +46,19 @@ _TOKEN_URL = url(
 _BATCH_URL = url(
     r'/api/batch', BatchHandler, name='batch'
 )
+_BISECT_URL = url(
+    r"/api/bisect/(?P<collection>.*)/(?P<id>.*)",
+    BisectHandler,
+    name="bisect"
+)
 
 APP_URLS = [
+    _BATCH_URL,
+    _BISECT_URL,
     _BOOT_URL,
     _COUNT_URL,
     _DEFCONF_URL,
     _JOB_URL,
     _SUBSCRIPTION_URL,
     _TOKEN_URL,
-    _BATCH_URL,
 ]
diff --git a/app/utils/bisect/__init__.py b/app/utils/bisect/__init__.py
new file mode 100644
index 0000000..4f2a913
--- /dev/null
+++ b/app/utils/bisect/__init__.py
@@ -0,0 +1,195 @@
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""All the bisect operations that the app can perform."""
+
+from bson.json_util import default
+from json import (
+    dumps as j_dump,
+    loads as j_load,
+)
+from pymongo import DESCENDING
+
+from models import (
+    BOOT_COLLECTION,
+    DEFCONFIG_COLLECTION,
+    BOARD_KEY,
+    CREATED_KEY,
+    DEFCONFIG_KEY,
+    JOB_KEY,
+    JOB_ID_KEY,
+    KERNEL_KEY,
+    METADATA_KEY,
+    STATUS_KEY,
+    ARCHITECTURE_KEY,
+    DIRNAME_KEY,
+    PASS_STATUS,
+    BISECT_BOOT_CREATED_KEY,
+    BISECT_BOOT_METADATA_KEY,
+    BISECT_BOOT_STATUS_KEY,
+    BISECT_DEFCONFIG_CREATED_KEY,
+    BISECT_DEFCONFIG_METADATA_KEY,
+    BISECT_DEFCONFIG_STATUS_KEY,
+    BISECT_DEFCONFIG_ARCHITECTURE_KEY,
+)
+from utils.db import (
+    find,
+    find_one,
+    get_db_connection,
+)
+
+
+BOOT_SEARCH_FIELDS = [
+    BOARD_KEY,
+    CREATED_KEY,
+    DEFCONFIG_KEY,
+    JOB_ID_KEY,
+    JOB_KEY,
+    KERNEL_KEY,
+    METADATA_KEY,
+    STATUS_KEY,
+]
+
+BOOT_DEFCONFIG_SEARCH_FIELDS = [
+    ARCHITECTURE_KEY,
+    CREATED_KEY,
+    DIRNAME_KEY,
+    METADATA_KEY,
+    STATUS_KEY,
+]
+
+BOOT_SORT = [(CREATED_KEY, DESCENDING)]
+
+
+def _combine_defconfig_values(boot_doc, db_options):
+    """Combine the boot document values with their own defconfing.
+
+    It returns a list of dictionaries whose structure is a combination
+    of the values from the boot document and its associated defconfing.
+
+    :param boot_doc: The boot document to retrieve the defconfig of.
+    :type boot_doc: dict
+    :param db_options: The mongodb database connection parameters.
+    :type db_options: dict
+    :return A list of dictionaries.
+    """
+    database = get_db_connection(db_options)
+
+    boot_doc_get = boot_doc.get
+
+    job = boot_doc_get(JOB_KEY)
+    kernel = boot_doc_get(KERNEL_KEY)
+    defconfig = boot_doc_get(DEFCONFIG_KEY)
+
+    combined_values = {
+        JOB_KEY: job,
+        KERNEL_KEY: kernel,
+        DEFCONFIG_KEY: defconfig,
+        BISECT_BOOT_STATUS_KEY: boot_doc_get(STATUS_KEY),
+        BISECT_BOOT_CREATED_KEY: boot_doc_get(CREATED_KEY),
+        BISECT_BOOT_METADATA_KEY: boot_doc_get(METADATA_KEY),
+        DIRNAME_KEY: "",
+        BISECT_DEFCONFIG_CREATED_KEY: "",
+        BISECT_DEFCONFIG_ARCHITECTURE_KEY: "",
+        BISECT_DEFCONFIG_STATUS_KEY: "",
+        BISECT_DEFCONFIG_METADATA_KEY: []
+    }
+
+    defconf_id = job + "-" + kernel + "-" + defconfig
+    defconf_doc = find_one(
+        database[DEFCONFIG_COLLECTION],
+        defconf_id,
+        fields=BOOT_DEFCONFIG_SEARCH_FIELDS
+    )
+
+    if defconf_doc:
+        defconf_doc_get = defconf_doc.get
+        combined_values[DIRNAME_KEY] = defconf_doc_get(DIRNAME_KEY)
+        combined_values[BISECT_DEFCONFIG_CREATED_KEY] = defconf_doc_get(
+            CREATED_KEY
+        )
+        combined_values[BISECT_DEFCONFIG_ARCHITECTURE_KEY] = defconf_doc_get(
+            ARCHITECTURE_KEY
+        )
+        combined_values[BISECT_DEFCONFIG_STATUS_KEY] = defconf_doc_get(
+            STATUS_KEY
+        )
+        combined_values[BISECT_DEFCONFIG_METADATA_KEY] = defconf_doc_get(
+            METADATA_KEY
+        )
+
+    return combined_values
+
+
+def execute_boot_bisection(doc_id, db_options):
+    """Perform a bisect-like on the provided boot report.
+
+    It searches all the previous boot reports starting from the provided one
+    until it finds one whose boot passed. After that, it looks for all the
+    defconfig reports and combines the value into a single data structure.
+
+    :param doc_id: The boot document ID.
+    :type doc_id: str
+    :param db_options: The mongodb database connection parameters.
+    :type db_options: dict
+    :return A numeric value for the result status and a list dictionaries.
+    """
+    database = get_db_connection(db_options)
+
+    start_doc = find_one(
+        database[BOOT_COLLECTION], doc_id, fields=BOOT_SEARCH_FIELDS
+    )
+
+    result = []
+    code = 200
+
+    if start_doc:
+        start_doc_get = start_doc.get
+        spec = {
+            BOARD_KEY: start_doc_get(BOARD_KEY),
+            DEFCONFIG_KEY: start_doc_get(DEFCONFIG_KEY),
+            JOB_KEY: start_doc_get(JOB_KEY),
+            CREATED_KEY: {
+                "$lt": start_doc_get(CREATED_KEY)
+            }
+        }
+
+        all_valid_docs = [(start_doc, db_options)]
+
+        all_prev_docs = find(
+            database[BOOT_COLLECTION],
+            0,
+            0,
+            spec=spec,
+            fields=BOOT_SEARCH_FIELDS,
+            sort=BOOT_SORT
+        )
+
+        if all_prev_docs:
+            for prev_doc in all_prev_docs:
+                if prev_doc[STATUS_KEY] == PASS_STATUS:
+                    all_valid_docs.append((prev_doc, db_options))
+                    break
+                all_valid_docs.append((prev_doc, db_options))
+
+        func = _combine_defconfig_values
+        # TODO: we have to save the result in a new collection.
+        result = [
+            j_load(j_dump(func(doc, opt), default=default))
+            for doc, opt in all_valid_docs
+        ]
+    else:
+        code = 404
+        result = None
+
+    return code, result