bisect: Add save feature on bisect.

  * Bisect data is now saved.
  * Add bisect data model.
  * Add tests.
  * Refactor save method to return the document ID.

Change-Id: I58146d5136538cf9851aee06d8896726ab3b706b

3 files changed, 178 insertions, 47 deletions

diff --git a/app/utils/bisect/__init__.py b/app/utils/bisect/__init__.py
index 4f2a913..28d08a7 100644
--- a/app/utils/bisect/__init__.py
+++ b/app/utils/bisect/__init__.py
@@ -13,39 +13,49 @@
 
 """All the bisect operations that the app can perform."""
 
+from bson import tz_util
 from bson.json_util import default
+from datetime import datetime
 from json import (
     dumps as j_dump,
     loads as j_load,
 )
 from pymongo import DESCENDING
+from types import DictionaryType
 
 from models import (
-    BOOT_COLLECTION,
-    DEFCONFIG_COLLECTION,
-    BOARD_KEY,
-    CREATED_KEY,
-    DEFCONFIG_KEY,
-    JOB_KEY,
-    JOB_ID_KEY,
-    KERNEL_KEY,
-    METADATA_KEY,
-    STATUS_KEY,
     ARCHITECTURE_KEY,
-    DIRNAME_KEY,
-    PASS_STATUS,
     BISECT_BOOT_CREATED_KEY,
     BISECT_BOOT_METADATA_KEY,
     BISECT_BOOT_STATUS_KEY,
+    BISECT_COLLECTION,
+    BISECT_DEFCONFIG_ARCHITECTURE_KEY,
     BISECT_DEFCONFIG_CREATED_KEY,
     BISECT_DEFCONFIG_METADATA_KEY,
     BISECT_DEFCONFIG_STATUS_KEY,
-    BISECT_DEFCONFIG_ARCHITECTURE_KEY,
+    BOARD_KEY,
+    BOOT_COLLECTION,
+    CREATED_KEY,
+    DEFCONFIG_COLLECTION,
+    DEFCONFIG_KEY,
+    DIRNAME_KEY,
+    DOC_ID_KEY,
+    GIT_COMMIT_KEY,
+    GIT_URL_KEY,
+    JOB_ID_KEY,
+    JOB_KEY,
+    KERNEL_KEY,
+    METADATA_KEY,
+    PASS_STATUS,
+    STATUS_KEY,
 )
+from models.bisect import BootBisectDocument
+from utils import LOG
 from utils.db import (
     find,
     find_one,
     get_db_connection,
+    save,
 )
 
 
@@ -102,7 +112,7 @@ def _combine_defconfig_values(boot_doc, db_options):
         BISECT_DEFCONFIG_CREATED_KEY: "",
         BISECT_DEFCONFIG_ARCHITECTURE_KEY: "",
         BISECT_DEFCONFIG_STATUS_KEY: "",
-        BISECT_DEFCONFIG_METADATA_KEY: []
+        BISECT_DEFCONFIG_METADATA_KEY: {}
     }
 
     defconf_id = job + "-" + kernel + "-" + defconfig
@@ -145,51 +155,116 @@ def execute_boot_bisection(doc_id, db_options):
     :return A numeric value for the result status and a list dictionaries.
     """
     database = get_db_connection(db_options)
+    result = []
+    code = 200
 
     start_doc = find_one(
         database[BOOT_COLLECTION], doc_id, fields=BOOT_SEARCH_FIELDS
     )
 
-    result = []
-    code = 200
-
-    if start_doc:
+    if all([start_doc, isinstance(start_doc, DictionaryType)]):
         start_doc_get = start_doc.get
-        spec = {
-            BOARD_KEY: start_doc_get(BOARD_KEY),
-            DEFCONFIG_KEY: start_doc_get(DEFCONFIG_KEY),
-            JOB_KEY: start_doc_get(JOB_KEY),
-            CREATED_KEY: {
-                "$lt": start_doc_get(CREATED_KEY)
+
+        if start_doc_get(STATUS_KEY) == PASS_STATUS:
+            code = 400
+            result = None
+        else:
+            bisect_doc = BootBisectDocument(doc_id)
+            bisect_doc.job = start_doc_get(JOB_ID_KEY)
+            bisect_doc.created_on = datetime.now(tz=tz_util.utc)
+            bisect_doc.board = start_doc_get(BOARD_KEY)
+
+            spec = {
+                BOARD_KEY: start_doc_get(BOARD_KEY),
+                DEFCONFIG_KEY: start_doc_get(DEFCONFIG_KEY),
+                JOB_KEY: start_doc_get(JOB_KEY),
+                CREATED_KEY: {
+                    "$lt": start_doc_get(CREATED_KEY)
+                }
             }
-        }
 
-        all_valid_docs = [(start_doc, db_options)]
+            # The function to apply to each boot document to find its defconfig
+            # one and combine the values.
+            func = _combine_defconfig_values
 
-        all_prev_docs = find(
-            database[BOOT_COLLECTION],
-            0,
-            0,
-            spec=spec,
-            fields=BOOT_SEARCH_FIELDS,
-            sort=BOOT_SORT
-        )
+            bad_doc = func(start_doc, db_options)
+            bad_doc_meta = bad_doc[BISECT_DEFCONFIG_METADATA_KEY].get
+
+            bisect_doc.bad_commit_date = bad_doc[BISECT_DEFCONFIG_CREATED_KEY]
+            bisect_doc.bad_commit = bad_doc_meta(GIT_COMMIT_KEY)
+            bisect_doc.bad_commit_url = bad_doc_meta(GIT_URL_KEY)
+
+            all_valid_docs = [bad_doc]
+
+            # Search through all the previous boot reports, until one that
+            # passed is found, and combine them with their defconfig document.
+            all_prev_docs = find(
+                database[BOOT_COLLECTION],
+                0,
+                0,
+                spec=spec,
+                fields=BOOT_SEARCH_FIELDS,
+                sort=BOOT_SORT
+            )
 
-        if all_prev_docs:
-            for prev_doc in all_prev_docs:
-                if prev_doc[STATUS_KEY] == PASS_STATUS:
-                    all_valid_docs.append((prev_doc, db_options))
-                    break
-                all_valid_docs.append((prev_doc, db_options))
-
-        func = _combine_defconfig_values
-        # TODO: we have to save the result in a new collection.
-        result = [
-            j_load(j_dump(func(doc, opt), default=default))
-            for doc, opt in all_valid_docs
-        ]
+            if all_prev_docs:
+                all_valid_docs.extend(
+                    [
+                        func(doc, db_options)
+                        for doc in _get_docs_until_pass(all_prev_docs)
+                    ]
+                )
+                # The last doc should be the good one, in case it is, add the
+                # values to the bisect_doc.
+                good_doc = all_valid_docs[-1]
+                if good_doc[BISECT_BOOT_STATUS_KEY] == PASS_STATUS:
+                    good_doc_meta = good_doc[BISECT_DEFCONFIG_METADATA_KEY].get
+                    bisect_doc.good_commit = good_doc_meta(GIT_COMMIT_KEY)
+                    bisect_doc.good_commit_url = good_doc_meta(GIT_URL_KEY)
+                    bisect_doc.good_commit_date = \
+                        good_doc[BISECT_DEFCONFIG_CREATED_KEY]
+
+            # Store everything in the bisect_data list of the bisect_doc.
+            bisect_doc.bisect_data = all_valid_docs
+
+            return_code, saved_id = save(database, bisect_doc, manipulate=True)
+            if return_code == 201:
+                bisect_doc._id = saved_id
+            else:
+                LOG.error("Error savind bisect data %s", doc_id)
+            result = [j_load(j_dump(bisect_doc.to_dict(), default=default))]
     else:
         code = 404
         result = None
 
     return code, result
+
+
+def _get_docs_until_pass(doc_list):
+    """Iterate through the docs until one that passed is found.
+
+    Yield all documents until one that passed is found, returning it as well
+    and breaking the loop.
+
+    :param doc_list: A list of documents (`BaseDocument`) as dictionaries.
+    :type doc_list: list
+    """
+    for doc in doc_list:
+        if doc[STATUS_KEY] == PASS_STATUS:
+            yield doc
+            break
+        yield doc
+
+
+def _find_bisect(doc_id, database):
+    """Look for an already available bisect object in the database.
+
+    :param doc_id: The ID of the document to search.
+    :type doc_id: str
+    :param collection: The collection where to search the document.
+    :type collection: str
+    :param database: The connection to the database.
+    :type database: `pymongo.MongoClient`
+    :return The document found or None.
+    """
+    return find_one(database[BISECT_COLLECTION], doc_id, field=DOC_ID_KEY)
diff --git a/app/utils/bisect/tests/__init__.py b/app/utils/bisect/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/app/utils/bisect/tests/__init__.py
diff --git a/app/utils/bisect/tests/test_bisect.py b/app/utils/bisect/tests/test_bisect.py
new file mode 100644
index 0000000..f211048
--- /dev/null
+++ b/app/utils/bisect/tests/test_bisect.py
@@ -0,0 +1,56 @@
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import unittest
+
+from utils.bisect import _get_docs_until_pass
+
+
+class BisectUtilsTest(unittest.TestCase):
+
+    def test_get_docs_until_pass_no_pass(self):
+        doc_list = [
+            {"status": "foo", "id": 1},
+            {"status": "foo", "id": 2},
+            {"status": "foo", "id": 3},
+            {"status": "foo", "id": 4},
+            {"status": "foo", "id": 5},
+        ]
+
+        retrieved_list = [doc for doc in _get_docs_until_pass(doc_list)]
+        self.assertListEqual(doc_list, retrieved_list)
+
+    def test_get_docs_until_pass_with_pass_last(self):
+        doc_list = [
+            {"status": "foo", "id": 1},
+            {"status": "foo", "id": 2},
+            {"status": "foo", "id": 3},
+            {"status": "foo", "id": 4},
+            {"status": "PASS", "id": 5},
+        ]
+
+        retrieved_list = [doc for doc in _get_docs_until_pass(doc_list)]
+        self.assertListEqual(doc_list, retrieved_list)
+
+    def test_get_docs_until_pass_with_pass(self):
+        doc_list = [
+            {"status": "foo", "id": 1},
+            {"status": "foo", "id": 2},
+            {"status": "PASS", "id": 3},
+            {"status": "foo", "id": 4},
+            {"status": "foo", "id": 5},
+        ]
+
+        retrieved_list = [doc for doc in _get_docs_until_pass(doc_list)]
+        self.assertEqual(len(retrieved_list), 3)
+        self.assertListEqual(doc_list[:3], retrieved_list)