diff options
-rw-r--r-- | app/handlers/bisect.py | 26 | ||||
-rw-r--r-- | app/handlers/tests/test_bisect_handler.py | 17 | ||||
-rw-r--r-- | app/models/__init__.py | 9 | ||||
-rw-r--r-- | app/models/base.py | 2 | ||||
-rw-r--r-- | app/models/bisect.py | 197 | ||||
-rw-r--r-- | app/models/tests/test_bisect_model.py | 125 | ||||
-rw-r--r-- | app/models/tests/test_boot_model.py | 2 | ||||
-rw-r--r-- | app/tests/__init__.py | 2 | ||||
-rw-r--r-- | app/utils/bisect/__init__.py | 169 | ||||
-rw-r--r-- | app/utils/bisect/tests/__init__.py | 0 | ||||
-rw-r--r-- | app/utils/bisect/tests/test_bisect.py | 56 | ||||
-rw-r--r-- | app/utils/bootimport.py | 5 | ||||
-rw-r--r-- | app/utils/db.py | 16 | ||||
-rw-r--r-- | app/utils/docimport.py | 6 |
14 files changed, 565 insertions, 67 deletions
diff --git a/app/handlers/bisect.py b/app/handlers/bisect.py index a0e5b97..3986469 100644 --- a/app/handlers/bisect.py +++ b/app/handlers/bisect.py @@ -23,8 +23,12 @@ from handlers.common import NOT_VALID_TOKEN from handlers.response import HandlerResponse from taskqueue.tasks import boot_bisect +from utils.db import find_one + from models import ( + BISECT_COLLECTION, BOOT_COLLECTION, + DOC_ID_KEY, ) BISECT_COLLECTIONS = [ @@ -49,6 +53,10 @@ class BisectHandler(BaseHandler): ) ) + @property + def collection(self): + return BISECT_COLLECTION + def execute_get(self, *args, **kwargs): """This is the actual GET operation. @@ -57,12 +65,20 @@ class BisectHandler(BaseHandler): """ response = None + # TODO: handle fields data structure. if self.validate_req_token("GET"): if kwargs: collection = kwargs.get("collection", None) doc_id = kwargs.get("id", None) if all([collection, doc_id]): - response = self._get_bisect(collection, doc_id) + bisect_result = find_one( + self.db[self.collection], doc_id, field=DOC_ID_KEY + ) + if bisect_result: + response = HandlerResponse(200) + response.result = bisect_result + else: + response = self._get_bisect(collection, doc_id) else: response = HandlerResponse(400) else: @@ -85,10 +101,10 @@ class BisectHandler(BaseHandler): response = None if collection in BISECT_COLLECTIONS: + db_options = self.settings["dboptions"] + if collection == BOOT_COLLECTION: - response = self.execute_boot_bisect( - doc_id, self.settings["dboptions"] - ) + response = self.execute_boot_bisect(doc_id, db_options) else: response = HandlerResponse(400) response.reason = ( @@ -116,4 +132,6 @@ class BisectHandler(BaseHandler): response.status_code, response.result = result.get() if response.status_code == 404: response.reason = "Boot report not found" + elif response.status_code == 400: + response.reason = "Boot report cannot be bisected: is it failed?" return response diff --git a/app/handlers/tests/test_bisect_handler.py b/app/handlers/tests/test_bisect_handler.py index c052b19..0ed9c6b 100644 --- a/app/handlers/tests/test_bisect_handler.py +++ b/app/handlers/tests/test_bisect_handler.py @@ -105,3 +105,20 @@ class TestBisectHandler(testing.AsyncHTTPTestCase, testing.LogTrapTestCase): response = self.fetch('/api/bisect/boot/foo', headers=headers) self.assertEqual(response.code, 404) + + def test_boot_bisect_no_faile(self): + headers = {'Authorization': 'foo'} + + self.task_return_value.get.return_value = 400, None + + response = self.fetch('/api/bisect/boot/foo', headers=headers) + self.assertEqual(response.code, 400) + + @patch("handlers.bisect.find_one") + def test_boot_bisect_with_result(self, mocked_find): + headers = {'Authorization': 'foo'} + + mocked_find.return_value = [{"foo": "bar"}] + + response = self.fetch('/api/bisect/boot/foo', headers=headers) + self.assertEqual(response.code, 200) diff --git a/app/models/__init__.py b/app/models/__init__.py index 78a69ab..213868d 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -30,6 +30,7 @@ CROSS_COMPILE_KEY = 'cross_compile' DATE_RANGE_KEY = 'date_range' DEFCONFIG_KEY = 'defconfig' DIRNAME_KEY = 'dirname' +DOC_ID_KEY = 'doc_id' DTB_ADDR_KEY = 'dtb_addr' DTB_KEY = 'dtb' EMAIL_KEY = 'email' @@ -104,6 +105,7 @@ DEFCONFIG_COLLECTION = 'defconfig' JOB_COLLECTION = 'job' SUBSCRIPTION_COLLECTION = 'subscription' TOKEN_COLLECTION = 'api-token' +BISECT_COLLECTION = 'bisect' # Bisect values. BISECT_BOOT_STATUS_KEY = 'boot_status' @@ -113,3 +115,10 @@ BISECT_DEFCONFIG_STATUS_KEY = 'defconfig_status' BISECT_DEFCONFIG_CREATED_KEY = 'defconfig_created' BISECT_DEFCONFIG_METADATA_KEY = 'defconfig_metadata' BISECT_DEFCONFIG_ARCHITECTURE_KEY = 'defconfig_arch' +BISECT_DATA_KEY = 'bisect_data' +BISECT_GOOD_COMMIT_KEY = 'good_commit' +BISECT_BAD_COMMIT_KEY = 'bad_commit' +BISECT_GOOD_COMMIT_DATE = 'good_commit_date' +BISECT_BAD_COMMIT_DATE = 'bad_commit_date' +BISECT_GOOD_COMMIT_URL = 'good_commit_url' +BISECT_BAD_COMMIT_URL = 'bad_commit_url' diff --git a/app/models/base.py b/app/models/base.py index 3272507..8c42605 100644 --- a/app/models/base.py +++ b/app/models/base.py @@ -1,5 +1,3 @@ -# Copyright (C) 2014 Linaro Ltd. -# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of the diff --git a/app/models/bisect.py b/app/models/bisect.py new file mode 100644 index 0000000..86b3aca --- /dev/null +++ b/app/models/bisect.py @@ -0,0 +1,197 @@ +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +"""Bisect mongodb document models.""" + +from models import ( + BISECT_BAD_COMMIT_DATE, + BISECT_BAD_COMMIT_KEY, + BISECT_BAD_COMMIT_URL, + BISECT_COLLECTION, + BISECT_DATA_KEY, + BISECT_GOOD_COMMIT_DATE, + BISECT_GOOD_COMMIT_KEY, + BISECT_GOOD_COMMIT_URL, + BOARD_KEY, + CREATED_KEY, + DOC_ID_KEY, + ID_KEY, + JOB_KEY, +) +from models.base import BaseDocument + + +class BisectDocument(BaseDocument): + """The bisect document model class.""" + + def __init__(self, name): + super(BisectDocument, self).__init__(name) + + self._id = None + self._job = None + self._bisect_data = [] + self._bad_commit = None + self._good_commit = None + self._bad_commit_date = None + self._good_commit_date = None + self._bad_commit_url = None + self._good_commit_url = None + + @property + def collection(self): + """The name of this document collection. + + Where document of this kind will be stored. + """ + return BISECT_COLLECTION + + @property + def id(self): + """The ID of this object in the database. + + This value should be returned by mongodb. + """ + return self._id + + @property + def doc_id(self): + """The interl doc ID.""" + return self._name + + @id.setter + def id(self, value): + """Set the ID of this object.""" + self._id = value + + @property + def job(self): + """The job this document is part of.""" + return self._job + + @job.setter + def job(self, value): + """Set the job this document is part of.""" + self._job = value + + @property + def bad_commit_date(self): + """The date of the bad commit.""" + return self._bad_commit_date + + @bad_commit_date.setter + def bad_commit_date(self, value): + """Set the date of the bad commit.""" + self._bad_commit_date = value + + @property + def bad_commit(self): + """The bad commit hash value.""" + return self._bad_commit + + @bad_commit.setter + def bad_commit(self, value): + """Set the bad commit hash value.""" + self._bad_commit = value + + @property + def bad_commit_url(self): + """The URL of the bad commit.""" + return self._bad_commit_url + + @bad_commit_url.setter + def bad_commit_url(self, value): + """Set the URL of the bad commit.""" + self._bad_commit_url = value + + @property + def good_commit(self): + """The good commit hash value.""" + return self._good_commit + + @good_commit.setter + def good_commit(self, value): + """Set the good commit hash value.""" + self._good_commit = value + + @property + def good_commit_date(self): + """The date of the good commit.""" + return self._good_commit_date + + @good_commit_date.setter + def good_commit_date(self, value): + """Set the date of the good commit.""" + self._good_commit_date = value + + @property + def good_commit_url(self): + """The URL of the good commit.""" + return self._good_commit_url + + @good_commit_url.setter + def good_commit_url(self, value): + """Set the URL of the good commit.""" + self._good_commit_url = value + + @property + def bisect_data(self): + """Get all the bisect data, ranging from the bad to the good commit.""" + return self._bisect_data + + @bisect_data.setter + def bisect_data(self, value): + """Set the bisect data.""" + self._bisect_data = value + + def to_dict(self): + bisect_dict = { + CREATED_KEY: self._created_on, + JOB_KEY: self._job, + DOC_ID_KEY: self._name, + BISECT_DATA_KEY: self._bisect_data, + BISECT_GOOD_COMMIT_KEY: self._good_commit, + BISECT_GOOD_COMMIT_DATE: self._good_commit_date, + BISECT_GOOD_COMMIT_URL: self._good_commit_url, + BISECT_BAD_COMMIT_KEY: self._bad_commit, + BISECT_BAD_COMMIT_DATE: self._bad_commit_date, + BISECT_BAD_COMMIT_URL: self._bad_commit_url, + } + + if self._id: + bisect_dict[ID_KEY] = self._id + + return bisect_dict + + +class BootBisectDocument(BisectDocument): + """The bisect document class for boot bisection.""" + + def __init__(self, name): + super(BootBisectDocument, self).__init__(name) + + self._board = None + + @property + def board(self): + """The board this document belongs to.""" + return self._board + + @board.setter + def board(self, value): + """Set the board name this document belongs to.""" + self._board = value + + def to_dict(self): + boot_b_dict = super(BootBisectDocument, self).to_dict() + boot_b_dict[BOARD_KEY] = self._board + return boot_b_dict diff --git a/app/models/tests/test_bisect_model.py b/app/models/tests/test_bisect_model.py new file mode 100644 index 0000000..5f5b629 --- /dev/null +++ b/app/models/tests/test_bisect_model.py @@ -0,0 +1,125 @@ +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import unittest + +from models.base import BaseDocument +from models.bisect import BisectDocument, BootBisectDocument + + +class TestBisectModel(unittest.TestCase): + + def test_bisect_base_document(self): + bisect_doc = BisectDocument("foo") + self.assertIsInstance(bisect_doc, BaseDocument) + + def test_boot_bisect_document(self): + bisect_doc = BootBisectDocument("bar") + self.assertIsInstance(bisect_doc, BisectDocument) + self.assertIsInstance(bisect_doc, BaseDocument) + + def test_bisect_base_document_collection(self): + bisect_doc = BisectDocument("foo") + self.assertEqual(bisect_doc.collection, "bisect") + + def test_bisect_boot_document_collection(self): + bisect_doc = BootBisectDocument("foo") + self.assertEqual(bisect_doc.collection, "bisect") + + def test_bisect_base_to_dict(self): + bisect_doc = BisectDocument("foo") + + expected = { + "created_on": None, + "job": None, + "doc_id": "foo", + "bisect_data": [], + "good_commit": None, + "good_commit_date": None, + "good_commit_url": None, + "bad_commit": None, + "bad_commit_date": None, + "bad_commit_url": None, + } + self.assertDictEqual(expected, bisect_doc.to_dict()) + + def test_bisect_base_to_dict_with_id(self): + bisect_doc = BisectDocument("foo") + bisect_doc.id = "bar" + + expected = { + "_id": "bar", + "created_on": None, + "job": None, + "doc_id": "foo", + "bisect_data": [], + "good_commit": None, + "good_commit_date": None, + "good_commit_url": None, + "bad_commit": None, + "bad_commit_date": None, + "bad_commit_url": None, + } + self.assertDictEqual(expected, bisect_doc.to_dict()) + + def test_bisect_boot_to_dict(self): + bisect_doc = BootBisectDocument("foo") + bisect_doc.id = "bar" + bisect_doc.board = "baz" + + expected = { + "_id": "bar", + "board": "baz", + "created_on": None, + "job": None, + "doc_id": "foo", + "bisect_data": [], + "good_commit": None, + "good_commit_date": None, + "good_commit_url": None, + "bad_commit": None, + "bad_commit_date": None, + "bad_commit_url": None, + } + self.assertDictEqual(expected, bisect_doc.to_dict()) + + def test_bisect_base_properties(self): + bisect_doc = BootBisectDocument("foo") + bisect_doc.id = "bar" + bisect_doc.created_on = "now" + bisect_doc.job = "fooz" + bisect_doc.bisect_data = [1, 2, 3] + bisect_doc.good_commit = "1" + bisect_doc.good_commit_date = "now" + bisect_doc.good_commit_url = "url" + bisect_doc.bad_commit = "2" + bisect_doc.bad_commit_date = "now" + bisect_doc.bad_commit_url = "url" + + self.assertEqual(bisect_doc.id, "bar") + self.assertEqual(bisect_doc.doc_id, "foo") + self.assertEqual(bisect_doc.created_on, "now") + self.assertEqual(bisect_doc.job, "fooz") + self.assertEqual(bisect_doc.bisect_data, [1, 2, 3]) + self.assertEqual(bisect_doc.good_commit, "1") + self.assertEqual(bisect_doc.good_commit_date, "now") + self.assertEqual(bisect_doc.good_commit_url, "url") + self.assertEqual(bisect_doc.bad_commit, "2") + self.assertEqual(bisect_doc.bad_commit_date, "now") + self.assertEqual(bisect_doc.bad_commit_url, "url") + + def test_bisect_boot_properties(self): + bisect_doc = BootBisectDocument("foo") + bisect_doc.board = "bar" + + self.assertEqual(bisect_doc.board, "bar") diff --git a/app/models/tests/test_boot_model.py b/app/models/tests/test_boot_model.py index c04230c..b45e3b8 100644 --- a/app/models/tests/test_boot_model.py +++ b/app/models/tests/test_boot_model.py @@ -1,5 +1,3 @@ -# Copyright (C) 2014 Linaro Ltd. -# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of the diff --git a/app/tests/__init__.py b/app/tests/__init__.py index 400d11f..1a6c5e8 100644 --- a/app/tests/__init__.py +++ b/app/tests/__init__.py @@ -30,10 +30,12 @@ def test_modules(): 'handlers.tests.test_job_handler', 'handlers.tests.test_subscription_handler', 'handlers.tests.test_token_handler', + 'models.tests.test_bisect_model', 'models.tests.test_boot_model', 'models.tests.test_models', 'models.tests.test_token_model', 'utils.batch.tests.test_batch_common', + 'utils.bisect.tests.test_bisect', 'utils.tests.test_bootimport', 'utils.tests.test_docimport', 'utils.tests.test_meta_parser', diff --git a/app/utils/bisect/__init__.py b/app/utils/bisect/__init__.py index 4f2a913..28d08a7 100644 --- a/app/utils/bisect/__init__.py +++ b/app/utils/bisect/__init__.py @@ -13,39 +13,49 @@ """All the bisect operations that the app can perform.""" +from bson import tz_util from bson.json_util import default +from datetime import datetime from json import ( dumps as j_dump, loads as j_load, ) from pymongo import DESCENDING +from types import DictionaryType from models import ( - BOOT_COLLECTION, - DEFCONFIG_COLLECTION, - BOARD_KEY, - CREATED_KEY, - DEFCONFIG_KEY, - JOB_KEY, - JOB_ID_KEY, - KERNEL_KEY, - METADATA_KEY, - STATUS_KEY, ARCHITECTURE_KEY, - DIRNAME_KEY, - PASS_STATUS, BISECT_BOOT_CREATED_KEY, BISECT_BOOT_METADATA_KEY, BISECT_BOOT_STATUS_KEY, + BISECT_COLLECTION, + BISECT_DEFCONFIG_ARCHITECTURE_KEY, BISECT_DEFCONFIG_CREATED_KEY, BISECT_DEFCONFIG_METADATA_KEY, BISECT_DEFCONFIG_STATUS_KEY, - BISECT_DEFCONFIG_ARCHITECTURE_KEY, + BOARD_KEY, + BOOT_COLLECTION, + CREATED_KEY, + DEFCONFIG_COLLECTION, + DEFCONFIG_KEY, + DIRNAME_KEY, + DOC_ID_KEY, + GIT_COMMIT_KEY, + GIT_URL_KEY, + JOB_ID_KEY, + JOB_KEY, + KERNEL_KEY, + METADATA_KEY, + PASS_STATUS, + STATUS_KEY, ) +from models.bisect import BootBisectDocument +from utils import LOG from utils.db import ( find, find_one, get_db_connection, + save, ) @@ -102,7 +112,7 @@ def _combine_defconfig_values(boot_doc, db_options): BISECT_DEFCONFIG_CREATED_KEY: "", BISECT_DEFCONFIG_ARCHITECTURE_KEY: "", BISECT_DEFCONFIG_STATUS_KEY: "", - BISECT_DEFCONFIG_METADATA_KEY: [] + BISECT_DEFCONFIG_METADATA_KEY: {} } defconf_id = job + "-" + kernel + "-" + defconfig @@ -145,51 +155,116 @@ def execute_boot_bisection(doc_id, db_options): :return A numeric value for the result status and a list dictionaries. """ database = get_db_connection(db_options) + result = [] + code = 200 start_doc = find_one( database[BOOT_COLLECTION], doc_id, fields=BOOT_SEARCH_FIELDS ) - result = [] - code = 200 - - if start_doc: + if all([start_doc, isinstance(start_doc, DictionaryType)]): start_doc_get = start_doc.get - spec = { - BOARD_KEY: start_doc_get(BOARD_KEY), - DEFCONFIG_KEY: start_doc_get(DEFCONFIG_KEY), - JOB_KEY: start_doc_get(JOB_KEY), - CREATED_KEY: { - "$lt": start_doc_get(CREATED_KEY) + + if start_doc_get(STATUS_KEY) == PASS_STATUS: + code = 400 + result = None + else: + bisect_doc = BootBisectDocument(doc_id) + bisect_doc.job = start_doc_get(JOB_ID_KEY) + bisect_doc.created_on = datetime.now(tz=tz_util.utc) + bisect_doc.board = start_doc_get(BOARD_KEY) + + spec = { + BOARD_KEY: start_doc_get(BOARD_KEY), + DEFCONFIG_KEY: start_doc_get(DEFCONFIG_KEY), + JOB_KEY: start_doc_get(JOB_KEY), + CREATED_KEY: { + "$lt": start_doc_get(CREATED_KEY) + } } - } - all_valid_docs = [(start_doc, db_options)] + # The function to apply to each boot document to find its defconfig + # one and combine the values. + func = _combine_defconfig_values - all_prev_docs = find( - database[BOOT_COLLECTION], - 0, - 0, - spec=spec, - fields=BOOT_SEARCH_FIELDS, - sort=BOOT_SORT - ) + bad_doc = func(start_doc, db_options) + bad_doc_meta = bad_doc[BISECT_DEFCONFIG_METADATA_KEY].get + + bisect_doc.bad_commit_date = bad_doc[BISECT_DEFCONFIG_CREATED_KEY] + bisect_doc.bad_commit = bad_doc_meta(GIT_COMMIT_KEY) + bisect_doc.bad_commit_url = bad_doc_meta(GIT_URL_KEY) + + all_valid_docs = [bad_doc] + + # Search through all the previous boot reports, until one that + # passed is found, and combine them with their defconfig document. + all_prev_docs = find( + database[BOOT_COLLECTION], + 0, + 0, + spec=spec, + fields=BOOT_SEARCH_FIELDS, + sort=BOOT_SORT + ) - if all_prev_docs: - for prev_doc in all_prev_docs: - if prev_doc[STATUS_KEY] == PASS_STATUS: - all_valid_docs.append((prev_doc, db_options)) - break - all_valid_docs.append((prev_doc, db_options)) - - func = _combine_defconfig_values - # TODO: we have to save the result in a new collection. - result = [ - j_load(j_dump(func(doc, opt), default=default)) - for doc, opt in all_valid_docs - ] + if all_prev_docs: + all_valid_docs.extend( + [ + func(doc, db_options) + for doc in _get_docs_until_pass(all_prev_docs) + ] + ) + # The last doc should be the good one, in case it is, add the + # values to the bisect_doc. + good_doc = all_valid_docs[-1] + if good_doc[BISECT_BOOT_STATUS_KEY] == PASS_STATUS: + good_doc_meta = good_doc[BISECT_DEFCONFIG_METADATA_KEY].get + bisect_doc.good_commit = good_doc_meta(GIT_COMMIT_KEY) + bisect_doc.good_commit_url = good_doc_meta(GIT_URL_KEY) + bisect_doc.good_commit_date = \ + good_doc[BISECT_DEFCONFIG_CREATED_KEY] + + # Store everything in the bisect_data list of the bisect_doc. + bisect_doc.bisect_data = all_valid_docs + + return_code, saved_id = save(database, bisect_doc, manipulate=True) + if return_code == 201: + bisect_doc._id = saved_id + else: + LOG.error("Error savind bisect data %s", doc_id) + result = [j_load(j_dump(bisect_doc.to_dict(), default=default))] else: code = 404 result = None return code, result + + +def _get_docs_until_pass(doc_list): + """Iterate through the docs until one that passed is found. + + Yield all documents until one that passed is found, returning it as well + and breaking the loop. + + :param doc_list: A list of documents (`BaseDocument`) as dictionaries. + :type doc_list: list + """ + for doc in doc_list: + if doc[STATUS_KEY] == PASS_STATUS: + yield doc + break + yield doc + + +def _find_bisect(doc_id, database): + """Look for an already available bisect object in the database. + + :param doc_id: The ID of the document to search. + :type doc_id: str + :param collection: The collection where to search the document. + :type collection: str + :param database: The connection to the database. + :type database: `pymongo.MongoClient` + :return The document found or None. + """ + return find_one(database[BISECT_COLLECTION], doc_id, field=DOC_ID_KEY) diff --git a/app/utils/bisect/tests/__init__.py b/app/utils/bisect/tests/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/app/utils/bisect/tests/__init__.py diff --git a/app/utils/bisect/tests/test_bisect.py b/app/utils/bisect/tests/test_bisect.py new file mode 100644 index 0000000..f211048 --- /dev/null +++ b/app/utils/bisect/tests/test_bisect.py @@ -0,0 +1,56 @@ +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import unittest + +from utils.bisect import _get_docs_until_pass + + +class BisectUtilsTest(unittest.TestCase): + + def test_get_docs_until_pass_no_pass(self): + doc_list = [ + {"status": "foo", "id": 1}, + {"status": "foo", "id": 2}, + {"status": "foo", "id": 3}, + {"status": "foo", "id": 4}, + {"status": "foo", "id": 5}, + ] + + retrieved_list = [doc for doc in _get_docs_until_pass(doc_list)] + self.assertListEqual(doc_list, retrieved_list) + + def test_get_docs_until_pass_with_pass_last(self): + doc_list = [ + {"status": "foo", "id": 1}, + {"status": "foo", "id": 2}, + {"status": "foo", "id": 3}, + {"status": "foo", "id": 4}, + {"status": "PASS", "id": 5}, + ] + + retrieved_list = [doc for doc in _get_docs_until_pass(doc_list)] + self.assertListEqual(doc_list, retrieved_list) + + def test_get_docs_until_pass_with_pass(self): + doc_list = [ + {"status": "foo", "id": 1}, + {"status": "foo", "id": 2}, + {"status": "PASS", "id": 3}, + {"status": "foo", "id": 4}, + {"status": "foo", "id": 5}, + ] + + retrieved_list = [doc for doc in _get_docs_until_pass(doc_list)] + self.assertEqual(len(retrieved_list), 3) + self.assertListEqual(doc_list[:3], retrieved_list) diff --git a/app/utils/bootimport.py b/app/utils/bootimport.py index 1bb6240..49202b9 100644 --- a/app/utils/bootimport.py +++ b/app/utils/bootimport.py @@ -85,10 +85,7 @@ def import_and_save_boot(json_obj, db_options, base_path=BASE_PATH): docs = parse_boot_from_json(json_obj, base_path) if docs: - try: - save(database, docs) - finally: - database.connection.disconnect() + save(database, docs) else: LOG.info("No boot log imported") diff --git a/app/utils/db.py b/app/utils/db.py index 41ea5f9..243daf2 100644 --- a/app/utils/db.py +++ b/app/utils/db.py @@ -167,16 +167,21 @@ def count(collection): return collection.count() -def save(database, documents): +def save(database, documents, manipulate=False): """Save documents into the database. :param database: The database where to save. :param documents: The document to save, can be a list or a single document: the type of each document must be: BaseDocument or a subclass. :type list, BaseDocument - :return 201 if the save has success, 500 in case of an error. + :param manipulate: If the passed documents have to be manipulated by + mongodb. Default to False. + :type manipulate: bool + :return 201 if the save has success, 500 in case of an error. If manipulate + is True, return also the mongodb created ID. """ ret_value = 201 + doc_id = [] if not isinstance(documents, types.ListType): documents = [documents] @@ -192,13 +197,18 @@ def save(database, documents): continue try: - database[document.collection].save(to_save, manipulate=False) + doc_id = database[document.collection].save( + to_save, manipulate=manipulate + ) except OperationFailure, ex: LOG.error("Error saving the following document: %s", to_save.name) LOG.exception(str(ex)) ret_value = 500 break + if manipulate: + ret_value = (ret_value, doc_id) + return ret_value diff --git a/app/utils/docimport.py b/app/utils/docimport.py index 2763a42..f6ae929 100644 --- a/app/utils/docimport.py +++ b/app/utils/docimport.py @@ -78,11 +78,7 @@ def import_and_save_job(json_obj, db_options, base_path=BASE_PATH): LOG.info( "Importing %d documents with job ID: %s", len(docs), job_id ) - - try: - save(database, docs) - finally: - database.connection.disconnect() + save(database, docs) else: LOG.info("No jobs to save") |