diff options
| -rw-r--r-- | app/handlers/bisect.py | 32 | ||||
| -rw-r--r-- | app/models/__init__.py | 3 | ||||
| -rw-r--r-- | app/models/bisect.py | 21 | ||||
| -rw-r--r-- | app/models/tests/test_bisect_model.py | 33 | ||||
| -rw-r--r-- | app/taskqueue/tasks.py | 16 | ||||
| -rw-r--r-- | app/utils/bisect/__init__.py | 138 |
6 files changed, 240 insertions, 3 deletions
diff --git a/app/handlers/bisect.py b/app/handlers/bisect.py index 2b4de28..2a242f2 100644 --- a/app/handlers/bisect.py +++ b/app/handlers/bisect.py @@ -115,6 +115,9 @@ class BisectHandler(hbase.BaseHandler): if collection == models.BOOT_COLLECTION: response = self.execute_boot_bisect(doc_id, db_options, fields) + elif collection == models.DEFCONFIG_COLLECTION: + response = self.execute_defconfig_bisect( + doc_id, db_options, fields) else: response = hresponse.HandlerResponse(400) response.reason = ( @@ -148,3 +151,32 @@ class BisectHandler(hbase.BaseHandler): elif response.status_code == 400: response.reason = "Boot report cannot be bisected: is it failed?" return response + + @staticmethod + def execute_defconfig_bisect(doc_id, db_options, fields=None): + """Execute the defconfig bisect operation. + + :param doc_id: The ID of the document to execute the bisect on. + :type doc_id: str + :param db_options: The mongodb database connection parameters. + :type db_options: dict + :param fields: A `fields` data structure with the fields to return or + exclude. Default to None. + :type fields: list or dict + :return A `HandlerResponse` object. + """ + response = hresponse.HandlerResponse() + + result = taskt.defconfig_bisect.apply_async( + [doc_id, db_options, fields] + ) + while not result.ready(): + pass + + response.status_code, response.result = result.get() + if response.status_code == 404: + response.reason = "Defconfig not found" + elif response.status_code == 400: + response.reason = "Defconfig cannot be bisected: is it failed?" + + return response diff --git a/app/models/__init__.py b/app/models/__init__.py index de1c0fc..55b4ff9 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -198,5 +198,6 @@ VALID_JOB_STATUS = [ # The valid collections for the bisect handler. BISECT_VALID_COLLECTIONS = [ - BOOT_COLLECTION + BOOT_COLLECTION, + DEFCONFIG_COLLECTION, ] diff --git a/app/models/bisect.py b/app/models/bisect.py index a05cc43..4d8d8e9 100644 --- a/app/models/bisect.py +++ b/app/models/bisect.py @@ -135,3 +135,24 @@ class BootBisectDocument(BisectDocument): boot_b_dict[models.DEFCONFIG_ID_KEY] = self.defconfig_id boot_b_dict[models.BOOT_ID_KEY] = self.boot_id return boot_b_dict + + +class DefconfigBisectDocument(BisectDocument): + """The bisect document class for defconfig/build bisection.""" + + def __init__(self, name): + super(DefconfigBisectDocument, self).__init__(name) + + self.defconfig = None + self.defconfig_id = None + self.defconfig_full = None + self.arch = None + + def to_dict(self): + def_b_dict = super(DefconfigBisectDocument, self).to_dict() + def_b_dict[models.DEFCONFIG_ID_KEY] = self.defconfig_id + def_b_dict[models.DEFCONFIG_KEY] = self.defconfig + def_b_dict[models.DEFCONFIG_FULL_KEY] = self.defconfig_full + def_b_dict[models.ARCHITECTURE_KEY] = self.arch + + return def_b_dict diff --git a/app/models/tests/test_bisect_model.py b/app/models/tests/test_bisect_model.py index 3cc1b9b..874835b 100644 --- a/app/models/tests/test_bisect_model.py +++ b/app/models/tests/test_bisect_model.py @@ -135,3 +135,36 @@ class TestBisectModel(unittest.TestCase): bisect_doc.board = "bar" self.assertEqual(bisect_doc.board, "bar") + + def test_bisect_defconfig_to_dict(self): + bisect_doc = modbs.DefconfigBisectDocument("foo") + bisect_doc.id = "bar" + bisect_doc.defconfig_id = "defconfig-id" + bisect_doc.defconfig = "defconfig-name" + bisect_doc.version = "1.0" + bisect_doc.job = "job" + bisect_doc.job_id = "job-id" + bisect_doc.defconfig_full = "defconfig-full" + bisect_doc.arch = "arm" + + expected = { + "_id": "bar", + "created_on": None, + "job": "job", + "name": "foo", + "bisect_data": [], + "good_commit": None, + "good_commit_date": None, + "good_commit_url": None, + "bad_commit": None, + "bad_commit_date": None, + "bad_commit_url": None, + "version": "1.0", + "defconfig_id": "defconfig-id", + "defconfig": "defconfig-name", + "job_id": "job-id", + "defconfig_full": "defconfig-full", + "arch": "arm" + } + + self.assertDictEqual(expected, bisect_doc.to_dict()) diff --git a/app/taskqueue/tasks.py b/app/taskqueue/tasks.py index d170085..5d8131f 100644 --- a/app/taskqueue/tasks.py +++ b/app/taskqueue/tasks.py @@ -99,6 +99,22 @@ def boot_bisect(doc_id, db_options, fields=None): return utils.bisect.execute_boot_bisection(doc_id, db_options, fields) +@taskc.app.task(name="defconfig-bisect") +def defconfig_bisect(doc_id, db_options, fields=None): + """Run a defconfig bisect operation on the passed defconfig document id. + + :param doc_id: The boot document ID. + :type doc_id: str + :param db_options: The mongodb database connection parameters. + :type db_options: dict + :param fields: A `fields` data structure with the fields to return or + exclude. Default to None. + :type fields: list or dict + :return The result of the boot bisect operation. + """ + return utils.bisect.execute_defconfig_bisection(doc_id, db_options, fields) + + def run_batch_group(batch_op_list, db_options): """Execute a list of batch operations. diff --git a/app/utils/bisect/__init__.py b/app/utils/bisect/__init__.py index 6e3825f..2709edb 100644 --- a/app/utils/bisect/__init__.py +++ b/app/utils/bisect/__init__.py @@ -54,6 +54,23 @@ BOOT_DEFCONFIG_SEARCH_FIELDS = [ BOOT_SORT = [(models.CREATED_KEY, pymongo.DESCENDING)] +DEFCONFIG_SEARCH_FIELDS = [ + models.ARCHITECTURE_KEY, + models.CREATED_KEY, + models.DEFCONFIG_FULL_KEY, + models.DEFCONFIG_KEY, + models.GIT_COMMIT_KEY, + models.GIT_DESCRIBE_KEY, + models.GIT_URL_KEY, + models.ID_KEY, + models.JOB_ID_KEY, + models.JOB_KEY, + models.KERNEL_KEY, + models.STATUS_KEY, +] + +DEFCONFIG_SORT = [(models.CREATED_KEY, pymongo.DESCENDING)] + def _combine_defconfig_values(boot_doc, db_options): """Combine the boot document values with their own defconfing. @@ -166,7 +183,8 @@ def execute_boot_bisection(doc_id, db_options, fields=None): result = None else: bisect_doc = mbisect.BootBisectDocument(obj_id) - bisect_doc.job = start_doc_get(models.JOB_ID_KEY) + bisect_doc.version = "1.0" + bisect_doc.job = start_doc_get(models.JOB_KEY, None) bisect_doc.job_id = start_doc_get(models.JOB_ID_KEY, None) bisect_doc.defconfig_id = start_doc_get( models.DEFCONFIG_ID_KEY, None) @@ -235,7 +253,123 @@ def execute_boot_bisection(doc_id, db_options, fields=None): if return_code == 201: bisect_doc.id = saved_id else: - utils.LOG.error("Error savind bisect data %s", doc_id) + utils.LOG.error("Error saving bisect data %s", doc_id) + + bisect_doc = _update_doc_fields(bisect_doc, fields) + result = [ + json.loads( + json.dumps( + bisect_doc, + default=bson.json_util.default, + ensure_ascii=False + ) + ) + ] + else: + code = 404 + result = None + + return code, result + + +def execute_defconfig_bisection(doc_id, db_options, fields=None): + """Calculate bisect data for the provided defconfig report. + + It searches all the previous defconfig built starting from the provided one + until it finds one that passed. After that, it combines the value into a + single data structure. + + :param doc_id: The boot document ID. + :type doc_id: str + :param db_options: The mongodb database connection parameters. + :type db_options: dict + :param fields: A `fields` data structure with the fields to return or + exclude. Default to None. + :type fields: list or dict + :return A numeric value for the result status and a list of dictionaries. + """ + database = utils.db.get_db_connection(db_options) + result = [] + code = 200 + + obj_id = bson.objectid.ObjectId(doc_id) + start_doc = utils.db.find_one( + database[models.DEFCONFIG_COLLECTION], + [obj_id], fields=DEFCONFIG_SEARCH_FIELDS + ) + + if all([start_doc, isinstance(start_doc, types.DictionaryType)]): + start_doc_get = start_doc.get + + if start_doc_get(models.STATUS_KEY) == models.PASS_STATUS: + code = 400 + result = None + else: + bisect_doc = mbisect.DefconfigBisectDocument(obj_id) + bisect_doc.version = "1.0" + bisect_doc.arch = start_doc_get(models.ARCHITECTURE_KEY, None) + bisect_doc.job = start_doc_get(models.JOB_KEY, None) + bisect_doc.job_id = start_doc_get(models.JOB_ID_KEY, None) + bisect_doc.defconfig_id = start_doc_get(models.ID_KEY) + bisect_doc.defconfig = start_doc_get(models.DEFCONFIG_KEY, None) + bisect_doc.created_on = datetime.datetime.now(tz=bson.tz_util.utc) + bisect_doc.bad_commit_date = start_doc_get(models.CREATED_KEY) + bisect_doc.bad_commit = start_doc_get(models.GIT_COMMIT_KEY) + bisect_doc.bad_commit_url = start_doc_get(models.GIT_URL_KEY) + + spec = { + models.ARCHITECTURE_KEY: start_doc_get( + models.ARCHITECTURE_KEY), + models.CREATED_KEY: { + "$lt": start_doc_get(models.CREATED_KEY) + }, + models.DEFCONFIG_FULL_KEY: start_doc_get( + models.DEFCONFIG_FULL_KEY), + models.DEFCONFIG_KEY: start_doc_get(models.DEFCONFIG_KEY), + models.JOB_KEY: start_doc_get(models.JOB_KEY), + } + + all_valid_docs = [start_doc] + + # Search through all the previous defconfigs, until one that + # passed is found. + all_prev_docs = utils.db.find( + database[models.BOOT_COLLECTION], + 0, + 0, + spec=spec, + fields=DEFCONFIG_SEARCH_FIELDS, + sort=DEFCONFIG_SORT + ) + + if all_prev_docs: + all_valid_docs.extend( + [ + doc for doc in _get_docs_until_pass(all_prev_docs) + ] + ) + + # The last doc should be the good one, in case it is, add the + # values to the bisect_doc. + good_doc = all_valid_docs[-1] + if (good_doc[models.STATUS_KEY] == models.PASS_STATUS): + good_doc_get = good_doc.get + bisect_doc.good_commit = good_doc_get( + models.GIT_COMMIT_KEY) + bisect_doc.good_commit_url = good_doc_get( + models.GIT_URL_KEY) + bisect_doc.good_commit_date = good_doc_get( + models.CREATED_KEY) + + # Store everything in the bisect data. + bisect_doc.bisect_data = all_valid_docs + + return_code, saved_id = utils.db.save( + database, bisect_doc, manipulate=True) + if return_code == 201: + bisect_doc.id = saved_id + else: + utils.LOG.error("Error saving bisect data %s", doc_id) bisect_doc = _update_doc_fields(bisect_doc, fields) result = [ |