Add defconfig_full field.

  * Introduce new defconfig_full field for boot and defconfig
    model in order to store the full name of the defconfig.
    This is necessary since builds can still use the same
    defconfig name, but add config fragments. This field will
    store the defconfig name plus all the config fragments used.

Change-Id: Ie9a592c6b97a4eb4861806cdd3597f2be9ab1334

13 files changed, 132 insertions, 34 deletions

diff --git a/app/handlers/common.py b/app/handlers/common.py
index 9047093..109251e 100644
--- a/app/handlers/common.py
+++ b/app/handlers/common.py
@@ -65,6 +65,7 @@ BOOT_VALID_KEYS = {
             models.BOOT_RETRIES_KEY,
             models.BOOT_TIME_KEY,
             models.BOOT_WARNINGS_KEY,
+            models.DEFCONFIG_FULL_KEY,
             models.DEFCONFIG_KEY,
             models.DTB_ADDR_KEY,
             models.DTB_APPEND_KEY,
@@ -92,6 +93,7 @@ BOOT_VALID_KEYS = {
         models.ARCHITECTURE_KEY,
         models.BOARD_KEY,
         models.CREATED_KEY,
+        models.DEFCONFIG_FULL_KEY,
         models.DEFCONFIG_ID_KEY,
         models.DEFCONFIG_KEY,
         models.ENDIANNESS_KEY,
@@ -108,6 +110,7 @@ BOOT_VALID_KEYS = {
     ],
     'DELETE': [
         models.BOARD_KEY,
+        models.DEFCONFIG_FULL_KEY,
         models.DEFCONFIG_ID_KEY,
         models.DEFCONFIG_KEY,
         models.ID_KEY,
@@ -123,6 +126,7 @@ COUNT_VALID_KEYS = {
         models.ARCHITECTURE_KEY,
         models.BOARD_KEY,
         models.CREATED_KEY,
+        models.DEFCONFIG_FULL_KEY,
         models.DEFCONFIG_ID_KEY,
         models.DEFCONFIG_KEY,
         models.ERRORS_KEY,
@@ -152,6 +156,7 @@ DEFCONFIG_VALID_KEYS = {
         models.ARCHITECTURE_KEY,
         models.BUILD_LOG_KEY,
         models.CREATED_KEY,
+        models.DEFCONFIG_FULL_KEY,
         models.DEFCONFIG_KEY,
         models.DIRNAME_KEY,
         models.ERRORS_KEY,
diff --git a/app/models/__init__.py b/app/models/__init__.py
index 2bf4bde..1c1ee16 100644
--- a/app/models/__init__.py
+++ b/app/models/__init__.py
@@ -43,6 +43,7 @@ COUNT_KEY = "count"
 CREATED_KEY = 'created_on'
 CROSS_COMPILE_KEY = 'cross_compile'
 DATE_RANGE_KEY = 'date_range'
+DEFCONFIG_FULL_KEY = 'defconfig_full'
 DEFCONFIG_ID_KEY = 'defconfig_id'
 DEFCONFIG_KEY = 'defconfig'
 DIRNAME_KEY = 'dirname'
diff --git a/app/models/boot.py b/app/models/boot.py
index 60eeb24..73ad1a5 100644
--- a/app/models/boot.py
+++ b/app/models/boot.py
@@ -27,7 +27,9 @@ class BootDocument(modb.BaseDocument):
     Each document is a single booted board.
     """
 
-    def __init__(self, board, job, kernel, defconfig, lab_name):
+    def __init__(
+            self, board, job, kernel, defconfig, lab_name,
+            defconfig_full=None):
         """A new BootDocument.
 
         :param board: The name of the board.
@@ -44,9 +46,9 @@ class BootDocument(modb.BaseDocument):
         self._board = board
         self._name = models.BOOT_DOCUMENT_NAME % {
             models.BOARD_KEY: board,
+            models.DEFCONFIG_KEY: defconfig_full or defconfig,
             models.JOB_KEY: job,
             models.KERNEL_KEY: kernel,
-            models.DEFCONFIG_KEY: defconfig,
         }
         self._lab_name = lab_name
         self._job = job
@@ -56,6 +58,7 @@ class BootDocument(modb.BaseDocument):
 
         self._kernel = kernel
         self._defconfig = defconfig
+        self._defconfig_full = defconfig_full
 
         self._metadata = {}
         self._job_id = None
@@ -436,6 +439,19 @@ class BootDocument(modb.BaseDocument):
         """Set the board architecture."""
         self._arch = value
 
+    @property
+    def defconfig_full(self):
+        """The full defconfig name.
+
+        This parameter contains also the config fragments information.
+        """
+        return self._defconfig_full
+
+    @defconfig_full.setter
+    def defconfig_full(self, value):
+        """Set the full defconfig name."""
+        self._defconfig_full = value
+
     def to_dict(self):
         boot_dict = {
             models.ARCHITECTURE_KEY: self.arch,
@@ -444,6 +460,7 @@ class BootDocument(modb.BaseDocument):
             models.BOOT_LOG_KEY: self.boot_log,
             models.BOOT_RESULT_DESC_KEY: self.boot_result_description,
             models.CREATED_KEY: self.created_on,
+            models.DEFCONFIG_FULL_KEY: self.defconfig_full,
             models.DEFCONFIG_ID_KEY: self.defconfig_id,
             models.DEFCONFIG_KEY: self.defconfig,
             models.DTB_ADDR_KEY: self.dtb_addr,
diff --git a/app/models/defconfig.py b/app/models/defconfig.py
index 04047d1..38573ea 100644
--- a/app/models/defconfig.py
+++ b/app/models/defconfig.py
@@ -27,12 +27,12 @@ class DefconfigDocument(modb.BaseDocument):
 
     """This class represents a defconfig folder as seen on the file system."""
 
-    def __init__(self, job, kernel, defconfig):
+    def __init__(self, job, kernel, defconfig, defconfig_full=None):
 
         doc_name = {
             models.JOB_KEY: job,
             models.KERNEL_KEY: kernel,
-            models.DEFCONFIG_KEY: defconfig
+            models.DEFCONFIG_KEY: defconfig_full or defconfig
         }
 
         self._name = models.DEFCONFIG_DOCUMENT_NAME % doc_name
@@ -42,6 +42,7 @@ class DefconfigDocument(modb.BaseDocument):
         self._job = job
         self._kernel = kernel
         self._defconfig = defconfig
+        self._defconfig_full = defconfig_full
         self._job_id = None
         self._dirname = None
         self._status = None
@@ -376,6 +377,19 @@ class DefconfigDocument(modb.BaseDocument):
         """Set the config fragment used."""
         self._kconfig_fragments = value
 
+    @property
+    def defconfig_full(self):
+        """The full defconfig name.
+
+        This parameter contains also the config fragments information.
+        """
+        return self._defconfig_full
+
+    @defconfig_full.setter
+    def defconfig_full(self, value):
+        """Set the full defconfig name."""
+        self._defconfig_full = value
+
     def to_dict(self):
         defconf_dict = {
             models.ARCHITECTURE_KEY: self.arch,
@@ -383,6 +397,7 @@ class DefconfigDocument(modb.BaseDocument):
             models.BUILD_PLATFORM_KEY: self.build_platform,
             models.BUILD_TIME_KEY: self.build_time,
             models.CREATED_KEY: self.created_on,
+            models.DEFCONFIG_FULL_KEY: self.defconfig_full,
             models.DEFCONFIG_KEY: self.defconfig,
             models.DIRNAME_KEY: self.dirname,
             models.DTB_DIR_KEY: self.dtb_dir,
diff --git a/app/models/tests/test_boot_model.py b/app/models/tests/test_boot_model.py
index a50790e..3492c93 100644
--- a/app/models/tests/test_boot_model.py
+++ b/app/models/tests/test_boot_model.py
@@ -46,6 +46,7 @@ class TestBootModel(unittest.TestCase):
         boot_doc.git_url = "git-url"
         boot_doc.arch = "arm"
         boot_doc.fastboot_cmd = "fastboot"
+        boot_doc.defconfig_full = "defconfig"
 
         expected = {
             '_id': 'id',
@@ -80,7 +81,8 @@ class TestBootModel(unittest.TestCase):
             "git_describe": "git-describe",
             "git_url": "git-url",
             "arch": "arm",
-            "fastboot_cmd": "fastboot"
+            "fastboot_cmd": "fastboot",
+            "defconfig_full": 'defconfig'
         }
 
         self.assertDictEqual(expected, boot_doc.to_dict())
diff --git a/app/models/tests/test_defconfig_model.py b/app/models/tests/test_defconfig_model.py
index 9d7a6d7..d808424 100644
--- a/app/models/tests/test_defconfig_model.py
+++ b/app/models/tests/test_defconfig_model.py
@@ -29,8 +29,9 @@ class TestDefconfModel(unittest.TestCase):
         self.assertEqual(defconfig_doc.collection, 'defconfig')
 
     def test_defconfig_document_to_dict(self):
+        self.maxDiff = None
         defconf_doc = moddf.DefconfigDocument(
-            'job', 'kernel', 'defconfig'
+            'job', 'kernel', 'defconfig', 'defconfig_full'
         )
         defconf_doc.id = "defconfig_id"
         defconf_doc.job_id = "job_id"
@@ -59,7 +60,7 @@ class TestDefconfModel(unittest.TestCase):
         defconf_doc.kconfig_fragments = "config-frag"
 
         expected = {
-            "name": "job-kernel-defconfig",
+            "name": "job-kernel-defconfig_full",
             "_id": "defconfig_id",
             "job": "job",
             "kernel": "kernel",
@@ -88,7 +89,8 @@ class TestDefconfModel(unittest.TestCase):
             "modules": "modules-file",
             "modules_dir": "modules-dir",
             "build_log": "build.log",
-            "kconfig_fragments": "config-frag"
+            "kconfig_fragments": "config-frag",
+            "defconfig_full": "defconfig_full"
         }
 
         self.assertDictEqual(expected, defconf_doc.to_dict())
diff --git a/app/utils/bootimport.py b/app/utils/bootimport.py
index b2d4342..32c6f9a 100644
--- a/app/utils/bootimport.py
+++ b/app/utils/bootimport.py
@@ -172,9 +172,11 @@ def _parse_boot_from_json(boot_json, database):
         job = json_pop_f(models.JOB_KEY)
         kernel = json_pop_f(models.KERNEL_KEY)
         defconfig = json_pop_f(models.DEFCONFIG_KEY)
+        defconfig_full = json_pop_f(models.DEFCONFIG_FULL_KEY, defconfig)
         lab_name = json_pop_f(models.LAB_NAME_KEY)
 
-        boot_doc = modbt.BootDocument(board, job, kernel, defconfig, lab_name)
+        boot_doc = modbt.BootDocument(
+            board, job, kernel, defconfig, lab_name, defconfig_full)
         boot_doc.created_on = datetime.datetime.now(tz=bson.tz_util.utc)
         _update_boot_doc_from_json(boot_doc, boot_json, json_pop_f)
         _update_boot_doc_ids(boot_doc, database)
diff --git a/app/utils/docimport.py b/app/utils/docimport.py
index e0dff63..44d9b5a 100644
--- a/app/utils/docimport.py
+++ b/app/utils/docimport.py
@@ -205,11 +205,12 @@ def _parse_build_data(data_file, job, kernel):
 
         try:
             defconfig = data_pop(models.DEFCONFIG_KEY)
+            defconfig_full = data_pop(models.DEFCONFIG_FULL_KEY, defconfig)
             job = data_pop(models.JOB_KEY, None) or job
             kernel = data_pop(models.KERNEL_KEY, None) or kernel
 
             defconfig_doc = mdefconfig.DefconfigDocument(
-                job, kernel, defconfig
+                job, kernel, defconfig, defconfig_full
             )
 
             defconfig_doc.created_on = datetime.datetime.fromtimestamp(
diff --git a/app/utils/scripts/convert-models.py b/app/utils/scripts/convert-models.py
index fad8b0e..69822de 100644
--- a/app/utils/scripts/convert-models.py
+++ b/app/utils/scripts/convert-models.py
@@ -103,12 +103,18 @@ def convert_defconfig_collection(db, limit=0):
             doc_count += 1
             utils.LOG.info("Processing document #%s", doc_count)
 
+            metadata = doc_get("metadata", {})
+            meta_get = metadata.get
+            meta_pop = metadata.pop
+
+            arch = None
+            defconfig_full = None
+            kconfig_fragments = None
+
             job = doc_get("job")
             kernel = doc_get("kernel")
             defconfig = doc_get("defconfig")
 
-            arch = None
-
             if defconfig.startswith("arm-"):
                 defconfig = defconfig.replace("arm-", "", 1)
                 arch = "arm"
@@ -119,13 +125,29 @@ def convert_defconfig_collection(db, limit=0):
                 defconfig = defconfig.replace("x86-", "", 1)
                 arch = "x86"
 
-            def_doc = mdefconfig.DefconfigDocument(job, kernel, defconfig)
+            if meta_get("kconfig_fragments", None):
+                kconfig_fragments = meta_pop("kconfig_fragments")
+                fragment = \
+                    kconfig_fragments.replace(
+                        "frag-", "").replace(".config", "")
+                if fragment not in defconfig:
+                    defconfig_full = "+".join([defconfig, fragment])
+
+            if not defconfig_full:
+                defconfig_full = defconfig
+
+            def_doc = mdefconfig.DefconfigDocument(
+                job, kernel, defconfig, defconfig_full)
 
             def_doc.version = "1.0"
+            def_doc.kconfig_fragments = kconfig_fragments
+            def_doc.defconfig_full = defconfig_full
             def_doc.status = doc_get("status", models.UNKNOWN_STATUS)
+
             if not NEW_JOB_IDS.get(job + "-" + kernel, None):
-                utils.LOG.info("No job ID for %s-%s", job, kernel)
+                utils.LOG.error("No job ID for '%s-%s'", job, kernel)
             def_doc.job_id = NEW_JOB_IDS.get(job + "-" + kernel, None)
+
             def_doc.dirname = doc_get("dirname", None)
             def_doc.arch = doc_get("arch", None)
             def_doc.created_on = doc_get("created_on")
@@ -144,13 +166,8 @@ def convert_defconfig_collection(db, limit=0):
             def_doc.modules_dir = doc_get("modules_dir", None)
             def_doc.modules = doc_get("modules", None)
             def_doc.build_log = doc_get("build_log", None)
-            def_doc.kconfig_fragments = doc_get("kconfig_fragments", None)
 
-            metadata = doc_get("metadata", None)
             if metadata:
-                meta_pop = metadata.pop
-                meta_get = metadata.get
-
                 if (str(def_doc.errors) != str(meta_get("build_errors")) and
                         meta_get("build_errors") is not None):
                     def_doc.errors = int(meta_pop("build_errors", 0))
@@ -217,8 +234,8 @@ def convert_defconfig_collection(db, limit=0):
 
             ret_val, doc_id = utils.db.save(db, def_doc, manipulate=True)
             if ret_val == 201:
-                NEW_DEFCONFIG_IDS[job + "-" + kernel + "-" + defconfig] = \
-                    doc_id
+                NEW_DEFCONFIG_IDS[job + "-" + kernel + "-" + defconfig_full] = \
+                    (doc_id, defconfig, defconfig_full)
             else:
                 utils.LOG.error(
                     "Error saving new defconfig document for %s",
@@ -266,24 +283,37 @@ def convert_boot_collection(db, lab_name, limit=0):
                 defconfig = defconfig.replace("x86-", "", 1)
                 arch = "x86"
 
+            job_id = NEW_JOB_IDS.get(job + "-" + kernel, None)
+            if not job_id:
+                utils.LOG.error("No job ID found for %s-%s", job, kernel)
+
+            defconfig_id, build_defconfig, defconfig_full = \
+                NEW_DEFCONFIG_IDS.get(
+                    job + "-" + kernel + "-" + defconfig,
+                    [None, None, None]
+                )
+
+            if build_defconfig:
+                defconfig = build_defconfig
+            if not defconfig_full:
+                defconfig_full = defconfig
+
+            if not defconfig_id:
+                utils.LOG.error(
+                    "No defconfig ID found for %s-%s-%s (%s)",
+                    job, kernel, defconfig, defconfig_full
+                )
+
             boot_doc = mboot.BootDocument(
-                board, job, kernel, defconfig, lab_name)
+                board, job, kernel, defconfig, lab_name, defconfig_full
+            )
+
+            boot_doc.job_id = job_id
+            boot_doc.defconfig_id = defconfig_id
 
             boot_doc.arch = arch
             boot_doc.version = "1.0"
 
-            boot_doc.job_id = NEW_JOB_IDS.get(job + "-" + kernel, None)
-            if not boot_doc.job_id:
-                utils.LOG.warn("No job ID found for %s-%s", job, kernel)
-
-            boot_doc.defconfig_id = NEW_DEFCONFIG_IDS.get(
-                job + "-" + kernel + "-" + defconfig, None
-            )
-            if not boot_doc.job_id:
-                utils.LOG.warn(
-                    "No defconfig ID found for %s-%s-%s",
-                    job, kernel, defconfig
-                )
             boot_doc.created_on = doc_get("created_on", None)
             boot_doc.tine = doc_get("time", 0)
             boot_doc.warnings = doc_get("warnings", 0)
diff --git a/doc/collection-boot.rst b/doc/collection-boot.rst
index 3243f47..2ee0473 100644
--- a/doc/collection-boot.rst
+++ b/doc/collection-boot.rst
@@ -35,6 +35,7 @@ GET
  :query string job_id: The ID of a job.
  :query string kernel: The name of a kernel.
  :query string defconfig: The name of a defconfig.
+ :query string defconfig_full: The full name of a defconfig (with config fragments).
  :query string defconfig_id: The ID of a defconfig.
  :query string endian: The endianness of the board.
  :query string board: The name of a board.
@@ -196,6 +197,7 @@ DELETE
  :query string kernel: The name of a kernel.
  :query string defconfig_id: The ID of a defconfig.
  :query string defconfig: The name of a defconfig.
+ :query string defconfig_full: The full name of a defconfig (with config fragments).
  :query string board: The name of a board.
  :query string name: The name of a boot report.
 
diff --git a/doc/collection-defconfig.rst b/doc/collection-defconfig.rst
index 737ec51..6cf2d66 100644
--- a/doc/collection-defconfig.rst
+++ b/doc/collection-defconfig.rst
@@ -33,6 +33,7 @@ GET
  :query string job: The name of a job.
  :query string job_id: The ID of a job.
  :query string kernel: The name of a kernel.
+ :query string defconfig_full: The full name of a defconfig (with config fragments).
  :query string defconfig: The name of a defconfig.
  :query string name: The name of the defconfig report.
  :query string arch: The architecture on which the defconfig has been built.
diff --git a/doc/schema-boot.rst b/doc/schema-boot.rst
index 6530567..6654579 100644
--- a/doc/schema-boot.rst
+++ b/doc/schema-boot.rst
@@ -66,6 +66,11 @@ necessary to use its ``_id`` value.
                 "type": "string",
                 "description": "The name of the defconfig as reported by the CI loop"
             },
+            "defconfig_full": {
+                "type": "string",
+                "description": "The full name of the defconfig, can contain also config fragments information",
+                "default": "The defconfig value"
+            },
             "defconfig_id": {
                 "type": "object",
                 "description": "The ID of the associated build report",
@@ -208,6 +213,11 @@ have when sent to the server.
                 "type": "string",
                 "description": "The name of the defconfig as reported by the CI loop"
             },
+            "defconfig_full": {
+                "type": "string",
+                "description": "The full name of the defconfig with config fragments information",
+                "default": "The defconfig value"
+            },
             "board": {
                 "type": "string",
                 "description": "The name of the board: it must be a valid and recognized name"
diff --git a/doc/schema-defconfig.rst b/doc/schema-defconfig.rst
index 78acf35..b90f164 100644
--- a/doc/schema-defconfig.rst
+++ b/doc/schema-defconfig.rst
@@ -67,6 +67,11 @@ The following schema covers the data that is available with a GET request.
                 "type": "string",
                 "description": "The name of the defconfig as reported by the CI loop"
             },
+            "defconfig_full": {
+                "type": "string",
+                "description": "The full name of the defconfig, can contain also config fragments information",
+                "default": "The defconfig value"
+            },
             "dirname": {
                 "type": "string",
                 "description": "The name of the directory of the defconfig built; it can be different from the actual defconfig name"
@@ -192,6 +197,11 @@ server.
                 "type": "string",
                 "description": "The name of the defconfig as reported by the CI loop"
             },
+            "defconfig_full": {
+                "type": "string",
+                "description": "The full name of the defconfig with config fragments information",
+                "default": "The defconfig value"
+            },
             "build_result": {
                 "type": "string",
                 "description": "The status of the defconfig",