Merge branch 'master' into mainline-bisect

10 files changed, 125 insertions, 18 deletions

diff --git a/ansible/roles/init-conf/tasks/main.yml b/ansible/roles/init-conf/tasks/main.yml
index 1997f57..4908ff1 100644
--- a/ansible/roles/init-conf/tasks/main.yml
+++ b/ansible/roles/init-conf/tasks/main.yml
@@ -1,5 +1,13 @@
 ---
 
+- name: Make sure celery run directory exists
+  file: path=/var/run/celery
+        state=directory
+        owner={{ app_user }}
+        group={{ app_user }}
+  tags:
+    - init
+
 - name: Install Upstart conf
   template: src=kernel-ci-backend.conf
             dest=/etc/init/{{ init_service }}.conf
@@ -54,6 +62,28 @@
     - init
     - systemd
 
+- name: Install celery beat upstart configuration
+  template: src=kernelci-celery-beat.conf
+            dest=/etc/init/kernelci-celery-beat.conf
+            owner=root
+            group=root
+            mode=0644
+  when: ansible_lsb.id == "Ubuntu"
+  tags:
+    - init
+    - upstart
+
+- name: Install celery beat systemd service
+  template: src=kernelci-celery-beat.service
+            dest=/etc/systemd/system/kernelci-celery-beat.service
+            owner=root
+            group=root
+            mode=0644
+  when: ansible_lsb.id == "Debian"
+  tags:
+    - init
+    - systemd
+
 - name: Enable celery systemd service
   command:  /bin/systemctl enable kernelci-celery.service
             creates=/etc/systemd/system/multi-user.target.wants/kernelci-celery.service
@@ -64,3 +94,13 @@
     - init
     - systemd
 
+- name: Enable celery beat systemd service
+  command:  /bin/systemctl enable kernelci-celery-beat.service
+            creates=/etc/systemd/system/multi-user.target.wants/kernelci-celery-beat.service
+  when: ansible_lsb.id == "Debian"
+  notify:
+    - reload-systemd
+  tags:
+    - init
+    - systemd
+
diff --git a/ansible/roles/init-conf/templates/celery.conf b/ansible/roles/init-conf/templates/celery.conf
index 688b239..a7c64a2 100644
--- a/ansible/roles/init-conf/templates/celery.conf
+++ b/ansible/roles/init-conf/templates/celery.conf
@@ -2,8 +2,8 @@
 author "Milo Casagrande <milo.casagrande@linaro.org>"
 description "Configuration to run celery at startup."
 
-start on started redis-server
-stop on stopping redis-server
+start on started kernelci-backend
+stop on stopping kernelci-backend
 
 respawn
 respawn limit 10 5
@@ -19,9 +19,8 @@ chdir {{ install_base }}/{{ hostname }}/app
 
 script
     if [ -d {{ install_base }}/.venv/{{ hostname }} ]; then
-        . {{ install_base }}/.venv/{{ hostname }}/bin/activate
-        {{ install_base }}/.venv/{{ hostname }}/bin/celery worker --autoscale=4,1 --logfile=/var/log/celery/%h%I.log --loglevel=INFO --pidfile=/tmp/celery.pid --app=taskqueue
+        {{ install_base }}/.venv/{{ hostname }}/bin/python -OO -R {{ install_base }}/.venv/{{ hostname }}/bin/celery worker -Ofair --without-gossip --autoscale=4,1 --logfile=/var/log/celery/%h%I.log --loglevel=INFO --app=taskqueue
     else
-        exec celery worker --autoscale=4,1 --logfile=/var/log/celery/%h%I.log --loglevel=INFO --pidfile=/tmp/celery.pid --app=taskqueue
+        exec celery worker -Ofair --without-gossip --autoscale=4,1 --logfile=/var/log/celery/%h%I.log --loglevel=INFO --app=taskqueue
     fi
 end script
diff --git a/ansible/roles/init-conf/templates/kernelci-celery-beat.conf b/ansible/roles/init-conf/templates/kernelci-celery-beat.conf
new file mode 100644
index 0000000..b35e9ff
--- /dev/null
+++ b/ansible/roles/init-conf/templates/kernelci-celery-beat.conf
@@ -0,0 +1,26 @@
+# Managed by ansible, do not edit.
+author "Milo Casagrande <milo.casagrande@linaro.org>"
+description "Configuration to run celery at startup."
+
+start on started kernelci-celery
+stop on stopping kernelci-celery
+
+respawn
+respawn limit 10 5
+
+kill timeout 6
+
+limit nofile 4096 65536
+
+setuid {{ app_user }}
+setgid {{ app_user }}
+
+chdir {{ install_base }}/{{ hostname }}/app
+
+script
+    if [ -d {{ install_base }}/.venv/{{ hostname }} ]; then
+        {{ install_base }}/.venv/{{ hostname }}/bin/python -OO -R {{ install_base }}/.venv/{{ hostname }}/bin/celery beat --loglevel=INFO --schedule /var/run/celery/kernelci-beat.db --app=taskqueue
+    else
+        exec celery beat --loglevel=INFO --schedule /var/run/celery/kernelci-beat.db --app=taskqueue
+    fi
+end script
diff --git a/ansible/roles/init-conf/templates/kernelci-celery-beat.service b/ansible/roles/init-conf/templates/kernelci-celery-beat.service
new file mode 100644
index 0000000..a0d5add
--- /dev/null
+++ b/ansible/roles/init-conf/templates/kernelci-celery-beat.service
@@ -0,0 +1,23 @@
+[Unit]
+Description=Kernel CI Backend Celery Beat
+
+[Service]
+Type=simple
+User={{ app_user }}
+Group={{ app_user }}
+ProtectHome=true
+NoNewPrivileges=true
+PrivateTmp=true
+SyslogIdentifier=kernelci-celery-beat
+LimitNOFILE=65536
+RestartSec=5
+Restart=always
+PIDFile=/tmp/kernelci-celery-beat.pid
+WorkingDirectory={{ install_base }}/{{ hostname }}/app
+ExecStart={{ install_base }}/.venv/{{ hostname }}/bin/python -OO -R \
+    {{ install_base }}/.venv/{{ hostname }}/bin/celery beat \
+    --loglevel=INFO --schedule=/var/run/celery/kernelci-beat.db
+    --app=taskqueue
+
+[Install]
+WantedBy=multi-user.target
diff --git a/ansible/roles/init-conf/templates/kernelci-celery.service b/ansible/roles/init-conf/templates/kernelci-celery.service
index c3a969b..5462ec4 100644
--- a/ansible/roles/init-conf/templates/kernelci-celery.service
+++ b/ansible/roles/init-conf/templates/kernelci-celery.service
@@ -17,13 +17,13 @@ WorkingDirectory={{ install_base }}/{{ hostname }}/app
 {% if role == "production" %}
 ExecStart={{ install_base }}/.venv/{{ hostname }}/bin/python -OO -R \
     {{ install_base }}/.venv/{{ hostname }}/bin/celery worker \
-    --autoscale=16,4 --loglevel=INFO \
-    --pidfile=/tmp/kernelci-celery.pid --app=taskqueue
+    -Ofair --without-gossip --autoscale=16,4 --loglevel=INFO \
+    --app=taskqueue
 {% else %}
 ExecStart={{ install_base }}/.venv/{{ hostname }}/bin/python -OO -R \
     {{ install_base }}/.venv/{{ hostname }}/bin/celery worker \
-    --autoscale=5,1 --loglevel=INFO \
-    --pidfile=/tmp/kernelci-celery.pid --app=taskqueue
+    -Ofair --without-gossip --autoscale=5,1 --loglevel=INFO \
+    --app=taskqueue
 {% endif %}
 
 [Install]
diff --git a/app/handlers/__init__.py b/app/handlers/__init__.py
index a8e7208..d4302cd 100644
--- a/app/handlers/__init__.py
+++ b/app/handlers/__init__.py
@@ -1,2 +1,2 @@
-__version__ = "2015.1.2"
+__version__ = "2015.1.3"
 __versionfull__ = __version__
diff --git a/app/handlers/send.py b/app/handlers/send.py
index c6f9bf5..d7cf4b2 100644
--- a/app/handlers/send.py
+++ b/app/handlers/send.py
@@ -41,6 +41,14 @@ class SendHandler(hbase.BaseHandler):
         db_options = kwargs["db_options"]
         mail_options = self.settings["mailoptions"]
 
+        self.log.info(
+            "Email trigger received from IP '%s' for '%s-%s' at %s",
+            self.request.remote_ip,
+            json_obj.get("job", None),
+            json_obj.get("kernel", None),
+            datetime.datetime.utcnow()
+        )
+
         countdown = json_obj.get(models.DELAY_KEY, self.settings["senddelay"])
         if countdown is None:
             countdown = self.settings["senddelay"]
diff --git a/app/taskqueue/celeryconfig.py b/app/taskqueue/celeryconfig.py
index 9624262..0e2fd0e 100644
--- a/app/taskqueue/celeryconfig.py
+++ b/app/taskqueue/celeryconfig.py
@@ -17,10 +17,17 @@
 
 BROKER_URL = "redis://localhost"
 BROKER_POOL_LIMIT = 20
+BROKER_TRANSPORT_OPTIONS = {
+    "visibility_timeout": 10800,
+    "fanout_prefix": True,
+    "fanout_patterns": True
+}
 CELERY_ACCEPT_CONTENT = ["json"]
 CELERY_RESULT_SERIALIZER = "json"
 CELERY_TASK_SERIALIZER = "json"
+CELERY_TIMEZONE = "UTC"
 CELERY_ENABLE_UTC = True
+CELERY_IGNORE_RESULT = True
 CELERY_DISABLE_RATE_LIMITS = True
 CELERY_RESULT_BACKEND = "mongodb://localhost"
 CELERY_MONGODB_BACKEND_SETTINGS = {
diff --git a/app/taskqueue/tasks.py b/app/taskqueue/tasks.py
index d597ffa..130513e 100644
--- a/app/taskqueue/tasks.py
+++ b/app/taskqueue/tasks.py
@@ -44,7 +44,7 @@ def send_emails(job_id):
     pass
 
 
-@taskc.app.task(name='import-job', ignore_result=True)
+@taskc.app.task(name="import-job")
 def import_job(json_obj, db_options):
     """Just a wrapper around the real import function.
 
@@ -59,7 +59,7 @@ def import_job(json_obj, db_options):
     utils.docimport.import_and_save_job(json_obj, db_options)
 
 
-@taskc.app.task(name='import-boot', ignore_result=True)
+@taskc.app.task(name="import-boot")
 def import_boot(json_obj, db_options):
     """Just a wrapper around the real boot import function.
 
@@ -74,7 +74,7 @@ def import_boot(json_obj, db_options):
     utils.bootimport.import_and_save_boot(json_obj, db_options)
 
 
-@taskc.app.task(name='batch-executor')
+@taskc.app.task(name="batch-executor")
 def execute_batch(json_obj, db_options):
     """Run batch operations based on the passed JSON object.
 
@@ -138,7 +138,7 @@ def defconfig_bisect(doc_id, db_options, fields=None):
     return defconfigb.execute_defconfig_bisection(doc_id, db_options, fields)
 
 
-@taskc.app.task(name="schedule-boot-report")
+@taskc.app.task(name="schedule-boot-report", acks_late=True, track_started=True)
 def schedule_boot_report(json_obj, db_options, mail_options, countdown):
     """Schedule a second task to send the boot report.
 
@@ -184,7 +184,7 @@ def schedule_boot_report(json_obj, db_options, mail_options, countdown):
                 "cannot be sent", job, kernel)
 
 
-@taskc.app.task(name="send-boot-report")
+@taskc.app.task(name="send-boot-report", acks_late=True, track_started=True)
 def send_boot_report(job, kernel, lab_name, to_addrs, db_options, mail_options):
     """Create the boot report email and send it.
 
diff --git a/app/utils/report.py b/app/utils/report.py
index f21e7a9..781336f 100644
--- a/app/utils/report.py
+++ b/app/utils/report.py
@@ -142,9 +142,12 @@ def create_boot_report(job, kernel, lab_name, db_options):
         fields=BUILD_SEARCH_FIELDS)
 
     git_data = _parse_job_results(git_results)
-    git_commit = git_data[models.GIT_COMMIT_KEY]
-    git_url = git_data[models.GIT_URL_KEY]
-    git_branch = git_data[models.GIT_BRANCH_KEY]
+    if git_data:
+        git_commit = git_data[models.GIT_COMMIT_KEY]
+        git_url = git_data[models.GIT_URL_KEY]
+        git_branch = git_data[models.GIT_BRANCH_KEY]
+    else:
+        git_commit = git_url = git_branch = "Unknown"
 
     spec[models.STATUS_KEY] = models.OFFLINE_STATUS
 
@@ -239,6 +242,7 @@ def create_boot_report(job, kernel, lab_name, db_options):
 
     return email_body, subject
 
+
 def _parse_job_results(results):
     """Parse the job results from the database creating a new data structure.