basic grokmirror setup

This gets grokmirror stuff in place. The slave can be fully deployed
from scratch. The master requires 2 steps. The first is deploying this
the 2nd is then updating the gitolite-admin repo to have the POST_GIT
hook we've defined in this playbook

Change-Id: I0cd27eabb706f70c425648f7ecc90797dd7bbd31

16 files changed, 254 insertions, 13 deletions

diff --git a/per-service/git-servers/files/git.linaro.org.conf b/per-service/git-servers/files/git.linaro.org.conf
index 85d28031..399d1f72 100644
--- a/per-service/git-servers/files/git.linaro.org.conf
+++ b/per-service/git-servers/files/git.linaro.org.conf
@@ -7,6 +7,7 @@ SuexecUserGroup {{ git_user }} {{ git_user }}
 
 <VirtualHost *:80>
     ServerName {{ git_host }}
+    ServerAlias {{ inventory_hostname }}
     ServerAdmin webmaster@linaro.org
 
     CustomLog ${APACHE_LOG_DIR}/{{ git_host }}-access.log combined
diff --git a/per-service/git-servers/git-main.yml b/per-service/git-servers/git-main.yml
index 6914e276..8ec9e7d5 100644
--- a/per-service/git-servers/git-main.yml
+++ b/per-service/git-servers/git-main.yml
@@ -18,11 +18,12 @@
     - apache-conf
     - apache-website
     - {role: apache-site, src: "{{git_host}}.conf", config: "{{git_host}}", tags: [apache, website-conf] }
-    - url-rewrite
-    - cronjobs
+    - {role: url-rewrite, when: hosttype != 'git-slave'}
+    - {role: cronjobs, when: hosttype != 'git-slave'}
     - gitweb
     - upstart-git-daemon
-    - mirroring
+    - {role: mirroring, when: hosttype != 'git-slave'}
+    - {role: grokmirror, when: grokmirror_support}
 
 - hosts:
     - review-public
diff --git a/per-service/git-servers/group_vars/all b/per-service/git-servers/group_vars/all
index 161654d0..20ec3f05 100644
--- a/per-service/git-servers/group_vars/all
+++ b/per-service/git-servers/group_vars/all
@@ -7,6 +7,8 @@ mailto_address: linaro-infrastructure-errors@lists.linaro.org
 # If the server is public or private
 server_access: public
 
+grokmirror_support: false
+
 gerrit_managed: no
 gerrit_download_link: https://gerrit-releases.storage.googleapis.com/gerrit-2.8.6.1.war
 gerrit_root: /srv/gerrit
diff --git a/per-service/git-servers/roles/apache-conf/tasks/main.yml b/per-service/git-servers/roles/apache-conf/tasks/main.yml
index 767c35ae..bcaf0d4c 100644
--- a/per-service/git-servers/roles/apache-conf/tasks/main.yml
+++ b/per-service/git-servers/roles/apache-conf/tasks/main.yml
@@ -21,7 +21,7 @@
   with_items:
     - suexec
   notify: restart-apache
-  when: hosttype == "git-main"
+  when: hosttype in ["git-main", "git-slave"]
   tags:
     - install
     - apache-conf
diff --git a/per-service/git-servers/roles/apache-website/tasks/apache-website-git.yml b/per-service/git-servers/roles/apache-website/tasks/apache-website-git.yml
index fc061d7e..0abe66bd 100644
--- a/per-service/git-servers/roles/apache-website/tasks/apache-website-git.yml
+++ b/per-service/git-servers/roles/apache-website/tasks/apache-website-git.yml
@@ -7,7 +7,7 @@
         owner={{ git_user }}
         mode=0755
         path={{ apache_root }}/{{ git_host }}
-  when: hosttype == "git-main" or hosttype == "git-review"
+  when: hosttype in ["git-main", "git-review", "git-slave"]
   tags:
     - install
 
@@ -17,7 +17,7 @@
             owner={{ git_user }}
             group={{ git_user }}
             mode=0550
-  when: hosttype == "git-main" and server_access == "public"
+  when: hosttype in ["git-main", "git-slave"] and server_access == "public"
   tags:
     - install
 
@@ -27,6 +27,6 @@
             owner={{ git_user }}
             group={{ git_user }}
             mode=0550
-  when: hosttype == "git-main"
+  when: hosttype in ["git-main", "git-slave"]
   tags:
     - install
diff --git a/per-service/git-servers/roles/gitolite/tasks/gitolite-rc.yml b/per-service/git-servers/roles/gitolite/tasks/gitolite-rc.yml
index 45866001..d157533a 100644
--- a/per-service/git-servers/roles/gitolite/tasks/gitolite-rc.yml
+++ b/per-service/git-servers/roles/gitolite/tasks/gitolite-rc.yml
@@ -1,5 +1,4 @@
 # Manage gitolite.rc file.
-
 - name: Set the gitolite.rc file
   template: src=gitolite.rc
             dest=/home/{{ git_user }}/.gitolite.rc
diff --git a/per-service/git-servers/roles/gitolite/templates/gitolite.rc b/per-service/git-servers/roles/gitolite/templates/gitolite.rc
index 035006c0..9b053d71 100644
--- a/per-service/git-servers/roles/gitolite/templates/gitolite.rc
+++ b/per-service/git-servers/roles/gitolite/templates/gitolite.rc
@@ -169,6 +169,12 @@
         'post-compile/ssh-authkeys',
     ],
 
+{% if grokmirror_support %}
+    POST_GIT => [
+        'post-update.grokmirror',
+    ],
+{% endif %}
+
     # Custom script that is called to get groups authorization
     # based on the user that is performing the operation.
     GROUPLIST_PGM => '/home/{{ git_user }}/gitolite-tools/gitolite-groups',
diff --git a/per-service/git-servers/roles/gitweb/tasks/main.yml b/per-service/git-servers/roles/gitweb/tasks/main.yml
index 4be85589..ac616acf 100644
--- a/per-service/git-servers/roles/gitweb/tasks/main.yml
+++ b/per-service/git-servers/roles/gitweb/tasks/main.yml
@@ -15,7 +15,7 @@
     - gitweb-conf
 
 - name: Install gitweb Linaro theme
-  when: hosttype == "git-main"
+  when: hosttype in ["git-main", "git-slave"]
   git:  repo=http://git.linaro.org/git-ro/infrastructure/gitweb-linaro-theme.git
         dest={{ tools_checkout_dir }}/gitweb-linaro-theme
   tags:
@@ -23,7 +23,7 @@
     - gitweb
 
 - name: Locally copy gitweb Linaro theme
-  when: hosttype == "git-main"
+  when: hosttype in ["git-main", "git-slave"]
   shell:  cp *.* {{ apache_root }}/{{ git_host }}
           chdir={{ tools_checkout_dir }}/gitweb-linaro-theme
   tags:
@@ -31,7 +31,7 @@
     - gitweb
 
 - name: Fix gitweb Linaro theme permission
-  when: hosttype == "git-main"
+  when: hosttype in ["git-main", "git-slave"]
   shell:  chown {{ git_user }}:{{ git_user }} *
           chdir={{ apache_root }}/{{ git_host }}
   tags:
diff --git a/per-service/git-servers/roles/grokmirror/files/grok-git-daemon-export b/per-service/git-servers/roles/grokmirror/files/grok-git-daemon-export
new file mode 100644
index 00000000..9461fe51
--- /dev/null
+++ b/per-service/git-servers/roles/grokmirror/files/grok-git-daemon-export
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+cd $1
+touch git-daemon-export-ok
diff --git a/per-service/git-servers/roles/grokmirror/tasks/main.yml b/per-service/git-servers/roles/grokmirror/tasks/main.yml
new file mode 100644
index 00000000..0fe60da0
--- /dev/null
+++ b/per-service/git-servers/roles/grokmirror/tasks/main.yml
@@ -0,0 +1,31 @@
+- name: Install dependencies for grokmirror
+  apt: name={{item}}
+  with_items:
+    - python-setuptools
+    - python-anyjson
+    - python-git
+  tags:
+    - grokmirror
+
+- name: Clone grokmirror repository
+  git:  repo=https://github.com/mricon/grokmirror.git
+        dest=/srv/grokmirror
+        update=yes
+        accept_hostkey=yes
+  tags:
+    - grokmirror
+
+- name: Install grokmirror
+  command: python ./setup.py install
+           chdir=/srv/grokmirror
+           creates=/usr/local/bin/grok-manifest
+  tags:
+    - grokmirror
+
+- name: Setup grokmirror cron job
+  template: src=grokmirror.cron.d dest=/etc/cron.d/grokmirror
+            owner=root group=root mode=0644
+  tags:
+    - grokmirror
+
+- {include: slave.yml, when: hosttype == 'git-slave'}
diff --git a/per-service/git-servers/roles/grokmirror/tasks/slave.yml b/per-service/git-servers/roles/grokmirror/tasks/slave.yml
new file mode 100644
index 00000000..b340b25f
--- /dev/null
+++ b/per-service/git-servers/roles/grokmirror/tasks/slave.yml
@@ -0,0 +1,24 @@
+- name: Install grokmirror slave configuration
+  template: src=grokmirror-repos.conf dest=/etc/
+            owner=root group=root mode=0644
+  tags:
+    - grokmirror
+
+- name: Install grokmirror slave configuration fsck
+  template: src=grokmirror-fsck.conf dest=/etc/
+            owner=root group=root mode=0644
+  tags:
+    - grokmirror
+
+- name: Install grokmirror slave script
+  copy: src=grok-git-daemon-export dest=/usr/local/bin/
+        owner=root group=root mode=0755
+  tags:
+    - grokmirror
+
+- name: Setup repository directoy
+  file: path={{repo_root}} state=directory
+        owner={{git_user}} group={{git_user}} mode=0775
+  tags:
+    - grokmirror
+
diff --git a/per-service/git-servers/roles/grokmirror/templates/grokmirror-fsck.conf b/per-service/git-servers/roles/grokmirror/templates/grokmirror-fsck.conf
new file mode 100644
index 00000000..851e78f9
--- /dev/null
+++ b/per-service/git-servers/roles/grokmirror/templates/grokmirror-fsck.conf
@@ -0,0 +1,60 @@
+# You can have multiple sections, just name them appropriately
+[master-repo]
+# Where is the manifest containing the list of repositories?
+manifest = {{repo_root}}/manifest.js.gz
+#
+# Where are the repositories kept?
+#toplevel = /var/lib/git/mirror
+toplevel = {{repo_root}}/
+#
+# Where do we put the logs?
+#log = /var/log/mirror/kernelorg-fsck.log
+log = /tmp/grokmirror-fsck.log
+#
+# Log level can be "info" or "debug"
+#loglevel = info
+loglevel = info
+#
+# Make sure there is only one instance of grok-fsck running by
+# trying to exclusive-lock this file before we do anything.
+lock = /tmp/grokmirror-fsck.lock
+#
+# Where to keep the status file
+#statusfile = /var/lib/mirror/kernelorg-fsck.js
+statusfile = {{repo_root}}/grokmirror-fsck.js
+#
+# How often should we check each repository, in days.
+# Any newly added repository will have the first check within a random
+# period of 0 and $frequency, and then every $frequency after that,
+# to assure that not all repositories are checked on the same day.
+# Don't set to less than 7 unless you only mirror a few repositories
+# (or really like to thrash your disks).
+#frequency = 30
+frequency = 30
+#
+# Not all repositories take a long time to check. This option lets you
+# allocate some time after each run to check small repositories. Grok-fsck
+# will then sort all repos not already checked during the usual run by their
+# elapsed time, and run an fsck check on each of them until the allotted time
+# for quick checks is elapsed.
+# Note that if you only have a few repos, this pretty much obsoletes
+# "frequency" if it takes less than this much time to check them all.
+#quick_checks_max_min = 5
+quick_checks_max_min = 5
+#
+# Some errors are relatively benign and can be safely ignored. Add matching
+# substrings to this field to ignore them.
+ignore_errors = dangling commit
+                dangling blob
+                notice: HEAD points to an unborn branch
+                notice: No default references
+                contains zero-padded file modes
+#
+# Enable this option to repack the repository before calling git-fsck.
+# This will allow you to save some space if you have shared repositories.
+# To check if you have shared repositories, look at your manifest.js.gz to
+# see if any repository definition has a "reference" key.
+#repack = yes
+#
+# Default repack flags are -A -d -l -q, but you can specify your own here
+#repack_flags = -A -d -l -q
diff --git a/per-service/git-servers/roles/grokmirror/templates/grokmirror-repos.conf b/per-service/git-servers/roles/grokmirror/templates/grokmirror-repos.conf
new file mode 100644
index 00000000..be87e172
--- /dev/null
+++ b/per-service/git-servers/roles/grokmirror/templates/grokmirror-repos.conf
@@ -0,0 +1,106 @@
+# You can pull from multiple grok mirrors, just create
+# a separate section for each mirror. The name can be anything.
+[master-repo]
+# The host part of the mirror you're pulling from.
+#site = git://git.kernel.org
+site = http://{{git_master}}
+#
+# Where the grok manifest is published. The following protocols
+# are supported at this time:
+# http:// or https:// using If-Modified-Since http header
+# file:// (when manifest file is on NFS, for example)
+#manifest = http://git.kernel.org/manifest.js.gz
+manifest = http://{{git_master}}/manifest.js.gz
+#
+# Where are we going to put the mirror on our disk?
+#toplevel = /var/lib/git/mirror
+toplevel = {{repo_root}}/
+#
+# Where do we store our own manifest? Usually in the toplevel.
+#mymanifest = /var/lib/git/mirror/manifest.js.gz
+mymanifest = {{repo_root}}/manifest.js.gz
+#
+# Write out projects.list that can be used by gitweb or cgit.
+# Leave blank if you don't want a projects.list.
+#projectslist = /var/lib/git/mirror/projects.list
+projectslist = /home/{{git_user}}/projects.list
+#
+# When generating projects.list, start at this subpath instead
+# of at the toplevel. Useful when mirroring kernel or when generating
+# multiple gitweb/cgit configurations for the same tree.
+#projectslist_trimtop = /pub/scm/
+#projectslist_trimtop = /pub/scm/
+#
+# The default behaviour is to use cross-repository references (if present)
+# to set git alternates between projects. This helps both significantly
+# reduce the bandwidth during cloning and dramatically reduce the mirror size
+# on disk. If for some reason you always want to create non-shared repositories,
+# set ignore_repo_references to "yes"
+# NOTE: this will NOT remove alternates for repositories already mirrored. You
+# will need to run "git repack -a" and then manually remove the
+# objects/info/alternates file for each repository.
+#ignore_repo_references = no
+#
+# When generating projects.list, also create entries for symlinks.
+# Otherwise we assume they are just legacy and keep them out of
+# web interfaces.
+#projectslist_symlinks = yes
+projectslist_symlinks = no
+#
+# A simple hook to execute whenever a repository is modified.
+# It passes the full path to the git repository modified as the only
+# argument.
+#post_update_hook = /usr/local/bin/make-git-fairies-appear
+post_update_hook = /usr/local/bin/grok-git-daemon-export
+#
+# This prevents catastrophic mirror purges when our upstream master gives us
+# a manifest that is dramatically smaller than ours. The default is to refuse
+# the purge if the remote manifest has over 10% fewer repositories than what we
+# have, or in other words, if we have 100 repos and the remote manifest has
+# shrunk to 95 repos or less, we refuse to purge, suspecting that something has
+# gone wrong. You can set purgeprotect to a higher percentage, or override
+# it entirely with --force-purge commandline flag.
+#purgeprotect = 5
+purgeprotect = 5
+#
+# If owner is not specified in the manifest, who should be listed
+# as the default owner in tools like gitweb or cgit?
+#default_owner = Grokmirror User
+default_owner = Grokmirror User
+#
+# Where do we put the logs?
+#log = /var/log/mirror/kernelorg.log
+log = /tmp/grokmirror.log
+#
+# Log level can be "info" or "debug"
+#loglevel = info
+loglevel = info
+#
+# To speed up updates, grok-pull will use multiple threads. Please be
+# considerate to the mirror you're pulling from and don't set this very
+# high. You may also run into per-ip multiple session limits, so leave this
+# number at a nice low setting.
+#pull_threads = 5
+pull_threads = 5
+#
+# Use shell-globbing to list the repositories you would like to mirror.
+# If you want to mirror everything, just say "*". Separate multiple entries
+# with newline plus tab. Examples:
+#
+# mirror everything:
+#include = *
+#
+# mirror just the main kernel sources:
+#include = /pub/scm/linux/kernel/git/torvalds/linux.git
+#          /pub/scm/linux/kernel/git/stable/linux-stable.git
+#          /pub/scm/linux/kernel/git/next/linux-next.git
+#
+# mirror just git:
+#include = /pub/scm/git/*
+include = *
+#
+# This is processed after the include. If you want to exclude some specific
+# entries from an all-inclusive globbing above. E.g., to exclude all linux-2.4
+# git sources:
+#exclude = */linux-2.4*
+exclude =
diff --git a/per-service/git-servers/roles/grokmirror/templates/grokmirror.cron.d b/per-service/git-servers/roles/grokmirror/templates/grokmirror.cron.d
new file mode 100644
index 00000000..185a998b
--- /dev/null
+++ b/per-service/git-servers/roles/grokmirror/templates/grokmirror.cron.d
@@ -0,0 +1,7 @@
+{% if hosttype == "git-slave" %}
+* * * * *	git /usr/local/bin/grok-pull -p -c /etc/grokmirror-repos.conf
+00 02 * * *	git /usr/local/bin/grok-fsck -c /etc/grokmirror-fsck.conf
+{% endif %}
+{% if hosttype == "git-main" %}
+* * * * *	git /usr/local/bin/grok-manifest -m {{apache_root}}/{{git_host}}/manifest.js.gz -t {{repo_root}}/ -p
+{% endif %}
diff --git a/per-service/git-servers/roles/install-deps/tasks/main.yml b/per-service/git-servers/roles/install-deps/tasks/main.yml
index f9f20bc0..5a685cc5 100644
--- a/per-service/git-servers/roles/install-deps/tasks/main.yml
+++ b/per-service/git-servers/roles/install-deps/tasks/main.yml
@@ -27,7 +27,7 @@
   apt: name={{ item }}
   with_items:
     - apache2-suexec-custom
-  when: hosttype == "git-main"
+  when: hosttype in ["git-main", "git-slave"]
   tags:
     - deps
     - git
diff --git a/per-service/git-servers/roles/users/tasks/main.yml b/per-service/git-servers/roles/users/tasks/main.yml
index 3f062130..2d392418 100644
--- a/per-service/git-servers/roles/users/tasks/main.yml
+++ b/per-service/git-servers/roles/users/tasks/main.yml
@@ -1,7 +1,7 @@
 ---
 - name: Make sure the host git-user is available
   user: name={{ git_user }}
-  when: hosttype == "git-main"
+  when: hosttype in ["git-main", "git-slave"]
   tags:
     - deps
     - git