ssh-ldap: add a new way to configure ssh/ldap for systems

This is a deviation from sssd that gives us a really fast way to manage
LDAP groups/users. It uses the nss-updatedb program to pull down *all*
user and group information from LDAP (takes a couple of seconds). This
information is stored in the NSS "db" format, that can be configured via
nsswitch.conf. So all LDAP operations except for checking passwords can
be handled completely locally. Password checking (which is needed by sudo)
can be enhanced by using the libpam-ccreds which will cache a user's
password locally.

I also added something like a "tiered hierarchy" concept. Only one
system in the colo actually pulls down LDAP information. It keeps the
resulting DB in a directory exposed by Apache. All the other servers in
the colo simply grab the LDAP DB from this host. This reduces the load
on the LDAP server and it also makes the updates for all the systems in
the colo really quick.

Change-Id: If028d2adc7a88a7d8ae2a0a30c870a0c403883af

15 files changed, 361 insertions, 8 deletions

diff --git a/group_vars/aus-colo b/group_vars/aus-colo
new file mode 100644
index 00000000..cffab7e9
--- /dev/null
+++ b/group_vars/aus-colo
@@ -0,0 +1,17 @@
+login_groups:
+  - aus-colo
+  - users
+
+sudoers:
+  - andy.doan
+  - ben.copeland
+  - david.mandala
+  - luca.sokoll
+  - paul.sokolovsky
+  - philip.colmer
+
+ldap_base: dc=linaro,dc=org
+ldap_uri: "ldaps://login.linaro.org"
+ldap_binddn: cn=ldapbind,dc=linaro,dc=org
+
+ldap_cache_url: http://10.10.0.1/ldap-files.tgz
diff --git a/host_vars/aus-colo.linaro.org b/host_vars/aus-colo.linaro.org
index 3c12656a..9a5dc2d5 100644
--- a/host_vars/aus-colo.linaro.org
+++ b/host_vars/aus-colo.linaro.org
@@ -1,10 +1,9 @@
-sudoers:
- - andy.doan
- - ben.copeland
- - david.mandala
- - luca.sokoll
- - paul.sokolovsky
- - philip.colmer
+ldap_cache_url: ""
+
+login_groups:
+  - aus-colo
+  - aus-colo-users
+  - users
 
 r1_hosts:
  # Infrastructure machines
diff --git a/hosts b/hosts
index 8423bfda..230b99e9 100644
--- a/hosts
+++ b/hosts
@@ -1,7 +1,6 @@
 [aus-colo]
 aus-colo.linaro.org
 r1-x86-1.aus-colo.linaro.org
-r1-x86-2.aus-colo.linaro.org
 r1-maas-server.aus-colo.linaro.org
 r2-x86-1.aus-colo.linaro.org
 r3-x86-1.aus-colo.linaro.org
diff --git a/roles/ssh-ldap/files/linaro_ldap.py b/roles/ssh-ldap/files/linaro_ldap.py
new file mode 100644
index 00000000..4a5be6e7
--- /dev/null
+++ b/roles/ssh-ldap/files/linaro_ldap.py
@@ -0,0 +1,88 @@
+import contextlib
+import os
+import subprocess
+import tempfile
+import ldap
+
+
+# To provide alternative ldap bind credentials, override the LDAP_CONF
+# environment variable when calling your script that makes use of the this
+# library
+LDAP_CONF = os.environ.get('LDAP_CONF', '/etc/ldap.conf')
+
+
+@contextlib.contextmanager
+def ldap_client(config):
+    client = ldap.initialize(config["uri"], trace_level=0)
+    client.set_option(ldap.OPT_REFERRALS, 0)
+    client.simple_bind(config["binddn"], config["bindpw"])
+    try:
+        yield client
+    finally:
+        client.unbind()
+
+
+def build_config():
+    config = {}
+    with open(LDAP_CONF) as f:
+        for line in f:
+            if line.startswith('binddn'):
+                if "binddn" not in config:
+                    config["binddn"] = line.split(' ', 1)[1].strip()
+            elif line.startswith('bindpw'):
+                if "bindpw" not in config:
+                    config["bindpw"] = line.split(' ', 1)[1].strip()
+            elif line.startswith('base'):
+                if "basedn" not in config:
+                    config["basedn"] = line.split(' ', 1)[1].strip()
+            elif line.startswith('uri'):
+                if "uri" not in config:
+                    config["uri"] = line.split(' ', 1)[1].strip()
+    return config
+
+
+def validate_key(pubkey):
+    with tempfile.NamedTemporaryFile(delete=True) as f:
+        f.write(pubkey)
+        f.flush()
+        try:
+            args = ['ssh-keygen', '-l', '-f', f.name]
+            subprocess.check_output(args, stderr=subprocess.PIPE)
+        except:
+            return False
+    return True
+
+
+def do_query(search_attr='uid', search_pat='*', attrlist=[]):
+    config = build_config()
+    with ldap_client(config) as client:
+        result = client.search_s(
+            config["basedn"],
+            ldap.SCOPE_SUBTREE,
+            '(%s=%s)' % (search_attr, search_pat),
+            attrlist)
+    return result
+
+
+def get_users_and_keys(only_validated=False):
+    """Gets all the users and their associated SSH key.
+    :return A list of tuples (uid, ssh_key), only if the user has an SSH
+    key.
+    """
+    result = do_query(attrlist=['uid', 'sshPublicKey'])
+    all_users = {}
+    for row in result:
+        try:
+            # Just pick the first UID, it does not really matter how the
+            # user is called, it will access the git repository via the
+            # 'git' user.
+            uid = row[1]['uid'][0]
+            ssh_keys = row[1]['sshPublicKey']
+            for index, ssh_key in enumerate(ssh_keys):
+                if not only_validated or validate_key(ssh_key):
+                    key_name = '{0}@key_{1}.pub'.format(uid, index)
+                    all_users.setdefault(uid, []).append((key_name, ssh_key))
+        except KeyError:
+            # If there are no SSH keys, skip this user.
+            pass
+    return all_users
diff --git a/roles/ssh-ldap/files/mkhomedir b/roles/ssh-ldap/files/mkhomedir
new file mode 100644
index 00000000..fcd8d786
--- /dev/null
+++ b/roles/ssh-ldap/files/mkhomedir
@@ -0,0 +1,6 @@
+Name: Create home directory during login (MANAGED BY ANSIBLE)
+Default: yes
+Priority: 900
+Session-Type: Additional
+Session:
+        required        pam_mkhomedir.so umask=0022 skel=/etc/skel
diff --git a/roles/ssh-ldap/files/nsswitch.conf b/roles/ssh-ldap/files/nsswitch.conf
new file mode 100644
index 00000000..3538ca8f
--- /dev/null
+++ b/roles/ssh-ldap/files/nsswitch.conf
@@ -0,0 +1,14 @@
+# !!MANAGED BY ANSIBLE!!
+passwd:         files db
+group:          files db
+shadow:         files db
+
+hosts:          files dns
+networks:       files
+
+protocols:      db files
+services:       db files
+ethers:         db files
+rpc:            db files
+
+netgroup:       nis
diff --git a/roles/ssh-ldap/files/ssh_keys.py b/roles/ssh-ldap/files/ssh_keys.py
new file mode 100644
index 00000000..2cb479cc
--- /dev/null
+++ b/roles/ssh-ldap/files/ssh_keys.py
@@ -0,0 +1,56 @@
+#!/usr/bin/python2
+import json
+import os
+import subprocess
+import sys
+import tarfile
+import urllib2
+
+import linaro_ldap
+
+
+def web_sync(url):
+    if not os.path.exists('./tmp'):
+        os.mkdir('./tmp')
+    tf = urllib2.urlopen(url)
+    with tarfile.open(fileobj=tf, mode="r|gz") as tf:
+        tf.extractall(path='./tmp')
+
+    for p in os.listdir('./tmp'):
+        os.rename('./tmp/' + p, p)
+
+
+def ldap_sync():
+    fname = 'ssh_keys.json'
+    with open(fname + '.tmp', 'w') as f:
+        json.dump(linaro_ldap.get_users_and_keys(), f)
+    os.rename(f.name, fname)
+    subprocess.check_call(['/usr/sbin/nss_updatedb', 'ldap'])
+    with tarfile.open('ldap-files.tgz.tmp', 'w:gz') as tf:
+        tf.add('group.db')
+        tf.add('passwd.db')
+        tf.add('ssh_keys.json')
+    os.rename('ldap-files.tgz.tmp', 'ldap-files.tgz')
+
+
+def keys(user):
+    with open('ssh_keys.json') as f:
+        data = json.load(f)
+        keys = data.get(user)
+        if keys:
+            for key in keys:
+                print(key[1])
+
+
+if __name__ == '__main__':
+    if len(sys.argv) not in (2, 3):
+        sys.exit('Usage: %s --sync|<user>' % sys.argv[0])
+
+    os.chdir('/var/lib/misc')
+    if sys.argv[1] == '--sync':
+        if len(sys.argv) == 3:
+            web_sync(sys.argv[2])
+        else:
+            ldap_sync()
+    else:
+        keys(sys.argv[1])
diff --git a/roles/ssh-ldap/handlers/main.yml b/roles/ssh-ldap/handlers/main.yml
new file mode 100644
index 00000000..99a93e12
--- /dev/null
+++ b/roles/ssh-ldap/handlers/main.yml
@@ -0,0 +1,2 @@
+- name: restart-sshd
+  service: name=ssh state=restarted
diff --git a/roles/ssh-ldap/tasks/main.yml b/roles/ssh-ldap/tasks/main.yml
new file mode 100644
index 00000000..cc1ba17f
--- /dev/null
+++ b/roles/ssh-ldap/tasks/main.yml
@@ -0,0 +1,49 @@
+- name: Install packages
+  apt: pkg={{item}} state=installed
+  with_items:
+    - libnss-db
+    - libnss-ldap
+    - libpam-ccreds
+    - nss-updatedb
+    - python-ldap
+
+- name: Copy linaro_ldap script
+  copy: src=linaro_ldap.py dest=/usr/lib/python2.7/dist-packages mode=555 owner=root
+
+- name: Copy ssh_keys.py script
+  copy: src=ssh_keys.py dest=/etc/ssh/ssh_keys.py mode=555 owner=root
+  register: ssh_keys
+
+- name: Configure ldap.conf
+  template: src=ldap.conf dest=/etc/
+
+- name: See if offline LDAP cache exists
+  stat: path=/var/lib/misc/group.db
+  register: cache
+
+- name: Generate offline LDAP cache
+  when: cache.stat.exists == False or ssh_keys.changed
+  command: /etc/ssh/ssh_keys.py --sync {{ldap_cache_url}}
+
+- name: Configure nsswitch.conf
+  copy: src=nsswitch.conf dest=/etc/
+
+- name: Configure sudoers
+  template: src=ansible_sudoers dest=/etc/sudoers.d/
+            mode=0400 owner=root
+
+- name: Enable home directory creation
+  copy: src=mkhomedir dest=/usr/share/pam-configs/
+  register: mkhomedir
+
+- name: Update pam-auth-update
+  when: mkhomedir is defined and mkhomedir.changed
+  command: pam-auth-update --force --package
+
+- name: Configure sshd (authorized keys and allowgroups)
+  template: src=sshd_config dest=/etc/ssh/
+  notify:
+    - restart-sshd
+
+- name: Add cron job for syncing with LDAP
+  template: src=cron.d dest=/etc/cron.d/ldap-sync
diff --git a/roles/ssh-ldap/templates/ansible_sudoers b/roles/ssh-ldap/templates/ansible_sudoers
new file mode 100644
index 00000000..0f063f8b
--- /dev/null
+++ b/roles/ssh-ldap/templates/ansible_sudoers
@@ -0,0 +1,4 @@
+# !!MANAGED BY ANSIBLE!!
+{% for user in sudoers %}
+{{user}}	ALL=(ALL:ALL) ALL
+{% endfor %}
diff --git a/roles/ssh-ldap/templates/cron.d b/roles/ssh-ldap/templates/cron.d
new file mode 100644
index 00000000..bf73380d
--- /dev/null
+++ b/roles/ssh-ldap/templates/cron.d
@@ -0,0 +1,2 @@
+# !!MANAGED BY ANSIBLE!!
+0,30 * * * * root /etc/ssh/ssh_keys.py --sync {{ldap_cache_url}}
diff --git a/roles/ssh-ldap/templates/ldap-websync.sh b/roles/ssh-ldap/templates/ldap-websync.sh
new file mode 100644
index 00000000..ad367734
--- /dev/null
+++ b/roles/ssh-ldap/templates/ldap-websync.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/sh -e
+
+URL="{{ldap_cache_url}}"
+
+cd /var/lib/misc
+wget -O tmp.tgz $URL
+tar -xz tmp.tgz
+rm tmp.tgz
diff --git a/roles/ssh-ldap/templates/ldap.conf b/roles/ssh-ldap/templates/ldap.conf
new file mode 100644
index 00000000..2014b54c
--- /dev/null
+++ b/roles/ssh-ldap/templates/ldap.conf
@@ -0,0 +1,9 @@
+## !!MANAGED BY ANSIBLE!!
+
+base {{ldap_base}}
+uri {{ldap_uri}}
+binddn {{ldap_binddn}}
+bindpw {{ldap_bindpw}}
+ldap_version 3
+pam_password md5
+rootbinddn
diff --git a/roles/ssh-ldap/templates/sshd_config b/roles/ssh-ldap/templates/sshd_config
new file mode 100644
index 00000000..bc7e11ee
--- /dev/null
+++ b/roles/ssh-ldap/templates/sshd_config
@@ -0,0 +1,93 @@
+# !!MANGED BY ANSIBLE!!
+# Package generated configuration file
+# See the sshd_config(5) manpage for details
+
+# What ports, IPs and protocols we listen for
+Port 22
+# Use these options to restrict which interfaces/protocols sshd will bind to
+#ListenAddress ::
+#ListenAddress 0.0.0.0
+Protocol 2
+# HostKeys for protocol version 2
+HostKey /etc/ssh/ssh_host_rsa_key
+HostKey /etc/ssh/ssh_host_dsa_key
+HostKey /etc/ssh/ssh_host_ecdsa_key
+HostKey /etc/ssh/ssh_host_ed25519_key
+#Privilege Separation is turned on for security
+UsePrivilegeSeparation yes
+
+# Lifetime and size of ephemeral version 1 server key
+KeyRegenerationInterval 3600
+ServerKeyBits 1024
+
+# Logging
+SyslogFacility AUTH
+LogLevel INFO
+
+# Authentication:
+LoginGraceTime 120
+PermitRootLogin without-password
+StrictModes yes
+
+RSAAuthentication yes
+PubkeyAuthentication yes
+#AuthorizedKeysFile	%h/.ssh/authorized_keys
+
+# Don't read the user's ~/.rhosts and ~/.shosts files
+IgnoreRhosts yes
+# For this to work you will also need host keys in /etc/ssh_known_hosts
+RhostsRSAAuthentication no
+# similar for protocol version 2
+HostbasedAuthentication no
+# Uncomment if you don't trust ~/.ssh/known_hosts for RhostsRSAAuthentication
+#IgnoreUserKnownHosts yes
+
+# To enable empty passwords, change to yes (NOT RECOMMENDED)
+PermitEmptyPasswords no
+
+# Change to yes to enable challenge-response passwords (beware issues with
+# some PAM modules and threads)
+ChallengeResponseAuthentication no
+
+# Change to no to disable tunnelled clear text passwords
+PasswordAuthentication no
+
+# Kerberos options
+#KerberosAuthentication no
+#KerberosGetAFSToken no
+#KerberosOrLocalPasswd yes
+#KerberosTicketCleanup yes
+
+# GSSAPI options
+#GSSAPIAuthentication no
+#GSSAPICleanupCredentials yes
+
+X11Forwarding yes
+X11DisplayOffset 10
+PrintMotd no
+PrintLastLog yes
+TCPKeepAlive yes
+#UseLogin no
+
+#MaxStartups 10:30:60
+#Banner /etc/issue.net
+
+# Allow client to pass locale environment variables
+AcceptEnv LANG LC_*
+
+Subsystem sftp /usr/lib/openssh/sftp-server
+
+# Set this to 'yes' to enable PAM authentication, account processing,
+# and session processing. If this is enabled, PAM authentication will
+# be allowed through the ChallengeResponseAuthentication and
+# PasswordAuthentication.  Depending on your PAM configuration,
+# PAM authentication via ChallengeResponseAuthentication may bypass
+# the setting of "PermitRootLogin without-password".
+# If you just want the PAM account and session checks to run without
+# PAM authentication, then enable this but set PasswordAuthentication
+# and ChallengeResponseAuthentication to 'no'.
+UsePAM yes
+
+AuthorizedKeysCommandUser root
+AuthorizedKeysCommand /etc/ssh/ssh_keys.py
+AllowGroups {%for group in login_groups%}{{group}} {%endfor%}
diff --git a/ssh-ldap.yml b/ssh-ldap.yml
new file mode 100644
index 00000000..f6c72a81
--- /dev/null
+++ b/ssh-ldap.yml
@@ -0,0 +1,7 @@
+- name: Configure user access via ssh/ldap
+  hosts: aus-colo
+  become: yes
+  vars_files:
+    - ["{{secrets_dir}}/group_vars/aus-colo"]
+  roles:
+  - role: ssh-ldap