patchwork-tools: upgrades to enable python3

This change includes updates to the codebase in order
to allow it to run in python3.  Most of the changes
relate to the str->utf-8 switch in python3 and the retirement
of urllib2 as urllib.parse and urllib.request absorbed its
functionality present in python2.

In order to address some python3 bugs in Django it is upgraded from 1.11
to 1.11.29 (the last LTS release in the 1.11 series).

The python used by the virtualenv for the unit-test.sh script has been
changed to python3.

Change-Id: I3f8016fb405e4d5f0a8a47729b42fdab772588d2

15 files changed, 113 insertions, 98 deletions

diff --git a/alt_email.py b/alt_email.py
index 682b897..d678133 100755
--- a/alt_email.py
+++ b/alt_email.py
@@ -20,7 +20,6 @@ django.setup()
 
 # hack to make python 2.7 use unicode by default
 # since some of our usernames have non-ascii chars
-reload(sys)
 sys.setdefaultencoding('utf8')
 
 SKIP_PROFILES = [
diff --git a/gen_project_json.py b/gen_project_json.py
index a210985..9f89b9a 100755
--- a/gen_project_json.py
+++ b/gen_project_json.py
@@ -21,7 +21,6 @@ django.setup()
 
 # hack to make python 2.7 use unicode by default
 # since some of our usernames have non-ascii chars
-reload(sys)
 sys.setdefaultencoding('utf8')
 
 OUTFILE = "/tmp/projects.json"
diff --git a/gitrepo.py b/gitrepo.py
index 2340d8b..7fed039 100644
--- a/gitrepo.py
+++ b/gitrepo.py
@@ -9,41 +9,42 @@ import dulwich.repo
 from django.conf import settings
 from patchwork.parser import parse_patch
 
-log = logging.getLogger('gitrepo')
+log = logging.getLogger("gitrepo")
 
 
-def croncmd(args, cwd='./', timeout=None, get_fail_logger=None):
+def croncmd(args, cwd="./", timeout=None, get_fail_logger=None):
     if timeout:
-        args = ['timeout', str(timeout)] + args
+        args = ["timeout", str(timeout)] + args
 
     with tempfile.SpooledTemporaryFile(max_size=4096) as f:
         try:
             subprocess.check_call(args, cwd=cwd, stdout=f, stderr=f)
             if log.level == logging.DEBUG:
-                log.debug('Results of cmd(%s): %r', cwd, args)
+                log.debug("Results of cmd(%s): %r", cwd, args)
                 f.seek()
-                log.debug('COMBINED_OUTPUT\n%s', f.read())
+                log.debug("COMBINED_OUTPUT\n%s", f.read())
         except subprocess.CalledProcessError as e:
             logger = log.error
             if get_fail_logger:
                 logger = get_fail_logger()
-            logger('Unable to run command(%s): %r', cwd, args)
+            logger("Unable to run command(%s): %r", cwd, args)
             if timeout and e.returncode == 124:
-                logger('Command timed out')
+                logger("Command timed out")
             f.seek(0)
             if e.output:
-                logger('STDOUT:\n%s', e.output)
-                logger('STDERR:\n%s', f.read())
+                logger("STDOUT:\n%s", e.output)
+                logger("STDERR:\n%s", f.read())
             else:
-                logger('COMBINED OUTPUT:\n%s', f.read())
+                logger("COMBINED OUTPUT:\n%s", f.read())
             raise
 
 
 class Repo(object):
-    '''Our patchwork deployments try and automatically update patches by
+    """Our patchwork deployments try and automatically update patches by
     looking at the change history on a repository. This class provides a
     simple interface to analyze new commits
-    '''
+    """
+
     def __init__(self, repo_dir, name, scm_url):
         self.path = os.path.join(repo_dir, name)
         self.scm_url = scm_url
@@ -59,24 +60,27 @@ class Repo(object):
         return self.repo[key]
 
     def _clone(self):
-        croncmd(['git', 'clone', '--mirror', self.scm_url, self.path])
+        croncmd(["git", "clone", "--mirror", self.scm_url, self.path])
 
     def _pull(self):
-        fail_file = os.path.join(self.path, 'failures')
+        fail_file = os.path.join(self.path, "failures")
 
         def get_fail_logger():
-            with open(fail_file, 'a+') as f:
-                f.write('failed at: %s\n' % str(datetime.datetime.now()))
+            with open(fail_file, "a+") as f:
+                f.write("failed at: %s\n" % str(datetime.datetime.now()))
                 f.seek(0)
                 for count, line in enumerate(f, 1):
                     if count > 3:
                         return log.error
             return log.info
 
-        timeout = str(getattr(settings, 'REPO_TIMEOUT', 120))
+        timeout = str(getattr(settings, "REPO_TIMEOUT", 120))
         croncmd(
-            ['git', 'remote', '-v', 'update', '--prune'], self.path, timeout,
-            get_fail_logger)
+            ["git", "remote", "-v", "update", "--prune"],
+            self.path,
+            timeout,
+            get_fail_logger,
+        )
         if os.path.exists(fail_file):
             # clean out old failures, now that we've succeeded
             os.unlink(fail_file)
@@ -94,47 +98,49 @@ class Repo(object):
                 pass
 
     def process_unchecked_commits(self, save_state=True):
-        last_commit = os.path.join(self.path, 'patchwork-last-commit')
+        last_commit = os.path.join(self.path, "patchwork-last-commit")
         if os.path.exists(last_commit):
-            with open(last_commit) as f:
+            with open(last_commit, "r") as f:
                 start_at = f.read().strip()
         else:
-            start_at = 'HEAD~100'
+            start_at = "HEAD~100"
 
-        log.debug('looking for commits since: %s', start_at)
-        args = ['git', 'rev-list', '--reverse', start_at + '..HEAD']
-        with open('/dev/null', 'w') as f:
+        log.debug("looking for commits since: %s", start_at)
+        args = ["git", "rev-list", "--reverse", start_at + "..HEAD"]
+        with open("/dev/null", "w") as f:
             rc = subprocess.call(
-                ['git', 'show', start_at], cwd=self.path, stdout=f, stderr=f)
+                ["git", "show", start_at], cwd=self.path, stdout=f, stderr=f
+            )
         if rc != 0:
             # we may have had a branch who's history was re-written
             # just try and get changes for past day
-            args = ['git', 'rev-list', '--reverse',
-                    '--since', '1 day ago', 'HEAD']
+            args = ["git", "rev-list", "--reverse", "--since",
+                    "1 day ago", "HEAD"]
 
         try:
-            for x in subprocess.check_output(args, cwd=self.path).split('\n'):
+            for x in subprocess.check_output(
+                    args, cwd=self.path, text=True).split("\n"):
                 if x:
-                    yield self.repo[x]
+                    yield self.repo[x.encode("utf-8")]
                     start_at = x
         finally:
             if save_state:
-                with open(last_commit, 'w') as f:
+                with open(last_commit, "w") as f:
                     f.write(start_at)
 
     def get_patch(self, commit):
-        args = ['git', 'show', '--format=format:%e', '-M', str(commit.id)]
+        args = ["git", "show", "--format=format:%e", "-M", commit.id.decode()]
         patch = subprocess.check_output(args, cwd=self.path)
         # the patchwork parser code operates character by character so we must
         # convert to unicode so it can be handled properly
-        patch = patch.decode('utf-8', errors='replace')
+        patch = patch.decode("utf-8", errors="replace")
         # Don't try and process >5Mb patches, they flood the server
         if len(patch) > 5000000:
-            raise MemoryError('patch too large to process: %d' % len(patch))
+            raise MemoryError("patch too large to process: %d" % len(patch))
         patch = parse_patch(patch)[0]
         if patch is None:
             # happens for binary only patches like:
             # https://git.linaro.org/uefi/OpenPlatformPkg.git/commit/ \
             # ?id=7ab4bb34b2464a2491868264bdf2931f2acd6452
-            patch = ''
+            patch = ""
         return patch
diff --git a/linaro_metrics/parsemail.py b/linaro_metrics/parsemail.py
index d5817b6..18c9065 100644
--- a/linaro_metrics/parsemail.py
+++ b/linaro_metrics/parsemail.py
@@ -9,6 +9,9 @@ from django.conf import settings
 from patchwork.models import Patch, Person, Project
 from patchwork.parser import parse_patch, subject_check
 
+from builtins import str
+unicode = str
+
 log = logging.getLogger('import_emails')
 
 
diff --git a/linaro_metrics/sync_gerrit_changes.py b/linaro_metrics/sync_gerrit_changes.py
index ca47b66..04a9df3 100755
--- a/linaro_metrics/sync_gerrit_changes.py
+++ b/linaro_metrics/sync_gerrit_changes.py
@@ -10,7 +10,8 @@ import json
 import logging
 import textwrap
 import urllib
-import urllib2
+import urllib.request
+import urllib.parse
 
 from datetime import datetime
 
@@ -38,10 +39,10 @@ def get_user_changes(email, url_base, offset=0):
         'q': 'owner:' + email,
         'start': offset,
     }
-    url = url_base + '/changes/?' + urllib.urlencode(params)
+    url = url_base + '/changes/?' + urllib.parse.urlencode(params)
     log.debug('doing http get on: %s', url)
     try:
-        resp = urllib2.urlopen(url)
+        resp = urllib.request.urlopen(url)
         resp = resp.read()
         assert resp.startswith(")]}'")
         entries = json.loads(resp[4:])
@@ -51,7 +52,7 @@ def get_user_changes(email, url_base, offset=0):
         if e and e.get('_more_changes'):
             for e in get_user_changes(email, url_base, offset + len(entries)):
                 yield e
-    except urllib2.HTTPError as e:
+    except urllib.request.HTTPError as e:
         if e.code != 400:
             log.exception('Unable to GET: %s', url)
             sys.exit(1)
@@ -89,7 +90,7 @@ def create_or_update(url_base, project, email, change):
             log.warn('non-linaro user should be removed: %s', email)
             return changed
         if updated > tcs[0].last_state_change:
-            for k, v in fields.iteritems():
+            for k, v in fields.items():
                 setattr(p, k, v)
             changed = True
             p.save()
diff --git a/linaro_metrics/sync_github_changes.py b/linaro_metrics/sync_github_changes.py
index 5967f07..149650f 100755
--- a/linaro_metrics/sync_github_changes.py
+++ b/linaro_metrics/sync_github_changes.py
@@ -8,7 +8,7 @@ import re
 import sys
 import textwrap
 import time
-import urllib2
+from urllib.request import urlopen, HTTPError, Request
 
 from datetime import datetime
 
@@ -109,10 +109,10 @@ class Commit(object):
 
 def _get(url):
     headers = {'Authorization': 'token %s' % settings.GITHUB_OAUTH_TOKEN}
-    request = urllib2.Request(url, headers=headers)
+    request = Request(url, headers=headers)
     try:
-        return urllib2.urlopen(request)
-    except urllib2.HTTPError as e:
+        return urlopen(request)
+    except HTTPError as e:
         error_code = e.getcode()
         log.error('HTTP_%d while GETing %s:\n %s',
                   error_code, url, e.readlines())
@@ -190,7 +190,7 @@ def create_or_update(proj, owner, repo, author, pr):
         p = Patch.objects.get(msgid=msgid)
         tcs = TeamCredit.objects.filter(patch=p)
         if updated > tcs[0].last_state_change:
-            for k, v in fields.iteritems():
+            for k, v in fields.items():
                 setattr(p, k, v)
             p.save()
             TeamCredit.objects.filter(patch=p).update(
diff --git a/linaro_metrics/sync_users.py b/linaro_metrics/sync_users.py
index 6a5b0ee..dde4603 100755
--- a/linaro_metrics/sync_users.py
+++ b/linaro_metrics/sync_users.py
@@ -118,11 +118,11 @@ if __name__ == "__main__":
         try:
             user = User.objects.get(username=email)
 
-        except Exception as e:
+        except Exception:
             print("User does not exist %s... checking email" % email)
             try:
                 user = User.objects.get(email=email)
-            except Exception as e:
+            except Exception:
                 print("Email does not exist %s... Creating" % email)
 
                 user = User.objects.create(
diff --git a/linaro_metrics/tests/test_sync_gerrit_changes.py b/linaro_metrics/tests/test_sync_gerrit_changes.py
index 075227e..dcd93a5 100644
--- a/linaro_metrics/tests/test_sync_gerrit_changes.py
+++ b/linaro_metrics/tests/test_sync_gerrit_changes.py
@@ -15,7 +15,7 @@ from linaro_metrics.models import Team, TeamCredit, TeamMembership
 class TestSyncGerritChanges(TestCase):
     fixtures = ['default_states']
 
-    @mock.patch('urllib2.urlopen')
+    @mock.patch('urllib.request.urlopen')
     def test_get_user_changes_simple(self, urlopen):
         resp = mock.Mock()
         items = [
@@ -26,7 +26,7 @@ class TestSyncGerritChanges(TestCase):
         changes = list(sync_gerrit_changes.get_user_changes('foo@bar.com', ''))
         self.assertEqual(items, changes)
 
-    @mock.patch('urllib2.urlopen')
+    @mock.patch('urllib.request.urlopen')
     def test_get_user_changes_continue(self, urlopen):
         resp = mock.Mock()
         responses = [
diff --git a/linaro_metrics/views.py b/linaro_metrics/views.py
index 2b98b79..9fd8f3e 100644
--- a/linaro_metrics/views.py
+++ b/linaro_metrics/views.py
@@ -234,7 +234,7 @@ def report_project_activity(request):
         if x['month'] >= last_12:
             e['last_12'] += x['patch__pk__count']
     items = [(x, y['last_12'], y['last_6'], y['last_3'], y['last_1'])
-             for x, y in summary.iteritems()]
+             for x, y in summary.items()]
     summary = sorted(items, key=lambda x: x[1], reverse=True)[:20]
 
     qs_a = TeamCredit.patch_count_by_month(
@@ -254,7 +254,7 @@ def report_project_activity(request):
         if x['month'] >= last_12:
             e['last_12'] += x['patch__pk__count']
     items = [(x, y['last_12'], y['last_6'], y['last_3'], y['last_1'])
-             for x, y in accepted.iteritems()]
+             for x, y in accepted.items()]
     accepted = sorted(items, key=lambda x: x[1], reverse=True)[:20]
 
     context = {'summary': summary, 'accepted': accepted}
diff --git a/patch_matcher.py b/patch_matcher.py
index e6dfe34..064b922 100644
--- a/patch_matcher.py
+++ b/patch_matcher.py
@@ -36,8 +36,8 @@ def _patches_similar(name1, diff1, name2, diff2):
 
 
 def _get_patchwork_author_committer(commit):
-    _, auth_email = email.utils.parseaddr(commit.author)
-    _, comm_email = email.utils.parseaddr(commit.committer)
+    _, auth_email = email.utils.parseaddr(commit.author.decode())
+    _, comm_email = email.utils.parseaddr(commit.committer.decode())
     try:
         auth = Person.objects.get(email=auth_email)
     except Person.DoesNotExist:
@@ -74,5 +74,5 @@ def get_patches_matching_commit(project, repo, commit):
     """
     persons = [x for x in _get_patchwork_author_committer(commit) if x]
     patch = repo.get_patch(commit)
-    name = commit.message.split('\n')[0]
+    name = commit.message.decode().split('\n')[0]
     return get_patches_matching(project, persons, name, patch)
diff --git a/requirements.txt b/requirements.txt
index 010a63a..054e9b8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-Django==1.11
+Django==1.11.29
 djangorestframework==3.5
 django-filter==1.1
 sqlparse==0.2.2
diff --git a/tests/__init__.py b/tests/__init__.py
index ed6ac18..eaabb5e 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -37,11 +37,11 @@ class TestRepo(object):
 
     def last_commit(self):
         out = subprocess.check_output(
-            ['git', 'log', '--format=oneline', '-1'], cwd=self.path)
+            ['git', 'log', '--format=oneline', '-1'], cwd=self.path, text=True)
         return out.split(' ')[0]
 
     def create_patch(self, commit):
         patch = subprocess.check_output(
             ['git', 'format-patch', '%s^..%s' % (commit, commit)],
-            cwd=self.path)
+            cwd=self.path, text=True)
         return os.path.join(self.path, patch.strip())
diff --git a/tests/test_gitrepo.py b/tests/test_gitrepo.py
index d1ec23f..505c126 100644
--- a/tests/test_gitrepo.py
+++ b/tests/test_gitrepo.py
@@ -13,93 +13,100 @@ class TestGitRepo(unittest.TestCase):
 
         self.tmpdir = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, self.tmpdir)
-        self.main = gitrepo.Repo(self.tmpdir, 'main', None)
+        self.main = gitrepo.Repo(self.tmpdir, "main", None)
         os.mkdir(self.main.path)
-        subprocess.check_call(['git', 'init'], cwd=self.main.path)
+        subprocess.check_call(["git", "init"], cwd=self.main.path)
 
     def _add_commit(self, path, content, message):
-        with open(os.path.join(self.main.path, path), 'w') as f:
+        with open(os.path.join(self.main.path, path), "w") as f:
             f.write(content)
-        subprocess.check_call(['git', 'add', '.'], cwd=self.main.path)
         subprocess.check_call(
-            ['git', 'commit', '-a', '-m', message], cwd=self.main.path)
+            ["git", "add", "."], cwd=self.main.path, text=True)
+        subprocess.check_call(
+            ["git", "commit", "-a", "-m", message], cwd=self.main.path,
+            text=True
+        )
         return self._last_commit()
 
     def _last_commit(self):
         out = subprocess.check_output(
-            ['git', 'log', '--format=oneline', '-1'], cwd=self.main.path)
-        return out.split(' ')[0]
+            ["git", "log", "--format=oneline", "-1"], cwd=self.main.path,
+            text=True
+        )
+        return out.split(" ")[0]
 
     def test_pull_simple(self):
-        self._add_commit('foo1', 'commit1', 'commit1')
-        self._add_commit('foo2', 'commit2', 'commit2')
+        self._add_commit("foo1", "commit1", "commit1")
+        self._add_commit("foo2", "commit2", "commit2")
 
-        repo = gitrepo.Repo(self.tmpdir, 'clone', self.main.path)
+        repo = gitrepo.Repo(self.tmpdir, "clone", self.main.path)
         repo._clone()
 
         commits = []
-        commits.append(self._add_commit('foo3', 'commit3', 'commit3'))
-        commits.append(self._add_commit('foo4', 'commit4', 'commit4'))
+        commits.append(self._add_commit("foo3", "commit3", "commit3"))
+        commits.append(self._add_commit("foo4", "commit4", "commit4"))
 
         repo.update()
 
         out = subprocess.check_output(
-            ['git', 'log', '--format=oneline', '-2'], cwd=repo.path)
-        found = [x.split(' ')[0] for x in out.split('\n') if x]
+            ["git", "log", "--format=oneline", "-2"], cwd=repo.path, text=True
+        )
+        found = [x.split(" ")[0] for x in out.split("\n") if x]
         self.assertEqual(commits, list(reversed(found)))
 
     def test_pull_rewrite(self):
-        self._add_commit('foo1', 'commit1', 'commit1')
+        self._add_commit("foo1", "commit1", "commit1")
 
-        repo = gitrepo.Repo(self.tmpdir, 'clone', self.main.path)
+        repo = gitrepo.Repo(self.tmpdir, "clone", self.main.path)
         repo._clone()
 
         commits = []
-        commits.append(self._add_commit('foo3', 'commit3', 'commit3'))
+        commits.append(self._add_commit("foo3", "commit3", "commit3"))
         # now add this foo4 to be overwritten
-        self._add_commit('foo4', 'commit4', 'commit4')
+        self._add_commit("foo4", "commit4", "commit4")
         repo.update()
 
         subprocess.check_call(
-            ['git', 'reset', '--hard', 'HEAD^'], cwd=self.main.path)
-        commits.append(self._add_commit('foo4a', 'commit4a', 'commit4a'))
+            ["git", "reset", "--hard", "HEAD^"], cwd=self.main.path)
+        commits.append(self._add_commit("foo4a", "commit4a", "commit4a"))
         repo.update()
 
         out = subprocess.check_output(
-            ['git', 'log', '--format=oneline', '-2'], cwd=repo.path)
-        found = [x.split(' ')[0] for x in out.split('\n') if x]
+            ["git", "log", "--format=oneline", "-2"], cwd=repo.path
+        )
+        found = [x.split(" ")[0] for x in out.decode().split("\n") if x]
         self.assertEqual(commits, list(reversed(found)))
 
     def test_commits_to_check_empty(self):
-        '''works off an empty repo that's never been analyzed'''
+        """works off an empty repo that's never been analyzed"""
         commits = []
-        commits.append(self._add_commit('foo', 'foocontent', 'commit1'))
-        commits.append(self._add_commit('foo', 'foocontent2', 'commit2'))
+        commits.append(self._add_commit("foo", "foocontent", "commit1"))
+        commits.append(self._add_commit("foo", "foocontent2", "commit2"))
 
-        found = [x.id for x in self.main.process_unchecked_commits()]
+        found = [x.id.decode() for x in self.main.process_unchecked_commits()]
         self.assertEqual(commits, found)
 
     def test_commits_to_check_previous(self):
-        '''works off an empty repo that's been analyzed'''
-        self._add_commit('foo', 'foocontent', 'commit1')
+        """works off an empty repo that's been analyzed"""
+        self._add_commit("foo", "foocontent", "commit1")
         # force last commit file to be updated
         list(self.main.process_unchecked_commits())
 
         commits = []
-        commits.append(self._add_commit('foo', 'foocontent2', 'commit2'))
+        commits.append(self._add_commit("foo", "foocontent2", "commit2"))
 
-        found = [x.id for x in self.main.process_unchecked_commits()]
+        found = [x.id.decode() for x in self.main.process_unchecked_commits()]
         self.assertEqual(commits, found)
 
     def test_commits_to_check_rewrite(self):
-        '''can handle a history rewrite'''
+        """can handle a history rewrite"""
         commits = []
-        commits.append(self._add_commit('foo', 'foocontent', 'commit1'))
-        commits.append(self._add_commit('foo', 'foocontent2', 'commit2'))
+        commits.append(self._add_commit("foo", "foocontent", "commit1"))
+        commits.append(self._add_commit("foo", "foocontent2", "commit2"))
 
-        last_commit = os.path.join(self.main.path, 'patchwork-last-commit')
-        with open(last_commit, 'w') as f:
-            f.write('11111111111')  # invalid sha1, so we'll search back
+        last_commit = os.path.join(self.main.path, "patchwork-last-commit")
+        with open(last_commit, "w") as f:
+            f.write("11111111111")  # invalid sha1, so we'll search back
 
-        found = [x.id for x in self.main.process_unchecked_commits()]
+        found = [x.id.decode() for x in self.main.process_unchecked_commits()]
         self.assertEqual(commits, found)
diff --git a/unit-test.sh b/unit-test.sh
index 09801d1..031e55b 100755
--- a/unit-test.sh
+++ b/unit-test.sh
@@ -7,7 +7,7 @@ VENV_DIR="${VENV_DIR-$HERE/.venv}"
 
 if [ -z $VIRTUAL_ENV ] ; then
 	echo "creating venv: $VENV_DIR ..."
-	virtualenv --python=`which python2` $VENV_DIR
+	virtualenv --python=`which python3` $VENV_DIR
 	. $VENV_DIR/bin/activate
 	pip install -r requirements.txt
 	wget -q https://git.linaro.org/infrastructure/linaro-git-tools.git/plain/linaro_ldap.py
diff --git a/update_commited_patches.py b/update_commited_patches.py
index a2d73e9..9808da0 100755
--- a/update_commited_patches.py
+++ b/update_commited_patches.py
@@ -37,7 +37,7 @@ def _update_commit(project, repo, commit, dryrun):
     for i, patch in enumerate(patches):
         if i == 0:
             patch.state = accepted
-            patch.commit_ref = commit.id
+            patch.commit_ref = commit.id.decode()
         else:
             patch.state = superseded
         log.info('Updating patch %s, commit: %s, state: %s',