llvmbot monitor: Remove Buildkite support and libcxx bots

libcxx is no longer doing post-commit builds of the main branch.
Pre-commit builds continue but there's no need for us to monitor them,
folks will ping us if there are issues.

Change-Id: I063ec2a22b2c200da5d047bf80998ba4c884df68

5 files changed, 2 insertions, 215 deletions

diff --git a/monitor/README.txt b/monitor/README.txt
index ad7eda6..fd1e187 100644
--- a/monitor/README.txt
+++ b/monitor/README.txt
@@ -5,11 +5,8 @@ This is to be left running on a server or desktop as monitoring for your buildbo
 It purely reports the status of the builds. If you want hardware monitoring, look
 elsewhere.
 
-It supports Buildbot (as used by LLVM) and Buildkite (https://buildkite.com/, used by LLVM's
-libcxx project). It does not support LLVM Green Dragon (https://green.lab.llvm.org/green/).
-
-Buildkite does require an API key to be available, see the script for how to do that
-and contact your Builkite org admin to get one.
+It supports Buildbot (as used by LLVM) . It does not support LLVM Green Dragon
+(https://green.lab.llvm.org/green/).
 
 Currently we have one monitor running at http://llvm.validation.linaro.org/.
 Bookmark this if you have a need to check bot status at a glance.
@@ -20,9 +17,6 @@ JSON Format
 The JSON file describes the bots we want to monitor and which master/build service
 they connect to.
 
-Buildbot JSON Format
---------------------
-
 The base structure is a list of masters, which has a few properties and a list
 of builder groups, which in turn also have some properties and a list of bots
 (which in Buildbot terms are actually called "Builders" but we ended up calling
@@ -62,25 +56,6 @@ Note that "ignore" has two different behaviours:
  * On bots it shows the bot but ignores their status. Meaning that an ignored bot failing
    does not make the overall page status failed.
 
-Buildkite JSON Format
----------------------
-
-The Buildkite format follows the Buildbot format closely with some differences.
-
-  * Since Buildkite is a centralised service the "name" is always "Buildkite".
-  * The "base_url" is always "https://www.buildkite.com".
-  * There is a new key "buildkite_org". This is used to find our particular bots
-    via the API.
-  * The "builder_url" and "build_url" keys are used to form clickable links,
-    and are always "builders" and "builds".
-  * Bots have an extra key "buildkite_pipeline" (explained below).
-
-When querying Buildkite we make a request like this:
-"show me the results of the pipeline <buildkite_pipeline> in the organisation <buildkite_org>"
-
-From there the script looks for the bot's name in the last finished build. After that
-all the processing is the same as the Buildbot results.
-
 HTML Page
 ---------
 
diff --git a/monitor/bot-status.py b/monitor/bot-status.py
index f858ef6..a030d3e 100755
--- a/monitor/bot-status.py
+++ b/monitor/bot-status.py
@@ -25,9 +25,6 @@ import requests
 from textwrap import dedent
 from make_table import Table
 
-from buildkite_status import get_buildkite_bots_status
-
-
 def ignored(s):
     return "ignore" in s and s["ignore"]
 
@@ -132,9 +129,6 @@ def get_buildbot_bots_status(config):
     bot_cache = {}
 
     for server in filter(not_ignored, config):
-        if server["name"] == "Buildkite":
-            continue
-
         base_url = server["base_url"]
         logging.debug("Parsing server {}...".format(server["name"]))
         for builder in server["builders"]:
@@ -355,7 +349,6 @@ if __name__ == "__main__":
             status = pickle.load(f)
     else:
         status = get_buildbot_bots_status(config)
-        status.update(get_buildkite_bots_status(config))
         if args.cachefile:
             logging.debug("Writing status to cache file {}".format(args.cachefile))
             with open(args.cachefile, "wb") as f:
diff --git a/monitor/buildkite_status.py b/monitor/buildkite_status.py
deleted file mode 100755
index 39706c8..0000000
--- a/monitor/buildkite_status.py
+++ /dev/null
@@ -1,158 +0,0 @@
-#! /usr/bin/env python3
-
-import os
-import logging
-from datetime import timedelta
-from functools import lru_cache
-
-# Base dict order not guaranteed until 3.7+
-from collections import OrderedDict
-from datetime import datetime
-
-# To convert time stamps
-# (use datetime.datetime.fromisoformat in Python3.7+)
-import dateutil.parser
-from pybuildkite.buildkite import Buildkite, BuildState
-
-buildkite = Buildkite()
-
-
-def get_buildkite_token():
-    """
-    We are assuming that this script is run on a machine
-    that can only be accessed by users that would already be
-    able to generate such tokens.
-
-    We could use an env var but the monitor is run via cron
-    which would have to inject the token. So it's going to end
-    up on disk anyway.
-
-    The format of this file is just a single line which is the
-    token.
-    """
-    tokenfile_path = os.path.join("~", ".buildkite_token")
-    err_msg = "{}, Buildkite status will not be fetched."
-    token = None
-
-    try:
-        with open(os.path.expanduser(tokenfile_path)) as f:
-            # strip to remove ending newline
-            token = f.readline().strip()
-        if not token:
-            logging.debug(err_msg.format("Buildkite API token file was empty"))
-            token = None
-    except FileNotFoundError:
-        logging.debug(err_msg.format("Couldn't read Buildkite API token file"))
-
-    return token
-
-
-@lru_cache(maxsize=None)
-def get_pipeline_builds(organisation, pipeline):
-    return buildkite.builds().list_all_for_pipeline(
-        organisation,
-        pipeline,
-        # Finished means passed, failed, blocked or cancelled
-        states=[BuildState.FINISHED],
-        # There are two kinds of builds. Those triggered from phab reviews
-        # and those run perdidically to post-commit build the main and release
-        # branches.
-        # For the status page we ignore pre-commit and release builds.
-        branch="main",
-    )
-
-
-def get_last_finished_build_status(organisation, job_name, pipeline):
-    """
-    Get information about the last finished build of a particular
-    job. (where a job is one of the pipeline steps)
-    """
-    builds = get_pipeline_builds(organisation, pipeline)
-
-    for build in builds:
-        # Where each "job" is a config e.g. AArch64, MacOS, C++20 etc.
-        for job in build["jobs"]:
-            # Some jobs won't have a name. I think these are the wait steps
-            # between jobs or groups of jobs.
-            if job.get("name") != job_name:
-                continue
-
-            # Some jobs were never started due to failures earlier in the pipeline.
-            # This isn't our fault so ignore these.
-            started_time = job.get("started_at")
-            if started_time is None:
-                continue
-
-            # We use started because that's when it starts to
-            # run on the agent (don't want to include waiting time)
-            started = dateutil.parser.isoparse(started_time)
-            finished = dateutil.parser.isoparse(job["finished_at"])
-            # Buildkite gives us mS resoloution but we don't need that
-            duration = timedelta(seconds=int((finished - started).total_seconds()))
-            # Time from buildkite ends up as UTC but now has no
-            # timezone info by default
-            now = datetime.now(finished.tzinfo)
-            time_since = timedelta(seconds=int((now - finished).total_seconds()))
-
-            state = job["state"]
-            failed = state != "passed"
-            status = {
-                # 'url' is the API url, we want a clickable link
-                "builder_url": job["agent"]["web_url"],
-                # We use the job's url because it takes you to the
-                # specific agent, instead of the build's url
-                "build_url": job["web_url"],
-                "number": build["number"],
-                "state": state,
-                "duration": duration,
-                "time_since": time_since,
-                "fail": failed,
-            }
-            return status
-
-    logging.debug(
-        'Found no finished builds for job "{}" '
-        'on pipeline "{}"'.format(job_name, pipeline)
-    )
-    return dict(valid=False)
-
-
-def bot_key(job_name):
-    return ("https://www.buildkite.com", job_name)
-
-
-# Get status for all Buildkite bots in config
-# Return a dictionary of (base_url, bot name) -> status info
-def get_buildkite_bots_status(config):
-    status = dict()
-
-    token = get_buildkite_token()
-    buildkite.set_access_token(token)
-
-    for server in config:
-        if server["name"] != "Buildkite":
-            continue
-
-        if token is None:
-            # We cannot query buildkite but we don't want to mark these
-            # bots as offline because we don't know that.
-            for builder in server["builders"]:
-                for job in builder["bots"]:
-                    status[bot_key(job["name"])] = {"valid": False}
-            continue
-
-        # If we have a valid token then we can get the real status
-        organisation = server.get("buildkite_org")
-        if organisation is None:
-            raise RuntimeError("Buildkite servers must also have a 'buildkite_org'")
-
-        for builder in server["builders"]:
-            for job in builder["bots"]:
-                job_name = job["name"]
-                logging.debug('Getting status for buildkite job "{}"'.format(job_name))
-
-                status[bot_key(job_name)] = get_last_finished_build_status(
-                    organisation, job_name, job["buildkite_pipeline"]
-                )
-
-    return status
diff --git a/monitor/install.sh b/monitor/install.sh
index 1b8b703..db8e10d 100755
--- a/monitor/install.sh
+++ b/monitor/install.sh
@@ -49,9 +49,6 @@ fi
 if ! python3 -c 'import requests' > /dev/null; then
   echo 'python3-requests module missing'
 fi
-if ! python3 -c 'import pybuildkite' > /dev/null; then
-  echo 'python3-pybuildkite module missing'
-fi
 
 ############################# Install
 
diff --git a/monitor/linaro.json b/monitor/linaro.json
index 3f4bf12..637062c 100644
--- a/monitor/linaro.json
+++ b/monitor/linaro.json
@@ -77,25 +77,5 @@
         ]
       }
     ]
-  },
-  {
-    "name": "Buildkite",
-    "base_url": "https://www.buildkite.com",
-    "buildkite_org": "llvm-project",
-    "builder_url": "builders",
-    "build_url": "builds",
-    "builders": [
-      {
-        "name": "Libcxx",
-        "bots": [
-          { "name": "Armv7",                   "buildkite_pipeline": "libcxx-ci" },
-          { "name": "Armv7 -fno-exceptions",   "buildkite_pipeline": "libcxx-ci" },
-          { "name": "Armv8",                   "buildkite_pipeline": "libcxx-ci" },
-          { "name": "Armv8 -fno-exceptions",   "buildkite_pipeline": "libcxx-ci" },
-          { "name": "AArch64",                 "buildkite_pipeline": "libcxx-ci" },
-          { "name": "AArch64 -fno-exceptions", "buildkite_pipeline": "libcxx-ci" }
-        ]
-      }
-    ]
   }
 ]