es-scripts: add new scripts to support elasticsearch stability

This change adds in some more advanced scripts meant
to help improve our elasticsearch uptime.  All scripts
are python3 and will require virtualenv to run.

Change-Id: I6758b16b5f7fc83f33faa706423785fd07cc59c5

1 files changed, 42 insertions, 0 deletions

diff --git a/status-autoclear.py b/status-autoclear.py
new file mode 100755
index 0000000..23b88ff
--- /dev/null
+++ b/status-autoclear.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+# This script checks on the status of the cluster
+# and if it's in a failed state due to a shard
+# collision on allocation for the creation of
+# an index, it will automatically clear the
+# index.
+
+from elasticsearch import Elasticsearch
+from elasticsearch.exceptions import RequestError
+import requests
+import json
+from datetime import datetime,timedelta
+
+import sys
+
+ES_HOST="elasticsearch-production"
+ES_PORT=9200
+
+es = Elasticsearch([{'host':ES_HOST,'port':ES_PORT}])
+
+health = es.cluster.health()
+
+if health['status'] == 'red':
+    try:
+        alloc = es.cluster.allocation_explain()
+        # ferret out the stuck index and delete it
+        troublemaker = alloc["index"]
+        if alloc['unassigned_info']['reason'] == "ALLOCATION_FAILED":
+            # let's try to remove it
+            es.indices.delete(index=troublemaker, timeout=60)
+        else:
+            # haven't seen it before... dump error and get a human
+            print(alloc)
+            sys.exit(1)
+    except RequestError as re:
+        # something we don't know how to handle... get a human
+        raise re
+        sys.exit(1)
+
+# either not red, or the delete worked
+sys.exit(0)