DRILL-6289: Cluster view should show more relevant information

Protobuf change to carry HTTP port info
Allow CORS for access to remote Drillbit metrics
Cross-origin resource sharing (CORS) is required to ensure that the WebServer is able serve REST calls for status pages.
Materialize relevant metrics
  1. Heap memory (incl usage)
  2. Heap memory (incl usage)
  3. Average System Load (last 1 min)
  4. Option to view from other nodes (pop out)
  5. Added Glyphicons
Update System Table and related tests
  1. Updated System Table to show HTTP port
  2. Updated unit tests
Skip updating remote bit info when HTTPS (SSL) or Authentication is enabled.
Default CpuGaugeSet is public; Added Gauges
  * CPU Utiization by Drill
  * Uptime
Show ALL Buttons, but do HTTPS Check
Reduce power button to icon
Allowing CORS for /status/metrics only
Accounting for situations when JVM does not report Process CPU Load
  i.e. returned value is negative.
  See https://docs.oracle.com/javase/7/docs/jre/api/management/extension/com/sun/management/OperatingSystemMXBean.html#getProcessCpuLoad()
Addressed shutdown security conditions
Added C++ Client Protobuf
Added steps for Protobuf generation to protocol/readme.txt

This closes #1203

4 files changed, 102 insertions, 20 deletions

diff --git a/contrib/native/client/src/protobuf/Coordination.pb.cc b/contrib/native/client/src/protobuf/Coordination.pb.cc
index 14b3103fc..31ffcceb4 100644
--- a/contrib/native/client/src/protobuf/Coordination.pb.cc
+++ b/contrib/native/client/src/protobuf/Coordination.pb.cc
@@ -41,7 +41,7 @@ void protobuf_AssignDesc_Coordination_2eproto() {
       "Coordination.proto");
   GOOGLE_CHECK(file != NULL);
   DrillbitEndpoint_descriptor_ = file->message_type(0);
-  static const int DrillbitEndpoint_offsets_[7] = {
+  static const int DrillbitEndpoint_offsets_[8] = {
     GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(DrillbitEndpoint, address_),
     GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(DrillbitEndpoint, user_port_),
     GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(DrillbitEndpoint, control_port_),
@@ -49,6 +49,7 @@ void protobuf_AssignDesc_Coordination_2eproto() {
     GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(DrillbitEndpoint, roles_),
     GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(DrillbitEndpoint, version_),
     GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(DrillbitEndpoint, state_),
+    GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(DrillbitEndpoint, http_port_),
   };
   DrillbitEndpoint_reflection_ =
     new ::google::protobuf::internal::GeneratedMessageReflection(
@@ -136,21 +137,22 @@ void protobuf_AddDesc_Coordination_2eproto() {
   GOOGLE_PROTOBUF_VERIFY_VERSION;
 
   ::google::protobuf::DescriptorPool::InternalAddGeneratedFile(
-    "\n\022Coordination.proto\022\004exec\"\367\001\n\020DrillbitE"
+    "\n\022Coordination.proto\022\004exec\"\212\002\n\020DrillbitE"
     "ndpoint\022\017\n\007address\030\001 \001(\t\022\021\n\tuser_port\030\002 "
     "\001(\005\022\024\n\014control_port\030\003 \001(\005\022\021\n\tdata_port\030\004"
     " \001(\005\022\032\n\005roles\030\005 \001(\0132\013.exec.Roles\022\017\n\007vers"
     "ion\030\006 \001(\t\022+\n\005state\030\007 \001(\0162\034.exec.Drillbit"
-    "Endpoint.State\"<\n\005State\022\013\n\007STARTUP\020\000\022\n\n\006"
-    "ONLINE\020\001\022\r\n\tQUIESCENT\020\002\022\013\n\007OFFLINE\020\003\"i\n\024"
-    "DrillServiceInstance\022\n\n\002id\030\001 \001(\t\022\033\n\023regi"
-    "strationTimeUTC\030\002 \001(\003\022(\n\010endpoint\030\003 \001(\0132"
-    "\026.exec.DrillbitEndpoint\"\227\001\n\005Roles\022\027\n\tsql"
-    "_query\030\001 \001(\010:\004true\022\032\n\014logical_plan\030\002 \001(\010"
-    ":\004true\022\033\n\rphysical_plan\030\003 \001(\010:\004true\022\033\n\rj"
-    "ava_executor\030\004 \001(\010:\004true\022\037\n\021distributed_"
-    "cache\030\005 \001(\010:\004trueB3\n\033org.apache.drill.ex"
-    "ec.protoB\022CoordinationProtosH\001", 590);
+    "Endpoint.State\022\021\n\thttp_port\030\010 \001(\005\"<\n\005Sta"
+    "te\022\013\n\007STARTUP\020\000\022\n\n\006ONLINE\020\001\022\r\n\tQUIESCENT"
+    "\020\002\022\013\n\007OFFLINE\020\003\"i\n\024DrillServiceInstance\022"
+    "\n\n\002id\030\001 \001(\t\022\033\n\023registrationTimeUTC\030\002 \001(\003"
+    "\022(\n\010endpoint\030\003 \001(\0132\026.exec.DrillbitEndpoi"
+    "nt\"\227\001\n\005Roles\022\027\n\tsql_query\030\001 \001(\010:\004true\022\032\n"
+    "\014logical_plan\030\002 \001(\010:\004true\022\033\n\rphysical_pl"
+    "an\030\003 \001(\010:\004true\022\033\n\rjava_executor\030\004 \001(\010:\004t"
+    "rue\022\037\n\021distributed_cache\030\005 \001(\010:\004trueB3\n\033"
+    "org.apache.drill.exec.protoB\022Coordinatio"
+    "nProtosH\001", 609);
   ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile(
     "Coordination.proto", &protobuf_RegisterTypes);
   DrillbitEndpoint::default_instance_ = new DrillbitEndpoint();
@@ -204,6 +206,7 @@ const int DrillbitEndpoint::kDataPortFieldNumber;
 const int DrillbitEndpoint::kRolesFieldNumber;
 const int DrillbitEndpoint::kVersionFieldNumber;
 const int DrillbitEndpoint::kStateFieldNumber;
+const int DrillbitEndpoint::kHttpPortFieldNumber;
 #endif  // !_MSC_VER
 
 DrillbitEndpoint::DrillbitEndpoint()
@@ -230,6 +233,7 @@ void DrillbitEndpoint::SharedCtor() {
   roles_ = NULL;
   version_ = const_cast< ::std::string*>(&::google::protobuf::internal::kEmptyString);
   state_ = 0;
+  http_port_ = 0;
   ::memset(_has_bits_, 0, sizeof(_has_bits_));
 }
 
@@ -289,6 +293,7 @@ void DrillbitEndpoint::Clear() {
       }
     }
     state_ = 0;
+    http_port_ = 0;
   }
   ::memset(_has_bits_, 0, sizeof(_has_bits_));
   mutable_unknown_fields()->Clear();
@@ -412,6 +417,22 @@ bool DrillbitEndpoint::MergePartialFromCodedStream(
         } else {
           goto handle_uninterpreted;
         }
+        if (input->ExpectTag(64)) goto parse_http_port;
+        break;
+      }
+
+      // optional int32 http_port = 8;
+      case 8: {
+        if (::google::protobuf::internal::WireFormatLite::GetTagWireType(tag) ==
+            ::google::protobuf::internal::WireFormatLite::WIRETYPE_VARINT) {
+         parse_http_port:
+          DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive<
+                   ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>(
+                 input, &http_port_)));
+          set_has_http_port();
+        } else {
+          goto handle_uninterpreted;
+        }
         if (input->ExpectAtEnd()) return true;
         break;
       }
@@ -479,6 +500,11 @@ void DrillbitEndpoint::SerializeWithCachedSizes(
       7, this->state(), output);
   }
 
+  // optional int32 http_port = 8;
+  if (has_http_port()) {
+    ::google::protobuf::internal::WireFormatLite::WriteInt32(8, this->http_port(), output);
+  }
+
   if (!unknown_fields().empty()) {
     ::google::protobuf::internal::WireFormat::SerializeUnknownFields(
         unknown_fields(), output);
@@ -535,6 +561,11 @@ void DrillbitEndpoint::SerializeWithCachedSizes(
       7, this->state(), target);
   }
 
+  // optional int32 http_port = 8;
+  if (has_http_port()) {
+    target = ::google::protobuf::internal::WireFormatLite::WriteInt32ToArray(8, this->http_port(), target);
+  }
+
   if (!unknown_fields().empty()) {
     target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray(
         unknown_fields(), target);
@@ -594,6 +625,13 @@ int DrillbitEndpoint::ByteSize() const {
         ::google::protobuf::internal::WireFormatLite::EnumSize(this->state());
     }
 
+    // optional int32 http_port = 8;
+    if (has_http_port()) {
+      total_size += 1 +
+        ::google::protobuf::internal::WireFormatLite::Int32Size(
+          this->http_port());
+    }
+
   }
   if (!unknown_fields().empty()) {
     total_size +=
@@ -642,6 +680,9 @@ void DrillbitEndpoint::MergeFrom(const DrillbitEndpoint& from) {
     if (from.has_state()) {
       set_state(from.state());
     }
+    if (from.has_http_port()) {
+      set_http_port(from.http_port());
+    }
   }
   mutable_unknown_fields()->MergeFrom(from.unknown_fields());
 }
@@ -672,6 +713,7 @@ void DrillbitEndpoint::Swap(DrillbitEndpoint* other) {
     std::swap(roles_, other->roles_);
     std::swap(version_, other->version_);
     std::swap(state_, other->state_);
+    std::swap(http_port_, other->http_port_);
     std::swap(_has_bits_[0], other->_has_bits_[0]);
     _unknown_fields_.Swap(&other->_unknown_fields_);
     std::swap(_cached_size_, other->_cached_size_);
diff --git a/contrib/native/client/src/protobuf/Coordination.pb.h b/contrib/native/client/src/protobuf/Coordination.pb.h
index e52c70c40..07bbc8e33 100644
--- a/contrib/native/client/src/protobuf/Coordination.pb.h
+++ b/contrib/native/client/src/protobuf/Coordination.pb.h
@@ -202,6 +202,13 @@ class DrillbitEndpoint : public ::google::protobuf::Message {
   inline ::exec::DrillbitEndpoint_State state() const;
   inline void set_state(::exec::DrillbitEndpoint_State value);
 
+  // optional int32 http_port = 8;
+  inline bool has_http_port() const;
+  inline void clear_http_port();
+  static const int kHttpPortFieldNumber = 8;
+  inline ::google::protobuf::int32 http_port() const;
+  inline void set_http_port(::google::protobuf::int32 value);
+
   // @@protoc_insertion_point(class_scope:exec.DrillbitEndpoint)
  private:
   inline void set_has_address();
@@ -218,6 +225,8 @@ class DrillbitEndpoint : public ::google::protobuf::Message {
   inline void clear_has_version();
   inline void set_has_state();
   inline void clear_has_state();
+  inline void set_has_http_port();
+  inline void clear_has_http_port();
 
   ::google::protobuf::UnknownFieldSet _unknown_fields_;
 
@@ -228,9 +237,10 @@ class DrillbitEndpoint : public ::google::protobuf::Message {
   ::google::protobuf::int32 data_port_;
   int state_;
   ::std::string* version_;
+  ::google::protobuf::int32 http_port_;
 
   mutable int _cached_size_;
-  ::google::protobuf::uint32 _has_bits_[(7 + 31) / 32];
+  ::google::protobuf::uint32 _has_bits_[(8 + 31) / 32];
 
   friend void  protobuf_AddDesc_Coordination_2eproto();
   friend void protobuf_AssignDesc_Coordination_2eproto();
@@ -744,6 +754,28 @@ inline void DrillbitEndpoint::set_state(::exec::DrillbitEndpoint_State value) {
   state_ = value;
 }
 
+// optional int32 http_port = 8;
+inline bool DrillbitEndpoint::has_http_port() const {
+  return (_has_bits_[0] & 0x00000080u) != 0;
+}
+inline void DrillbitEndpoint::set_has_http_port() {
+  _has_bits_[0] |= 0x00000080u;
+}
+inline void DrillbitEndpoint::clear_has_http_port() {
+  _has_bits_[0] &= ~0x00000080u;
+}
+inline void DrillbitEndpoint::clear_http_port() {
+  http_port_ = 0;
+  clear_has_http_port();
+}
+inline ::google::protobuf::int32 DrillbitEndpoint::http_port() const {
+  return http_port_;
+}
+inline void DrillbitEndpoint::set_http_port(::google::protobuf::int32 value) {
+  set_has_http_port();
+  http_port_ = value;
+}
+
 // -------------------------------------------------------------------
 
 // DrillServiceInstance
diff --git a/contrib/native/client/src/protobuf/UserBitShared.pb.cc b/contrib/native/client/src/protobuf/UserBitShared.pb.cc
index 4350082e2..c5b25279b 100644
--- a/contrib/native/client/src/protobuf/UserBitShared.pb.cc
+++ b/contrib/native/client/src/protobuf/UserBitShared.pb.cc
@@ -749,7 +749,7 @@ void protobuf_AddDesc_UserBitShared_2eproto() {
     "agmentState\022\013\n\007SENDING\020\000\022\027\n\023AWAITING_ALL"
     "OCATION\020\001\022\013\n\007RUNNING\020\002\022\014\n\010FINISHED\020\003\022\r\n\t"
     "CANCELLED\020\004\022\n\n\006FAILED\020\005\022\032\n\026CANCELLATION_"
-    "REQUESTED\020\006*\227\006\n\020CoreOperatorType\022\021\n\rSING"
+    "REQUESTED\020\006*\302\006\n\020CoreOperatorType\022\021\n\rSING"
     "LE_SENDER\020\000\022\024\n\020BROADCAST_SENDER\020\001\022\n\n\006FIL"
     "TER\020\002\022\022\n\016HASH_AGGREGATE\020\003\022\r\n\tHASH_JOIN\020\004"
     "\022\016\n\nMERGE_JOIN\020\005\022\031\n\025HASH_PARTITION_SENDE"
@@ -769,10 +769,12 @@ void protobuf_AddDesc_UserBitShared_2eproto() {
     "ASE_SUB_SCAN\020!\022\n\n\006WINDOW\020\"\022\024\n\020NESTED_LOO"
     "P_JOIN\020#\022\021\n\rAVRO_SUB_SCAN\020$\022\021\n\rPCAP_SUB_"
     "SCAN\020%\022\022\n\016KAFKA_SUB_SCAN\020&\022\021\n\rKUDU_SUB_S"
-    "CAN\020\'*g\n\nSaslStatus\022\020\n\014SASL_UNKNOWN\020\000\022\016\n"
-    "\nSASL_START\020\001\022\024\n\020SASL_IN_PROGRESS\020\002\022\020\n\014S"
-    "ASL_SUCCESS\020\003\022\017\n\013SASL_FAILED\020\004B.\n\033org.ap"
-    "ache.drill.exec.protoB\rUserBitSharedH\001", 5078);
+    "CAN\020\'\022\013\n\007FLATTEN\020(\022\020\n\014LATERAL_JOIN\020)\022\n\n\006"
+    "UNNEST\020**g\n\nSaslStatus\022\020\n\014SASL_UNKNOWN\020\000"
+    "\022\016\n\nSASL_START\020\001\022\024\n\020SASL_IN_PROGRESS\020\002\022\020"
+    "\n\014SASL_SUCCESS\020\003\022\017\n\013SASL_FAILED\020\004B.\n\033org"
+    ".apache.drill.exec.protoB\rUserBitSharedH"
+    "\001", 5121);
   ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile(
     "UserBitShared.proto", &protobuf_RegisterTypes);
   UserCredentials::default_instance_ = new UserCredentials();
@@ -933,6 +935,9 @@ bool CoreOperatorType_IsValid(int value) {
     case 37:
     case 38:
     case 39:
+    case 40:
+    case 41:
+    case 42:
       return true;
     default:
       return false;
diff --git a/contrib/native/client/src/protobuf/UserBitShared.pb.h b/contrib/native/client/src/protobuf/UserBitShared.pb.h
index 5a49bf50d..1a6990657 100644
--- a/contrib/native/client/src/protobuf/UserBitShared.pb.h
+++ b/contrib/native/client/src/protobuf/UserBitShared.pb.h
@@ -243,11 +243,14 @@ enum CoreOperatorType {
   AVRO_SUB_SCAN = 36,
   PCAP_SUB_SCAN = 37,
   KAFKA_SUB_SCAN = 38,
-  KUDU_SUB_SCAN = 39
+  KUDU_SUB_SCAN = 39,
+  FLATTEN = 40,
+  LATERAL_JOIN = 41,
+  UNNEST = 42
 };
 bool CoreOperatorType_IsValid(int value);
 const CoreOperatorType CoreOperatorType_MIN = SINGLE_SENDER;
-const CoreOperatorType CoreOperatorType_MAX = KUDU_SUB_SCAN;
+const CoreOperatorType CoreOperatorType_MAX = UNNEST;
 const int CoreOperatorType_ARRAYSIZE = CoreOperatorType_MAX + 1;
 
 const ::google::protobuf::EnumDescriptor* CoreOperatorType_descriptor();