DRILL-6494: Drill Plugins Handler

- Storage Plugins Handler service is used op the Drill start-up stage and it updates storage plugins configs from
  storage-plugins-override.conf file. If plugins configs are present in the persistence store - they are updated,
  otherwise bootstrap plugins are updated and the result configs are loaded to persistence store. If the enabled
  status is absent in the storage-plugins-override.conf file, the last plugin config enabled status persists.
- 'drill.exec.storage.action_on_plugins_override_file' Boot option is added. This is the action, which should be
  performed on the storage-plugins-override.conf file after successful updating storage plugins configs.
  Possible values are: "none" (default), "rename" and "remove".
- The "NULL" issue with updating Hive plugin config by REST is solved. But clients are still being instantiated for disabled
  plugins - DRILL-6412.
- "org.honton.chas.hocon:jackson-dataformat-hocon" library is added for the proper deserializing HOCON conf file
- additional refactoring: "com.typesafe:config" and "org.apache.commons:commons-lang3" are placed into DependencyManagement
  block with proper versions; correct properties for metrics in "drill-override-example.conf" are specified

closes #1345

6 files changed, 19 insertions, 19 deletions

diff --git a/contrib/storage-hbase/src/main/resources/bootstrap-storage-plugins.json b/contrib/storage-hbase/src/main/resources/bootstrap-storage-plugins.json
index 3e0e8c041..530a407c8 100644
--- a/contrib/storage-hbase/src/main/resources/bootstrap-storage-plugins.json
+++ b/contrib/storage-hbase/src/main/resources/bootstrap-storage-plugins.json
@@ -2,11 +2,11 @@
   "storage":{
     hbase : {
       type:"hbase",
-      enabled: false,
       config : {
         "hbase.zookeeper.quorum" : "localhost",
         "hbase.zookeeper.property.clientPort" : 2181
-      }
+      },
+      enabled: false
     }
   }
 }
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
index d3115b8a6..e3cb3a2dd 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
@@ -27,7 +27,6 @@ import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import com.google.common.cache.RemovalListener;
-import com.google.common.cache.RemovalNotification;
 import org.apache.calcite.schema.SchemaPlus;
 import org.apache.drill.common.exceptions.DrillRuntimeException;
 import org.apache.drill.common.exceptions.ExecutionSetupException;
@@ -72,6 +71,7 @@ public class HiveSchemaFactory implements SchemaFactory {
     isDrillImpersonationEnabled = plugin.getContext().getConfig().getBoolean(ExecConstants.IMPERSONATION_ENABLED);
 
     try {
+      // TODO: DRILL-6412. Clients for plugin should be instantiated only for the case, when plugin is enabled
       processUserMetastoreClient =
           DrillHiveMetaStoreClient.createCloseableClientWithCaching(hiveConf);
     } catch (MetaException e) {
@@ -82,12 +82,9 @@ public class HiveSchemaFactory implements SchemaFactory {
         .newBuilder()
         .expireAfterAccess(10, TimeUnit.MINUTES)
         .maximumSize(5) // Up to 5 clients for impersonation-enabled.
-        .removalListener(new RemovalListener<String, DrillHiveMetaStoreClient>() {
-          @Override
-          public void onRemoval(RemovalNotification<String, DrillHiveMetaStoreClient> notification) {
-            DrillHiveMetaStoreClient client = notification.getValue();
-            client.close();
-          }
+        .removalListener((RemovalListener<String, DrillHiveMetaStoreClient>) notification -> {
+          DrillHiveMetaStoreClient client = notification.getValue();
+          client.close();
         })
         .build(new CacheLoader<String, DrillHiveMetaStoreClient>() {
           @Override
diff --git a/contrib/storage-hive/core/src/main/resources/bootstrap-storage-plugins.json b/contrib/storage-hive/core/src/main/resources/bootstrap-storage-plugins.json
index 5c7174e25..d06220fe7 100644
--- a/contrib/storage-hive/core/src/main/resources/bootstrap-storage-plugins.json
+++ b/contrib/storage-hive/core/src/main/resources/bootstrap-storage-plugins.json
@@ -2,14 +2,16 @@
   "storage":{
     hive : {
       type:"hive",
-      enabled: false,
       config : {
         "hive.metastore.uris" : "",
         "javax.jdo.option.ConnectionURL" : "jdbc:derby:;databaseName=../sample-data/drill_hive_db;create=true",
         "hive.metastore.warehouse.dir" : "/tmp/drill_hive_wh",
         "fs.default.name" : "file:///",
-        "hive.metastore.sasl.enabled" : "false"
-      }
+        "hive.metastore.sasl.enabled" : "false",
+        "hive.metastore.schema.verification": "false",
+        "datanucleus.schema.autoCreateAll": "true"
+      },
+      enabled: false
     }
   }
 }
diff --git a/contrib/storage-jdbc/src/test/resources/bootstrap-storage-plugins.json b/contrib/storage-jdbc/src/test/resources/bootstrap-storage-plugins.json
index add980847..4018d9247 100755
--- a/contrib/storage-jdbc/src/test/resources/bootstrap-storage-plugins.json
+++ b/contrib/storage-jdbc/src/test/resources/bootstrap-storage-plugins.json
@@ -2,15 +2,16 @@
     "storage" : {
         derby : {
           type    : "jdbc",
-          enabled : true,
           driver  : "org.apache.derby.jdbc.ClientDriver",
-          url     : "jdbc:derby://localhost:${derby.reserved.port}/memory:${derby.database.name};user=root;password=root"
+          url     : "jdbc:derby://localhost:${derby.reserved.port}/memory:${derby.database.name};user=root;password=root",
+          enabled : true
         },
         mysql : {
           type    : "jdbc",
           enabled : true,
           driver  : "com.mysql.jdbc.Driver",
-          url     : "jdbc:mysql://localhost:${mysql.reserved.port}/${mysql.database.name}?user=root&password=root&useJDBCCompliantTimezoneShift=true"
+          url     : "jdbc:mysql://localhost:${mysql.reserved.port}/${mysql.database.name}?user=root&password=root&useJDBCCompliantTimezoneShift=true",
+          enabled : true
       }
     }
 }
diff --git a/contrib/storage-kafka/src/main/resources/bootstrap-storage-plugins.json b/contrib/storage-kafka/src/main/resources/bootstrap-storage-plugins.json
index 406c03060..18a1df564 100644
--- a/contrib/storage-kafka/src/main/resources/bootstrap-storage-plugins.json
+++ b/contrib/storage-kafka/src/main/resources/bootstrap-storage-plugins.json
@@ -2,8 +2,8 @@
   "storage":{
     kafka : {
       type:"kafka",
-      enabled: false,
-      kafkaConsumerProps: {"bootstrap.servers":"localhost:9092", "group.id" : "drill-consumer"}
+      kafkaConsumerProps: {"bootstrap.servers":"localhost:9092", "group.id" : "drill-consumer"},
+      enabled: false
     }
   }
 }
diff --git a/contrib/storage-mongo/src/main/resources/bootstrap-storage-plugins.json b/contrib/storage-mongo/src/main/resources/bootstrap-storage-plugins.json
index b7d34f2fb..9983596d3 100644
--- a/contrib/storage-mongo/src/main/resources/bootstrap-storage-plugins.json
+++ b/contrib/storage-mongo/src/main/resources/bootstrap-storage-plugins.json
@@ -2,8 +2,8 @@
   "storage":{
     mongo : {
       type:"mongo",
-      enabled: false,
-      connection:"mongodb://localhost:27017/"
+      connection:"mongodb://localhost:27017/",
+      enabled: false
     }
   }
 }