diff options
author | Roman Shaposhnik <rvs@apache.org> | 2017-03-22 08:51:22 -0700 |
---|---|---|
committer | Roman Shaposhnik <rvs@apache.org> | 2017-03-23 10:27:16 -0700 |
commit | 5e342c45364ec97f5e3530769a1cc8bdbcf69bb0 (patch) | |
tree | d999a06ca34dd972b07b3c6d5782de147be54172 /bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java | |
parent | 77e0d6e01f3e43f4a7e51a588b2261780c587ae2 (diff) |
BIGTOP-2704. Include ODPi runtime tests option into the battery of smoke tests
Diffstat (limited to 'bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java')
-rw-r--r-- | bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java | 158 |
1 files changed, 158 insertions, 0 deletions
diff --git a/bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java b/bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java new file mode 100644 index 00000000..0ea49ce8 --- /dev/null +++ b/bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.odpi.specs.runtime.hive; + +import org.apache.commons.exec.CommandLine; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; +import org.apache.hive.hcatalog.data.schema.HCatSchema; +import org.apache.thrift.TException; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; + + +public class TestHCatalog { + private static final String JOBJAR = "odpi.test.hive.hcat.job.jar"; + private static final String HCATCORE = "odpi.test.hive.hcat.core.jar"; + + private static final Log LOG = LogFactory.getLog(TestHCatalog.class.getName()); + + private static IMetaStoreClient client = null; + private static HiveConf conf; + private static HCatSchema inputSchema; + private static HCatSchema outputSchema; + + private Random rand; + + @BeforeClass + public static void connect() throws MetaException { + if (JdbcConnector.testActive(JdbcConnector.TEST_HCATALOG, "Test HCatalog ")) { + String hiveConfDir = JdbcConnector.getProperty(JdbcConnector.HIVE_CONF_DIR, + "Hive conf directory "); + String hadoopConfDir = JdbcConnector.getProperty(JdbcConnector.HADOOP_CONF_DIR, + "Hadoop conf directory "); + conf = new HiveConf(); + String fileSep = System.getProperty("file.separator"); + conf.addResource(new Path(hadoopConfDir + fileSep + "core-site.xml")); + conf.addResource(new Path(hadoopConfDir + fileSep + "hdfs-site.xml")); + conf.addResource(new Path(hadoopConfDir + fileSep + "yarn-site.xml")); + conf.addResource(new Path(hadoopConfDir + fileSep + "mapred-site.xml")); + conf.addResource(new Path(hiveConfDir + fileSep + "hive-site.xml")); + client = new HiveMetaStoreClient(conf); + + } + } + + @Before + public void checkIfActive() { + Assume.assumeTrue(JdbcConnector.testActive(JdbcConnector.TEST_HCATALOG, "Test HCatalog ")); + rand = new Random(); + } + + @Test + public void hcatInputFormatOutputFormat() throws TException, IOException, ClassNotFoundException, + InterruptedException, URISyntaxException { + // Create a table to write to + final String inputTable = "odpi_hcat_input_table_" + rand.nextInt(Integer.MAX_VALUE); + SerDeInfo serde = new SerDeInfo("default_serde", + conf.getVar(HiveConf.ConfVars.HIVEDEFAULTSERDE), new HashMap<String, String>()); + FieldSchema schema = new FieldSchema("line", "string", ""); + inputSchema = new HCatSchema(Collections.singletonList(new HCatFieldSchema(schema.getName(), + HCatFieldSchema.Type.STRING, schema.getComment()))); + StorageDescriptor sd = new StorageDescriptor(Collections.singletonList(schema), null, + "org.apache.hadoop.mapred.TextInputFormat", + "org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat", false, 0, serde, null, null, + new HashMap<String, String>()); + Table table = new Table(inputTable, "default", "me", 0, 0, 0, sd, null, + new HashMap<String, String>(), null, null, TableType.MANAGED_TABLE.toString()); + client.createTable(table); + + final String outputTable = "odpi_hcat_output_table_" + rand.nextInt(Integer.MAX_VALUE); + sd = new StorageDescriptor(Arrays.asList( + new FieldSchema("word", "string", ""), + new FieldSchema("count", "int", "")), + null, "org.apache.hadoop.mapred.TextInputFormat", + "org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat", false, 0, serde, null, null, + new HashMap<String, String>()); + table = new Table(outputTable, "default", "me", 0, 0, 0, sd, null, + new HashMap<String, String>(), null, null, TableType.MANAGED_TABLE.toString()); + client.createTable(table); + outputSchema = new HCatSchema(Arrays.asList( + new HCatFieldSchema("word", HCatFieldSchema.Type.STRING, ""), + new HCatFieldSchema("count", HCatFieldSchema.Type.INT, ""))); + + // LATER Could I use HCatWriter here and the reader to read it? + // Write some stuff into a file in the location of the table + table = client.getTable("default", inputTable); + String inputFile = table.getSd().getLocation() + "/input"; + Path inputPath = new Path(inputFile); + FileSystem fs = FileSystem.get(conf); + FSDataOutputStream out = fs.create(inputPath); + out.writeChars("Mary had a little lamb\n"); + out.writeChars("its fleece was white as snow\n"); + out.writeChars("and everywhere that Mary went\n"); + out.writeChars("the lamb was sure to go\n"); + out.close(); + + Map<String, String> env = new HashMap<>(); + env.put("HADOOP_CLASSPATH", System.getProperty(HCATCORE, "")); + Map<String, String> results = HiveHelper.execCommand(new CommandLine("hive") + .addArgument("--service") + .addArgument("jar") + .addArgument(System.getProperty(JOBJAR)) + .addArgument(HCatalogMR.class.getName()) + .addArgument("-it") + .addArgument(inputTable) + .addArgument("-ot") + .addArgument(outputTable) + .addArgument("-is") + .addArgument(inputSchema.getSchemaAsTypeString()) + .addArgument("-os") + .addArgument(outputSchema.getSchemaAsTypeString()), env); + LOG.info(results.toString()); + Assert.assertEquals("HCat job failed", 0, Integer.parseInt(results.get("exitValue"))); + + client.dropTable("default", inputTable); + client.dropTable("default", outputTable); + } + +} |