/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at *
* http://www.apache.org/licenses/LICENSE-2.0 *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.odpi.specs.runtime.hive;
import org.apache.commons.exec.CommandLine;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hive.hcatalog.data.schema.HCatSchema;
import org.apache.thrift.TException;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
public class TestHCatalog {
private static final String JOBJAR = "odpi.test.hive.hcat.job.jar";
private static final String HCATCORE = "odpi.test.hive.hcat.core.jar";
private static final Log LOG = LogFactory.getLog(TestHCatalog.class.getName());
private static IMetaStoreClient client = null;
private static HiveConf conf;
private static HCatSchema inputSchema;
private static HCatSchema outputSchema;
private Random rand;
@BeforeClass
public static void connect() throws MetaException {
if (JdbcConnector.testActive(JdbcConnector.TEST_HCATALOG, "Test HCatalog ")) {
String hiveConfDir = JdbcConnector.getProperty(JdbcConnector.HIVE_CONF_DIR,
"Hive conf directory ");
String hadoopConfDir = JdbcConnector.getProperty(JdbcConnector.HADOOP_CONF_DIR,
"Hadoop conf directory ");
conf = new HiveConf();
String fileSep = System.getProperty("file.separator");
conf.addResource(new Path(hadoopConfDir + fileSep + "core-site.xml"));
conf.addResource(new Path(hadoopConfDir + fileSep + "hdfs-site.xml"));
conf.addResource(new Path(hadoopConfDir + fileSep + "yarn-site.xml"));
conf.addResource(new Path(hadoopConfDir + fileSep + "mapred-site.xml"));
conf.addResource(new Path(hiveConfDir + fileSep + "hive-site.xml"));
client = new HiveMetaStoreClient(conf);
}
}
@Before
public void checkIfActive() {
Assume.assumeTrue(JdbcConnector.testActive(JdbcConnector.TEST_HCATALOG, "Test HCatalog "));
rand = new Random();
}
@Test
public void hcatInputFormatOutputFormat() throws TException, IOException, ClassNotFoundException,
InterruptedException, URISyntaxException {
// Create a table to write to
final String inputTable = "odpi_hcat_input_table_" + rand.nextInt(Integer.MAX_VALUE);
SerDeInfo serde = new SerDeInfo("default_serde",
conf.getVar(HiveConf.ConfVars.HIVEDEFAULTSERDE), new HashMap