DRILL-1905: There are 3 parts of the fix: (1) Check compatibility during initial creation, not during copy() of Union rels, (2) Added implementations of isDistinct() and getFamily() in Drill specific derived classes, (3) In Calcite, add processing of ANY type in leastRestrictiveSqlType().

Bump up the Calcite version to 0.9-drill-r16

11 files changed, 51 insertions, 18 deletions

diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillUnionRelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillUnionRelBase.java
index 127798f20..932aa7648 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillUnionRelBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillUnionRelBase.java
@@ -32,10 +32,10 @@ import org.eigenbase.reltype.RelDataType;
 public abstract class DrillUnionRelBase extends UnionRelBase implements DrillRelNode {
 
   public DrillUnionRelBase(RelOptCluster cluster, RelTraitSet traits,
-      List<RelNode> inputs, boolean all) throws InvalidRelException {
+      List<RelNode> inputs, boolean all, boolean checkCompatibility) throws InvalidRelException {
     super(cluster, traits, inputs, all);
-    // if (! this.isHomogeneous(false /* don't compare names */)) {
-    if (! this.isCompatible(false /* don't compare names */, true /* allow substrings */)) {
+    if (checkCompatibility &&
+        !this.isCompatible(false /* don't compare names */, true /* allow substrings */)) {
       throw new InvalidRelException("Input row types of the Union are not compatible.");
     }
   }
@@ -50,4 +50,10 @@ public abstract class DrillUnionRelBase extends UnionRelBase implements DrillRel
     }
     return true;
   }
+
+  @Override
+  public boolean isDistinct() {
+    return !this.all;
+  }
+
 }
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRel.java
index 1fa70010e..39d5d4d0a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRel.java
@@ -38,15 +38,16 @@ import org.eigenbase.relopt.RelTraitSet;
 public class DrillUnionRel extends DrillUnionRelBase implements DrillRel {
   /** Creates a DrillUnionRel. */
   public DrillUnionRel(RelOptCluster cluster, RelTraitSet traits,
-      List<RelNode> inputs, boolean all) throws InvalidRelException {
-    super(cluster, traits, inputs, all);
+      List<RelNode> inputs, boolean all, boolean checkCompatibility) throws InvalidRelException {
+    super(cluster, traits, inputs, all, checkCompatibility);
   }
 
   @Override
   public DrillUnionRel copy(RelTraitSet traitSet, List<RelNode> inputs,
       boolean all) {
     try {
-      return new DrillUnionRel(getCluster(), traitSet, inputs, all);
+      return new DrillUnionRel(getCluster(), traitSet, inputs, all,
+          false /* don't check compatibility during copy */);
     } catch (InvalidRelException e) {
       throw new AssertionError(e) ;
     }
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java
index 93a039135..f71d2032c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillUnionRule.java
@@ -52,7 +52,8 @@ public class DrillUnionRule extends RelOptRule {
       convertedInputs.add(convertedInput);
     }
     try {
-      call.transformTo(new DrillUnionRel(union.getCluster(), traits, convertedInputs, union.all));
+      call.transformTo(new DrillUnionRel(union.getCluster(), traits, convertedInputs, union.all,
+          true /* check compatibility */));
     } catch (InvalidRelException e) {
       tracer.warning(e.toString()) ;
     }
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrel.java
index 33abb48a4..d5698f760 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrel.java
@@ -38,15 +38,18 @@ import com.google.common.collect.Lists;
 
 public class UnionAllPrel extends UnionPrel {
 
-  public UnionAllPrel(RelOptCluster cluster, RelTraitSet traits, List<RelNode> inputs) throws InvalidRelException {
-    super(cluster, traits, inputs, true /* all */);
+  public UnionAllPrel(RelOptCluster cluster, RelTraitSet traits, List<RelNode> inputs,
+      boolean checkCompatibility) throws InvalidRelException {
+    super(cluster, traits, inputs, true /* all */, checkCompatibility);
+
   }
 
 
   @Override
   public UnionRelBase copy(RelTraitSet traitSet, List<RelNode> inputs, boolean all) {
     try {
-      return new UnionAllPrel(this.getCluster(), traitSet, inputs);
+      return new UnionAllPrel(this.getCluster(), traitSet, inputs,
+          false /* don't check compatibility during copy */);
     }catch (InvalidRelException e) {
       throw new AssertionError(e);
     }
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java
index 869493f26..fa533f75f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionAllPrule.java
@@ -60,7 +60,9 @@ public class UnionAllPrule extends Prule {
       }
 
       traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON);
-      UnionAllPrel unionAll = new UnionAllPrel(union.getCluster(), traits, convertedInputList);
+      UnionAllPrel unionAll =
+          new UnionAllPrel(union.getCluster(), traits, convertedInputList,
+              false /* compatibility already checked during logical phase */);
 
       call.transformTo(unionAll);
 
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrel.java
index 3fa5659a4..05086edea 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrel.java
@@ -38,15 +38,17 @@ import com.google.common.collect.Lists;
 
 public class UnionDistinctPrel extends UnionPrel {
 
-  public UnionDistinctPrel(RelOptCluster cluster, RelTraitSet traits, List<RelNode> inputs) throws InvalidRelException {
-    super(cluster, traits, inputs, false /* all = false */);
+  public UnionDistinctPrel(RelOptCluster cluster, RelTraitSet traits, List<RelNode> inputs,
+      boolean checkCompatibility) throws InvalidRelException {
+    super(cluster, traits, inputs, false /* all = false */, checkCompatibility);
   }
 
 
   @Override
   public UnionRelBase copy(RelTraitSet traitSet, List<RelNode> inputs, boolean all) {
     try {
-      return new UnionDistinctPrel(this.getCluster(), traitSet, inputs);
+      return new UnionDistinctPrel(this.getCluster(), traitSet, inputs,
+          false /* don't check compatibility during copy */);
     }catch (InvalidRelException e) {
       throw new AssertionError(e);
     }
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrule.java
index 67a817e8b..fd85448a4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionDistinctPrule.java
@@ -60,7 +60,9 @@ public class UnionDistinctPrule extends Prule {
       }
 
       traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON);
-      UnionDistinctPrel unionDistinct = new UnionDistinctPrel(union.getCluster(), traits, convertedInputList);
+      UnionDistinctPrel unionDistinct =
+          new UnionDistinctPrel(union.getCluster(), traits, convertedInputList,
+              false /* compatibility already checked during logical phase */);
 
       call.transformTo(unionDistinct);
 
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionPrel.java
index a1f300b25..630348987 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionPrel.java
@@ -30,8 +30,9 @@ import org.eigenbase.relopt.RelTraitSet;
 
 public abstract class UnionPrel extends DrillUnionRelBase implements Prel{
 
-  public UnionPrel(RelOptCluster cluster, RelTraitSet traits, List<RelNode> inputs, boolean all) throws InvalidRelException{
-    super(cluster, traits, inputs, all);
+  public UnionPrel(RelOptCluster cluster, RelTraitSet traits, List<RelNode> inputs, boolean all,
+      boolean checkCompatibility) throws InvalidRelException {
+    super(cluster, traits, inputs, all, checkCompatibility);
   }
 
   @Override
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/types/RelDataTypeDrillImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/types/RelDataTypeDrillImpl.java
index eae351035..559cf68ee 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/types/RelDataTypeDrillImpl.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/types/RelDataTypeDrillImpl.java
@@ -21,6 +21,7 @@ import java.util.Collections;
 import java.util.List;
 
 import org.eigenbase.reltype.RelDataTypeFactory;
+import org.eigenbase.reltype.RelDataTypeFamily;
 import org.eigenbase.reltype.RelDataTypeField;
 import org.eigenbase.reltype.RelDataTypeImpl;
 import org.eigenbase.reltype.RelDataTypePrecedenceList;
@@ -112,4 +113,8 @@ public class RelDataTypeDrillImpl extends RelDataTypeImpl {
       return true;
     }
 
+    @Override
+    public RelDataTypeFamily getFamily() {
+      return getSqlTypeName().getFamily();
+    }
 }
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java b/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
index bf0eea0c9..36b062b85 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
@@ -17,10 +17,13 @@
  */
 package org.apache.drill;
 
+import org.apache.drill.common.util.TestTools;
 import org.junit.Test;
 
 public class TestUnionAll extends BaseTestQuery{
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestUnionAll.class);
+  static final String WORKING_PATH = TestTools.getWorkingPath();
+  static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
 
   @Test    // Simple Union-All over two scans
   public void testUnionAll1() throws Exception {
@@ -67,5 +70,12 @@ public class TestUnionAll extends BaseTestQuery{
     test("select n_name from cp.`tpch/nation.parquet` union all select r_comment from cp.`tpch/region.parquet`");
   }
 
+  @Test // DRILL-1905: Union-all of * column from JSON files in different directories
+  public void testUnionAll9() throws Exception {
+    String query1 = String.format("select * from dfs_test.`%s/multilevel/json/1994/Q1/orders_94_q1.json` " +
+             " union all select * from dfs_test.`%s/multilevel/json/1995/Q1/orders_95_q1.json`",
+             TEST_RES_PATH, TEST_RES_PATH);
+    test(query1);
+  }
 
 }
diff --git a/pom.xml b/pom.xml
index f8cb47b32..9a539e33d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -920,7 +920,7 @@
           <dependency>
             <groupId>net.hydromatic</groupId>
             <artifactId>optiq-core</artifactId>
-            <version>0.9-drill-r15</version>
+            <version>0.9-drill-r16</version>
             <exclusions>
               <exclusion>
                 <groupId>org.jgrapht</groupId>