private static object boxLiteral(ExprNodeConstantDesc constantDesc,
PredicateLeaf.Type type)
{
object lit = constantDesc.getValue();
if (lit == null)
{
return(null);
}
switch (type)
{
case PredicateLeaf.Type.LONG:
return(((Number)lit).longValue());
case PredicateLeaf.Type.STRING:
if (lit is HiveChar)
{
return(((HiveChar)lit).getPaddedValue());
}
else if (lit is String)
{
return(lit);
}
else
{
return(lit.toString());
}
case PredicateLeaf.Type.FLOAT:
if (lit is Float)
{
// converting a float directly to a double causes annoying conversion
// problems
return(Double.parseDouble(lit.toString()));
}
else
{
return(((Number)lit).doubleValue());
}
case PredicateLeaf.Type.TIMESTAMP:
return(Timestamp.valueOf(lit.toString()));
case PredicateLeaf.Type.DATE:
return(Date.valueOf(lit.toString()));
case PredicateLeaf.Type.DECIMAL:
LOG.warn("boxing " + lit);
return(new HiveDecimalWritable(lit.toString()));
case PredicateLeaf.Type.BOOLEAN:
return(lit);
default:
throw new ArgumentException("Unknown literal " + type);
}
}
private static object boxLiteral(ExprNodeConstantDesc constantDesc,
PredicateLeaf.Type type)
{
object lit = constantDesc.getValue();
if (lit == null)
{
return null;
}
switch (type)
{
case PredicateLeaf.Type.LONG:
return ((Number)lit).longValue();
case PredicateLeaf.Type.STRING:
if (lit is HiveChar)
{
return ((HiveChar)lit).getPaddedValue();
}
else if (lit is String)
{
return lit;
}
else
{
return lit.toString();
}
case PredicateLeaf.Type.FLOAT:
if (lit is Float)
{
// converting a float directly to a double causes annoying conversion
// problems
return Double.parseDouble(lit.toString());
}
else
{
return ((Number)lit).doubleValue();
}
case PredicateLeaf.Type.TIMESTAMP:
return Timestamp.valueOf(lit.toString());
case PredicateLeaf.Type.DATE:
return Date.valueOf(lit.toString());
case PredicateLeaf.Type.DECIMAL:
LOG.warn("boxing " + lit);
return new HiveDecimalWritable(lit.toString());
case PredicateLeaf.Type.BOOLEAN:
return lit;
default:
throw new ArgumentException("Unknown literal " + type);
}
}
public void testSplitEliminationComplexExpr()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(AllTypesRow));
using (Stream file = File.OpenWrite(testFilePath))
using (Writer writer = OrcFile.createWriter(testFilePath, file, conf, inspector,
100000, CompressionKind.NONE, 10000, 10000))
{
writeData(writer);
}
conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), "1000");
conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), "150000");
InputFormat @in = new OrcInputFormat();
FileInputFormat.setInputPaths(conf, testFilePath.ToString());
// predicate expression: userid <= 100 and subtype <= 1000.0
GenericUDF udf = new GenericUDFOPEqualOrLessThan();
List<ExprNodeDesc> childExpr = new List<ExprNodeDesc>();
ExprNodeColumnDesc col = new ExprNodeColumnDesc(typeof(long), "userid", "T", false);
ExprNodeConstantDesc con = new ExprNodeConstantDesc(100);
childExpr.Add(col);
childExpr.Add(con);
ExprNodeGenericFuncDesc en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
GenericUDF udf1 = new GenericUDFOPEqualOrLessThan();
List<ExprNodeDesc> childExpr1 = new List<ExprNodeDesc>();
ExprNodeColumnDesc col1 = new ExprNodeColumnDesc(typeof(double), "subtype", "T", false);
ExprNodeConstantDesc con1 = new ExprNodeConstantDesc(1000.0);
childExpr1.Add(col1);
childExpr1.Add(con1);
ExprNodeGenericFuncDesc en1 = new ExprNodeGenericFuncDesc(inspector, udf1, childExpr1);
GenericUDF udf2 = new GenericUDFOPAnd();
List<ExprNodeDesc> childExpr2 = new List<ExprNodeDesc>();
childExpr2.Add(en);
childExpr2.Add(en1);
ExprNodeGenericFuncDesc en2 = new ExprNodeGenericFuncDesc(inspector, udf2, childExpr2);
string sargStr = Utilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized", sargStr);
InputSplit[] splits = @in.getSplits(conf, 1);
Assert.Equal(2, splits.Length);
con = new ExprNodeConstantDesc(2);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
con1 = new ExprNodeConstantDesc(0.0);
childExpr1[1] = con1;
en1 = new ExprNodeGenericFuncDesc(inspector, udf1, childExpr1);
childExpr2[0] = en;
childExpr2[1] = en1;
en2 = new ExprNodeGenericFuncDesc(inspector, udf2, childExpr2);
sargStr = Utilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
// no stripe will satisfy the predicate
Assert.Equal(0, splits.Length);
con = new ExprNodeConstantDesc(2);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
con1 = new ExprNodeConstantDesc(1.0);
childExpr1[1] = con1;
en1 = new ExprNodeGenericFuncDesc(inspector, udf1, childExpr1);
childExpr2[0] = en;
childExpr2[1] = en1;
en2 = new ExprNodeGenericFuncDesc(inspector, udf2, childExpr2);
sargStr = Utilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
// only first stripe will satisfy condition and hence single split
Assert.Equal(1, splits.Length);
udf = new GenericUDFOPEqual();
con = new ExprNodeConstantDesc(13);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
con1 = new ExprNodeConstantDesc(80.0);
childExpr1[1] = con1;
en1 = new ExprNodeGenericFuncDesc(inspector, udf1, childExpr1);
childExpr2[0] = en;
childExpr2[1] = en1;
en2 = new ExprNodeGenericFuncDesc(inspector, udf2, childExpr2);
sargStr = Utilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
// first two stripes will satisfy condition and hence single split
Assert.Equal(2, splits.Length);
udf = new GenericUDFOPEqual();
con = new ExprNodeConstantDesc(13);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
udf1 = new GenericUDFOPEqual();
con1 = new ExprNodeConstantDesc(80.0);
childExpr1[1] = con1;
en1 = new ExprNodeGenericFuncDesc(inspector, udf1, childExpr1);
childExpr2[0] = en;
childExpr2[1] = en1;
en2 = new ExprNodeGenericFuncDesc(inspector, udf2, childExpr2);
sargStr = Utilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
// only second stripes will satisfy condition and hence single split
Assert.Equal(1, splits.Length);
}
public void testSplitEliminationSmallMaxSplit()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(AllTypesRow));
using (Stream file = File.OpenWrite(testFilePath))
using (Writer writer = OrcFile.createWriter(testFilePath, file, conf, inspector,
100000, CompressionKind.NONE, 10000, 10000))
{
writeData(writer);
}
conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), "1000");
conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), "5000");
InputFormat @in = new OrcInputFormat();
FileInputFormat.setInputPaths(conf, testFilePath.ToString());
GenericUDF udf = new GenericUDFOPEqualOrLessThan();
List<ExprNodeDesc> childExpr = new List<ExprNodeDesc>();
ExprNodeColumnDesc col = new ExprNodeColumnDesc(typeof(long), "userid", "T", false);
ExprNodeConstantDesc con = new ExprNodeConstantDesc(100);
childExpr.Add(col);
childExpr.Add(con);
ExprNodeGenericFuncDesc en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
string sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
InputSplit[] splits = @in.getSplits(conf, 1);
Assert.Equal(5, splits.Length);
con = new ExprNodeConstantDesc(1);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(0, splits.Length);
con = new ExprNodeConstantDesc(2);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(1, splits.Length);
con = new ExprNodeConstantDesc(5);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(2, splits.Length);
con = new ExprNodeConstantDesc(13);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(3, splits.Length);
con = new ExprNodeConstantDesc(29);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(4, splits.Length);
con = new ExprNodeConstantDesc(70);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(5, splits.Length);
}
public void testSplitEliminationSmallMaxSplit()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(AllTypesRow));
using (Stream file = File.OpenWrite(testFilePath))
using (Writer writer = OrcFile.createWriter(testFilePath, file, conf, inspector,
100000, CompressionKind.NONE, 10000, 10000))
{
writeData(writer);
}
conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), "1000");
conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), "5000");
InputFormat @in = new OrcInputFormat();
FileInputFormat.setInputPaths(conf, testFilePath.ToString());
GenericUDF udf = new GenericUDFOPEqualOrLessThan();
List <ExprNodeDesc> childExpr = new List <ExprNodeDesc>();
ExprNodeColumnDesc col = new ExprNodeColumnDesc(typeof(long), "userid", "T", false);
ExprNodeConstantDesc con = new ExprNodeConstantDesc(100);
childExpr.Add(col);
childExpr.Add(con);
ExprNodeGenericFuncDesc en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
string sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
InputSplit[] splits = @in.getSplits(conf, 1);
Assert.Equal(5, splits.Length);
con = new ExprNodeConstantDesc(1);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(0, splits.Length);
con = new ExprNodeConstantDesc(2);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(1, splits.Length);
con = new ExprNodeConstantDesc(5);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(2, splits.Length);
con = new ExprNodeConstantDesc(13);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(3, splits.Length);
con = new ExprNodeConstantDesc(29);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(4, splits.Length);
con = new ExprNodeConstantDesc(70);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(5, splits.Length);
}
public void testSplitEliminationComplexExpr()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(AllTypesRow));
using (Stream file = File.OpenWrite(testFilePath))
using (Writer writer = OrcFile.createWriter(testFilePath, file, conf, inspector,
100000, CompressionKind.NONE, 10000, 10000))
{
writeData(writer);
}
conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), "1000");
conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), "150000");
InputFormat @in = new OrcInputFormat();
FileInputFormat.setInputPaths(conf, testFilePath.ToString());
// predicate expression: userid <= 100 and subtype <= 1000.0
GenericUDF udf = new GenericUDFOPEqualOrLessThan();
List <ExprNodeDesc> childExpr = new List <ExprNodeDesc>();
ExprNodeColumnDesc col = new ExprNodeColumnDesc(typeof(long), "userid", "T", false);
ExprNodeConstantDesc con = new ExprNodeConstantDesc(100);
childExpr.Add(col);
childExpr.Add(con);
ExprNodeGenericFuncDesc en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
GenericUDF udf1 = new GenericUDFOPEqualOrLessThan();
List <ExprNodeDesc> childExpr1 = new List <ExprNodeDesc>();
ExprNodeColumnDesc col1 = new ExprNodeColumnDesc(typeof(double), "subtype", "T", false);
ExprNodeConstantDesc con1 = new ExprNodeConstantDesc(1000.0);
childExpr1.Add(col1);
childExpr1.Add(con1);
ExprNodeGenericFuncDesc en1 = new ExprNodeGenericFuncDesc(inspector, udf1, childExpr1);
GenericUDF udf2 = new GenericUDFOPAnd();
List <ExprNodeDesc> childExpr2 = new List <ExprNodeDesc>();
childExpr2.Add(en);
childExpr2.Add(en1);
ExprNodeGenericFuncDesc en2 = new ExprNodeGenericFuncDesc(inspector, udf2, childExpr2);
string sargStr = Utilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized", sargStr);
InputSplit[] splits = @in.getSplits(conf, 1);
Assert.Equal(2, splits.Length);
con = new ExprNodeConstantDesc(2);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
con1 = new ExprNodeConstantDesc(0.0);
childExpr1[1] = con1;
en1 = new ExprNodeGenericFuncDesc(inspector, udf1, childExpr1);
childExpr2[0] = en;
childExpr2[1] = en1;
en2 = new ExprNodeGenericFuncDesc(inspector, udf2, childExpr2);
sargStr = Utilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
// no stripe will satisfy the predicate
Assert.Equal(0, splits.Length);
con = new ExprNodeConstantDesc(2);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
con1 = new ExprNodeConstantDesc(1.0);
childExpr1[1] = con1;
en1 = new ExprNodeGenericFuncDesc(inspector, udf1, childExpr1);
childExpr2[0] = en;
childExpr2[1] = en1;
en2 = new ExprNodeGenericFuncDesc(inspector, udf2, childExpr2);
sargStr = Utilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
// only first stripe will satisfy condition and hence single split
Assert.Equal(1, splits.Length);
udf = new GenericUDFOPEqual();
con = new ExprNodeConstantDesc(13);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
con1 = new ExprNodeConstantDesc(80.0);
childExpr1[1] = con1;
en1 = new ExprNodeGenericFuncDesc(inspector, udf1, childExpr1);
childExpr2[0] = en;
childExpr2[1] = en1;
en2 = new ExprNodeGenericFuncDesc(inspector, udf2, childExpr2);
sargStr = Utilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
// first two stripes will satisfy condition and hence single split
Assert.Equal(2, splits.Length);
udf = new GenericUDFOPEqual();
con = new ExprNodeConstantDesc(13);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
udf1 = new GenericUDFOPEqual();
con1 = new ExprNodeConstantDesc(80.0);
childExpr1[1] = con1;
en1 = new ExprNodeGenericFuncDesc(inspector, udf1, childExpr1);
childExpr2[0] = en;
childExpr2[1] = en1;
en2 = new ExprNodeGenericFuncDesc(inspector, udf2, childExpr2);
sargStr = Utilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
// only second stripes will satisfy condition and hence single split
Assert.Equal(1, splits.Length);
}
