VectorizedOrcRecordReader(Reader file, Configuration conf,
FileSplit fileSplit)
{
List <OrcProto.Type> types = file.getTypes();
Reader.Options options = new Reader.Options();
this.offset = fileSplit.getStart();
this.length = fileSplit.getLength();
options.range(offset, length);
options.include(OrcInputFormat.genIncludedColumns(types, conf, true));
OrcInputFormat.setSearchArgument(options, types, conf, true);
this.reader = file.rowsOptions(options);
try
{
rbCtx = new VectorizedRowBatchCtx();
rbCtx.init(conf, fileSplit);
}
catch (Exception e)
{
throw;
}
}
public void testSplitEliminationComplexExpr()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(AllTypesRow));
using (Stream file = File.OpenWrite(testFilePath))
using (Writer writer = OrcFile.createWriter(testFilePath, file, conf, inspector,
100000, CompressionKind.NONE, 10000, 10000))
{
writeData(writer);
}
conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), "1000");
conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), "150000");
InputFormat @in = new OrcInputFormat();
FileInputFormat.setInputPaths(conf, testFilePath.ToString());
// predicate expression: userid <= 100 and subtype <= 1000.0
GenericUDF udf = new GenericUDFOPEqualOrLessThan();
List<ExprNodeDesc> childExpr = new List<ExprNodeDesc>();
ExprNodeColumnDesc col = new ExprNodeColumnDesc(typeof(long), "userid", "T", false);
ExprNodeConstantDesc con = new ExprNodeConstantDesc(100);
childExpr.Add(col);
childExpr.Add(con);
ExprNodeGenericFuncDesc en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
GenericUDF udf1 = new GenericUDFOPEqualOrLessThan();
List<ExprNodeDesc> childExpr1 = new List<ExprNodeDesc>();
ExprNodeColumnDesc col1 = new ExprNodeColumnDesc(typeof(double), "subtype", "T", false);
ExprNodeConstantDesc con1 = new ExprNodeConstantDesc(1000.0);
childExpr1.Add(col1);
childExpr1.Add(con1);
ExprNodeGenericFuncDesc en1 = new ExprNodeGenericFuncDesc(inspector, udf1, childExpr1);
GenericUDF udf2 = new GenericUDFOPAnd();
List<ExprNodeDesc> childExpr2 = new List<ExprNodeDesc>();
childExpr2.Add(en);
childExpr2.Add(en1);
ExprNodeGenericFuncDesc en2 = new ExprNodeGenericFuncDesc(inspector, udf2, childExpr2);
string sargStr = Utilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized", sargStr);
InputSplit[] splits = @in.getSplits(conf, 1);
Assert.Equal(2, splits.Length);
con = new ExprNodeConstantDesc(2);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
con1 = new ExprNodeConstantDesc(0.0);
childExpr1[1] = con1;
en1 = new ExprNodeGenericFuncDesc(inspector, udf1, childExpr1);
childExpr2[0] = en;
childExpr2[1] = en1;
en2 = new ExprNodeGenericFuncDesc(inspector, udf2, childExpr2);
sargStr = Utilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
// no stripe will satisfy the predicate
Assert.Equal(0, splits.Length);
con = new ExprNodeConstantDesc(2);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
con1 = new ExprNodeConstantDesc(1.0);
childExpr1[1] = con1;
en1 = new ExprNodeGenericFuncDesc(inspector, udf1, childExpr1);
childExpr2[0] = en;
childExpr2[1] = en1;
en2 = new ExprNodeGenericFuncDesc(inspector, udf2, childExpr2);
sargStr = Utilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
// only first stripe will satisfy condition and hence single split
Assert.Equal(1, splits.Length);
udf = new GenericUDFOPEqual();
con = new ExprNodeConstantDesc(13);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
con1 = new ExprNodeConstantDesc(80.0);
childExpr1[1] = con1;
en1 = new ExprNodeGenericFuncDesc(inspector, udf1, childExpr1);
childExpr2[0] = en;
childExpr2[1] = en1;
en2 = new ExprNodeGenericFuncDesc(inspector, udf2, childExpr2);
sargStr = Utilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
// first two stripes will satisfy condition and hence single split
Assert.Equal(2, splits.Length);
udf = new GenericUDFOPEqual();
con = new ExprNodeConstantDesc(13);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
udf1 = new GenericUDFOPEqual();
con1 = new ExprNodeConstantDesc(80.0);
childExpr1[1] = con1;
en1 = new ExprNodeGenericFuncDesc(inspector, udf1, childExpr1);
childExpr2[0] = en;
childExpr2[1] = en1;
en2 = new ExprNodeGenericFuncDesc(inspector, udf2, childExpr2);
sargStr = Utilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
// only second stripes will satisfy condition and hence single split
Assert.Equal(1, splits.Length);
}
public void testSplitEliminationSmallMaxSplit()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(AllTypesRow));
using (Stream file = File.OpenWrite(testFilePath))
using (Writer writer = OrcFile.createWriter(testFilePath, file, conf, inspector,
100000, CompressionKind.NONE, 10000, 10000))
{
writeData(writer);
}
conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), "1000");
conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), "5000");
InputFormat @in = new OrcInputFormat();
FileInputFormat.setInputPaths(conf, testFilePath.ToString());
GenericUDF udf = new GenericUDFOPEqualOrLessThan();
List <ExprNodeDesc> childExpr = new List <ExprNodeDesc>();
ExprNodeColumnDesc col = new ExprNodeColumnDesc(typeof(long), "userid", "T", false);
ExprNodeConstantDesc con = new ExprNodeConstantDesc(100);
childExpr.Add(col);
childExpr.Add(con);
ExprNodeGenericFuncDesc en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
string sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
InputSplit[] splits = @in.getSplits(conf, 1);
Assert.Equal(5, splits.Length);
con = new ExprNodeConstantDesc(1);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(0, splits.Length);
con = new ExprNodeConstantDesc(2);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(1, splits.Length);
con = new ExprNodeConstantDesc(5);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(2, splits.Length);
con = new ExprNodeConstantDesc(13);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(3, splits.Length);
con = new ExprNodeConstantDesc(29);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(4, splits.Length);
con = new ExprNodeConstantDesc(70);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(5, splits.Length);
}
public void testSplitEliminationSmallMaxSplit()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(AllTypesRow));
using (Stream file = File.OpenWrite(testFilePath))
using (Writer writer = OrcFile.createWriter(testFilePath, file, conf, inspector,
100000, CompressionKind.NONE, 10000, 10000))
{
writeData(writer);
}
conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), "1000");
conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), "5000");
InputFormat @in = new OrcInputFormat();
FileInputFormat.setInputPaths(conf, testFilePath.ToString());
GenericUDF udf = new GenericUDFOPEqualOrLessThan();
List<ExprNodeDesc> childExpr = new List<ExprNodeDesc>();
ExprNodeColumnDesc col = new ExprNodeColumnDesc(typeof(long), "userid", "T", false);
ExprNodeConstantDesc con = new ExprNodeConstantDesc(100);
childExpr.Add(col);
childExpr.Add(con);
ExprNodeGenericFuncDesc en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
string sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
InputSplit[] splits = @in.getSplits(conf, 1);
Assert.Equal(5, splits.Length);
con = new ExprNodeConstantDesc(1);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(0, splits.Length);
con = new ExprNodeConstantDesc(2);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(1, splits.Length);
con = new ExprNodeConstantDesc(5);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(2, splits.Length);
con = new ExprNodeConstantDesc(13);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(3, splits.Length);
con = new ExprNodeConstantDesc(29);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(4, splits.Length);
con = new ExprNodeConstantDesc(70);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
sargStr = Utilities.serializeExpression(en);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
Assert.Equal(5, splits.Length);
}
public void testSplitEliminationComplexExpr()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(AllTypesRow));
using (Stream file = File.OpenWrite(testFilePath))
using (Writer writer = OrcFile.createWriter(testFilePath, file, conf, inspector,
100000, CompressionKind.NONE, 10000, 10000))
{
writeData(writer);
}
conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), "1000");
conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), "150000");
InputFormat @in = new OrcInputFormat();
FileInputFormat.setInputPaths(conf, testFilePath.ToString());
// predicate expression: userid <= 100 and subtype <= 1000.0
GenericUDF udf = new GenericUDFOPEqualOrLessThan();
List <ExprNodeDesc> childExpr = new List <ExprNodeDesc>();
ExprNodeColumnDesc col = new ExprNodeColumnDesc(typeof(long), "userid", "T", false);
ExprNodeConstantDesc con = new ExprNodeConstantDesc(100);
childExpr.Add(col);
childExpr.Add(con);
ExprNodeGenericFuncDesc en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
GenericUDF udf1 = new GenericUDFOPEqualOrLessThan();
List <ExprNodeDesc> childExpr1 = new List <ExprNodeDesc>();
ExprNodeColumnDesc col1 = new ExprNodeColumnDesc(typeof(double), "subtype", "T", false);
ExprNodeConstantDesc con1 = new ExprNodeConstantDesc(1000.0);
childExpr1.Add(col1);
childExpr1.Add(con1);
ExprNodeGenericFuncDesc en1 = new ExprNodeGenericFuncDesc(inspector, udf1, childExpr1);
GenericUDF udf2 = new GenericUDFOPAnd();
List <ExprNodeDesc> childExpr2 = new List <ExprNodeDesc>();
childExpr2.Add(en);
childExpr2.Add(en1);
ExprNodeGenericFuncDesc en2 = new ExprNodeGenericFuncDesc(inspector, udf2, childExpr2);
string sargStr = Utilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized", sargStr);
InputSplit[] splits = @in.getSplits(conf, 1);
Assert.Equal(2, splits.Length);
con = new ExprNodeConstantDesc(2);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
con1 = new ExprNodeConstantDesc(0.0);
childExpr1[1] = con1;
en1 = new ExprNodeGenericFuncDesc(inspector, udf1, childExpr1);
childExpr2[0] = en;
childExpr2[1] = en1;
en2 = new ExprNodeGenericFuncDesc(inspector, udf2, childExpr2);
sargStr = Utilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
// no stripe will satisfy the predicate
Assert.Equal(0, splits.Length);
con = new ExprNodeConstantDesc(2);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
con1 = new ExprNodeConstantDesc(1.0);
childExpr1[1] = con1;
en1 = new ExprNodeGenericFuncDesc(inspector, udf1, childExpr1);
childExpr2[0] = en;
childExpr2[1] = en1;
en2 = new ExprNodeGenericFuncDesc(inspector, udf2, childExpr2);
sargStr = Utilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
// only first stripe will satisfy condition and hence single split
Assert.Equal(1, splits.Length);
udf = new GenericUDFOPEqual();
con = new ExprNodeConstantDesc(13);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
con1 = new ExprNodeConstantDesc(80.0);
childExpr1[1] = con1;
en1 = new ExprNodeGenericFuncDesc(inspector, udf1, childExpr1);
childExpr2[0] = en;
childExpr2[1] = en1;
en2 = new ExprNodeGenericFuncDesc(inspector, udf2, childExpr2);
sargStr = Utilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
// first two stripes will satisfy condition and hence single split
Assert.Equal(2, splits.Length);
udf = new GenericUDFOPEqual();
con = new ExprNodeConstantDesc(13);
childExpr[1] = con;
en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr);
udf1 = new GenericUDFOPEqual();
con1 = new ExprNodeConstantDesc(80.0);
childExpr1[1] = con1;
en1 = new ExprNodeGenericFuncDesc(inspector, udf1, childExpr1);
childExpr2[0] = en;
childExpr2[1] = en1;
en2 = new ExprNodeGenericFuncDesc(inspector, udf2, childExpr2);
sargStr = Utilities.serializeExpression(en2);
conf.set("hive.io.filter.expr.serialized", sargStr);
splits = @in.getSplits(conf, 1);
// only second stripes will satisfy condition and hence single split
Assert.Equal(1, splits.Length);
}
