public void testInspectorFromTypeInfo()
{
TypeInfo typeInfo =
TypeInfoUtils.getTypeInfoFromTypeString("struct<c1:boolean,c2:tinyint" +
",c3:smallint,c4:int,c5:bigint,c6:float,c7:double,c8:binary," +
"c9:string,c10:struct<c1:int>,c11:map<int,int>,c12:uniontype<int>" +
",c13:array<timestamp>>");
StructObjectInspector inspector = (StructObjectInspector)
OrcStruct.createObjectInspector(typeInfo);
Assert.Equal("struct<c1:boolean,c2:tinyint,c3:smallint,c4:int,c5:" +
"bigint,c6:float,c7:double,c8:binary,c9:string,c10:struct<" +
"c1:int>,c11:map<int,int>,c12:uniontype<int>,c13:array<timestamp>>",
inspector.getTypeName());
Assert.Equal(null,
inspector.getAllStructFieldRefs()[0].getFieldComment());
Assert.Equal(null, inspector.getStructFieldRef("UNKNOWN"));
OrcStruct s1 = new OrcStruct(13);
for (int i = 0; i < 13; ++i)
{
s1.setFieldValue(i, i);
}
List <object> list = new List <object> {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12
};
Assert.Equal(list, inspector.getStructFieldsDataAsList(s1));
ListObjectInspector listOI = (ListObjectInspector)
inspector.getAllStructFieldRefs()[12].getFieldObjectInspector();
Assert.Equal(ObjectInspectorCategory.LIST, listOI.getCategory());
Assert.Equal(10, listOI.getListElement(list, 10));
Assert.Equal(null, listOI.getListElement(list, -1));
Assert.Equal(null, listOI.getListElement(list, 13));
Assert.Equal(13, listOI.getListLength(list));
Dictionary <object, object> map = new Dictionary <object, object>()
{
{ 1, 2 },
{ 2, 4 },
{ 3, 6 },
};
MapObjectInspector mapOI = (MapObjectInspector)
inspector.getAllStructFieldRefs()[10].getFieldObjectInspector();
Assert.Equal(3, mapOI.getMapSize(map));
Assert.Equal(4, mapOI.getMapValueElement(map, 2));
}
public void testReadTimestampFormat_0_11(string readerTimeZone)
{
string oldFilePath = Path.Combine(TestHelpers.ResourcesDirectory, "orc-file-11-format.orc");
using (TestHelpers.SetTimeZoneInfo(readerTimeZone))
{
Reader reader = OrcFile.createReader(oldFilePath, OrcFile.readerOptions(conf));
StructObjectInspector readerInspector = (StructObjectInspector)reader.getObjectInspector();
IList <StructField> fields = readerInspector.getAllStructFieldRefs();
TimestampObjectInspector tso = (TimestampObjectInspector)readerInspector
.getStructFieldRef("ts").getFieldObjectInspector();
using (RecordReader rows = reader.rows())
{
object row = rows.next();
Assert.NotNull(row);
Assert.Equal(Timestamp.Parse("2000-03-12 15:00:00"),
tso.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,
fields[12])));
// check the contents of second row
Assert.Equal(true, rows.hasNext());
rows.seekToRow(7499);
row = rows.next();
Assert.Equal(Timestamp.Parse("2000-03-12 15:00:01"),
tso.getPrimitiveJavaObject(readerInspector.getStructFieldData(row,
fields[12])));
Assert.Equal(false, rows.hasNext());
}
}
}
public RecIdStrippingObjectInspector(ObjectInspector oi, int rowIdColNum)
{
if (!(oi is StructObjectInspector))
{
throw new InvalidOperationException("Serious problem, expected a StructObjectInspector, " +
"but got a " + oi.GetType().Name);
}
wrapped = (StructObjectInspector)oi;
IList <StructField> wrappedFields = wrapped.getAllStructFieldRefs();
fields = new List <StructField>(wrapped.getAllStructFieldRefs().Count);
for (int i = 0; i < wrappedFields.Count; i++)
{
if (i == rowIdColNum)
{
recId = wrappedFields[i];
}
else
{
fields.Add(wrappedFields[i]);
}
}
}
public void testStringAndBinaryStatistics()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(SimpleStruct));
using (Stream file = File.OpenWrite(TestFilePath))
using (Writer writer = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
.inspector(inspector)
.stripeSize(100000)
.bufferSize(10000)))
{
writer.addRow(new SimpleStruct(bytes(0, 1, 2, 3, 4), "foo"));
writer.addRow(new SimpleStruct(bytes(0, 1, 2, 3), "bar"));
writer.addRow(new SimpleStruct(bytes(0, 1, 2, 3, 4, 5), null));
writer.addRow(new SimpleStruct(null, "hi"));
writer.close();
Assert.Equal(4, writer.getNumberOfRows());
Assert.Equal(273, writer.getRawDataSize());
}
Reader reader = OrcFile.createReader(TestFilePath,
OrcFile.readerOptions(conf));
Assert.Equal(4, reader.getNumberOfRows());
Assert.Equal(273, reader.getRawDataSize());
Assert.Equal(15, reader.getRawDataSizeOfColumns(Lists.newArrayList("bytes1")));
Assert.Equal(258, reader.getRawDataSizeOfColumns(Lists.newArrayList("string1")));
Assert.Equal(273, reader.getRawDataSizeOfColumns(Lists.newArrayList("bytes1", "string1")));
// check the stats
ColumnStatistics[] stats = reader.getStatistics();
Assert.Equal(4, stats[0].getNumberOfValues());
Assert.Equal("count: 4 hasNull: False", stats[0].ToString());
Assert.Equal(3, stats[1].getNumberOfValues());
Assert.Equal(15, ((BinaryColumnStatistics)stats[1]).getSum());
Assert.Equal("count: 3 hasNull: True sum: 15", stats[1].ToString());
Assert.Equal(3, stats[2].getNumberOfValues());
Assert.Equal("bar", ((StringColumnStatistics)stats[2]).getMinimum());
Assert.Equal("hi", ((StringColumnStatistics)stats[2]).getMaximum());
Assert.Equal(8, ((StringColumnStatistics)stats[2]).getSum());
Assert.Equal("count: 3 hasNull: True min: bar max: hi sum: 8",
stats[2].ToString());
// check the inspectors
StructObjectInspector readerInspector =
(StructObjectInspector)reader.getObjectInspector();
Assert.Equal(ObjectInspectorCategory.STRUCT, readerInspector.getCategory());
Assert.Equal("struct<bytes1:binary,string1:string>", readerInspector.getTypeName());
IList <StructField> fields = readerInspector.getAllStructFieldRefs();
BinaryObjectInspector bi = (BinaryObjectInspector)readerInspector.
getStructFieldRef("bytes1").getFieldObjectInspector();
StringObjectInspector st = (StringObjectInspector)readerInspector.
getStructFieldRef("string1").getFieldObjectInspector();
using (RecordReader rows = reader.rows())
{
object row = rows.next();
Assert.NotNull(row);
// check the contents of the first row
Assert.Equal(bytes(0, 1, 2, 3, 4), bi.get(
readerInspector.getStructFieldData(row, fields[0])));
Assert.Equal("foo", st.getPrimitiveJavaObject(readerInspector.
getStructFieldData(row, fields[1])));
// check the contents of second row
Assert.Equal(true, rows.hasNext());
row = rows.next();
Assert.Equal(bytes(0, 1, 2, 3), bi.get(
readerInspector.getStructFieldData(row, fields[0])));
Assert.Equal("bar", st.getPrimitiveJavaObject(readerInspector.
getStructFieldData(row, fields[1])));
// check the contents of second row
Assert.Equal(true, rows.hasNext());
row = rows.next();
Assert.Equal(bytes(0, 1, 2, 3, 4, 5), bi.get(
readerInspector.getStructFieldData(row, fields[0])));
Assert.Null(st.getPrimitiveJavaObject(readerInspector.
getStructFieldData(row, fields[1])));
// check the contents of second row
Assert.Equal(true, rows.hasNext());
row = rows.next();
Assert.Null(bi.get(
readerInspector.getStructFieldData(row, fields[0])));
Assert.Equal("hi", st.getPrimitiveJavaObject(readerInspector.
getStructFieldData(row, fields[1])));
Assert.Equal(false, rows.hasNext());
}
}
public RecIdStrippingObjectInspector(ObjectInspector oi, int rowIdColNum)
{
if (!(oi is StructObjectInspector))
{
throw new InvalidOperationException("Serious problem, expected a StructObjectInspector, " +
"but got a " + oi.GetType().Name);
}
wrapped = (StructObjectInspector)oi;
IList<StructField> wrappedFields = wrapped.getAllStructFieldRefs();
fields = new List<StructField>(wrapped.getAllStructFieldRefs().Count);
for (int i = 0; i < wrappedFields.Count; i++)
{
if (i == rowIdColNum)
{
recId = wrappedFields[i];
}
else
{
fields.Add(wrappedFields[i]);
}
}
}