public void testHalfDistinctCheckDisabled()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(string));
int[] input = new int[20000];
// conf.setBoolean(ConfVars.HIVE_ORC_ROW_INDEX_STRIDE_DICTIONARY_CHECK.varname, false);
using (Stream file = File.OpenWrite(TestFilePath))
using (Writer writer = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
.inspector(inspector)
.compress(CompressionKind.NONE)
.bufferSize(10000)))
{
Random rand = new Random(123);
for (int i = 0; i < 20000; i++)
{
input[i] = rand.Next(10000);
}
for (int i = 0; i < 20000; i++)
{
writer.addRow(input[i].ToString());
}
}
Reader reader = OrcFile.createReader(TestFilePath, OrcFile.readerOptions(conf));
using (RecordReader rows = reader.rows())
{
int idx = 0;
while (rows.hasNext())
{
object row = rows.next();
Assert.Equal(input[idx++].ToString(), row);
}
// make sure the encoding type is correct
foreach (StripeInformation stripe in reader.getStripes())
{
// hacky but does the job, this casting will work as long this test resides
// within the same package as ORC reader
OrcProto.StripeFooter footer = ((RecordReaderImpl)rows).readStripeFooter(stripe);
for (int i = 0; i < footer.ColumnsCount; ++i)
{
OrcProto.ColumnEncoding encoding = footer.GetColumns(i);
Assert.Equal(OrcProto.ColumnEncoding.Types.Kind.DICTIONARY_V2, encoding.Kind);
}
}
}
}
public void testTooManyDistinctV11AlwaysDictionary()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(string));
using (Stream file = File.OpenWrite(TestFilePath))
using (Writer writer = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
.inspector(inspector)
.compress(CompressionKind.NONE)
.version(OrcFile.Version.V_0_11)
.bufferSize(10000)))
{
for (int i = 0; i < 20000; i++)
{
writer.addRow(i.ToString());
}
}
Reader reader = OrcFile.createReader(TestFilePath, OrcFile.readerOptions(conf));
using (RecordReader rows = reader.rows())
{
int idx = 0;
while (rows.hasNext())
{
object row = rows.next();
Assert.Equal((idx++).ToString(), row);
}
// make sure the encoding type is correct
foreach (StripeInformation stripe in reader.getStripes())
{
// hacky but does the job, this casting will work as long this test resides
// within the same package as ORC reader
OrcProto.StripeFooter footer = ((RecordReaderImpl)rows).readStripeFooter(stripe);
for (int i = 0; i < footer.ColumnsCount; ++i)
{
OrcProto.ColumnEncoding encoding = footer.GetColumns(i);
Assert.Equal(OrcProto.ColumnEncoding.Types.Kind.DICTIONARY, encoding.Kind);
}
}
}
}