private void ReadColumn(BinaryReader reader, Thrift.Encoding encoding, long maxValues,
out IList values,
out List <int> indexes)
{
//dictionary encoding uses RLE to encode data
switch (encoding)
{
case Thrift.Encoding.PLAIN:
values = _dataTypeHandler.Read(_thriftSchemaElement, reader, _parquetOptions);
indexes = null;
break;
case Thrift.Encoding.RLE:
values = null;
indexes = RunLengthBitPackingHybridValuesReader.Read(reader, _thriftSchemaElement.Type_length);
break;
case Thrift.Encoding.PLAIN_DICTIONARY:
values = null;
indexes = ReadPlainDictionary(reader, maxValues);
break;
default:
throw new ParquetException($"encoding {encoding} is not supported.");
}
}
private void ReadColumn(BinaryReader reader, Thrift.Encoding encoding, long totalValues, int maxReadCount, ColumnRawData cd)
{
//dictionary encoding uses RLE to encode data
if (cd.values == null)
{
cd.values = _dataTypeHandler.GetArray((int)totalValues, false, false);
}
switch (encoding)
{
case Thrift.Encoding.PLAIN:
cd.valuesOffset += _dataTypeHandler.Read(reader, _thriftSchemaElement, cd.values, cd.valuesOffset);
break;
case Thrift.Encoding.RLE:
if (cd.indexes == null)
{
cd.indexes = new int[(int)totalValues];
}
int indexCount = RunLengthBitPackingHybridValuesReader.Read(reader, _thriftSchemaElement.Type_length, cd.indexes, 0, maxReadCount);
_dataTypeHandler.MergeDictionary(cd.dictionary, cd.indexes, cd.values, cd.valuesOffset, indexCount);
cd.valuesOffset += indexCount;
break;
case Thrift.Encoding.PLAIN_DICTIONARY:
if (cd.indexes == null)
{
cd.indexes = new int[(int)totalValues];
}
indexCount = ReadPlainDictionary(reader, maxReadCount, cd.indexes, 0);
_dataTypeHandler.MergeDictionary(cd.dictionary, cd.indexes, cd.values, cd.valuesOffset, indexCount);
cd.valuesOffset += indexCount;
break;
default:
throw new ParquetException($"encoding {encoding} is not supported.");
}
}
private void ReadColumn(BinaryReader reader, Thrift.Encoding encoding, long maxValues,
ref Array values, ref int valuesOffset,
ref int[] indexes, ref int indexesOffset)
{
//dictionary encoding uses RLE to encode data
switch (encoding)
{
case Thrift.Encoding.PLAIN:
if (values == null)
{
values = _dataTypeHandler.GetArray((int)maxValues, false, false);
}
valuesOffset += _dataTypeHandler.Read(reader, _thriftSchemaElement, values, valuesOffset, _parquetOptions);
break;
case Thrift.Encoding.RLE:
if (indexes == null)
{
indexes = new int[(int)maxValues];
}
indexesOffset += RunLengthBitPackingHybridValuesReader.Read(reader, _thriftSchemaElement.Type_length, indexes, indexesOffset);
break;
case Thrift.Encoding.PLAIN_DICTIONARY:
if (indexes == null)
{
indexes = new int[(int)maxValues];
}
indexesOffset += ReadPlainDictionary(reader, maxValues, indexes, indexesOffset);
break;
default:
throw new ParquetException($"encoding {encoding} is not supported.");
}
}
private List <int> ReadColumnValues(BinaryReader reader, Thrift.Encoding encoding, IList destination, long maxValues)
{
//dictionary encoding uses RLE to encode data
switch (encoding)
{
case Thrift.Encoding.PLAIN:
_plainReader.Read(reader, _schema, destination, maxValues);
return(null);
case Thrift.Encoding.RLE:
var rleIndexes = new List <int>();
_rleReader.Read(reader, _schema, rleIndexes, maxValues);
return(rleIndexes);
case Thrift.Encoding.PLAIN_DICTIONARY:
var dicIndexes = new List <int>();
_dictionaryReader.Read(reader, _schema, dicIndexes, maxValues);
return(dicIndexes);
default:
throw new ParquetException($"encoding {encoding} is not supported."); //todo: replace with own exception type
}
}
