public static void AddColumn(this StripeFooter footer, ColumnEncodingKind columnEncodingKind,
uint dictionarySize = 0)
{
var columnEncoding = new ColumnEncoding
{
Kind = columnEncodingKind,
DictionarySize = dictionarySize
};
footer.Columns.Add(columnEncoding);
}
internal StripeStreamReader(Stream inputStream, uint columnId, StreamKind streamKind,
ColumnEncodingKind encodingKind, long inputStreamOffset, ulong compressedLength,
CompressionKind compressionKind)
{
_inputStream = inputStream;
ColumnId = columnId;
StreamKind = streamKind;
ColumnEncodingKind = encodingKind;
_inputStreamOffset = inputStreamOffset;
_compressedLength = compressedLength;
_compressionKind = compressionKind;
}
private void EnsureEncodingKindIsSet(IList <string> values)
{
if (ColumnEncodingIsValid())
{
return;
}
//Detect the encoding type
var nonNullValues = values.Where(v => v != null);
var uniqueValues = nonNullValues.Distinct().Count();
var totalValues = nonNullValues.Count();
ColumnEncoding = (double)uniqueValues / (double)totalValues <= _uniqueStringThresholdRatio
? ColumnEncodingKind.DictionaryV2
: ColumnEncodingKind.DirectV2;
}
