public BinaryWriter(bool shouldAlignLengths, OrcCompressedBufferFactory bufferFactory, uint columnId)
{
_shouldAlignLengths = shouldAlignLengths;
ColumnId = columnId;
_presentBuffer = bufferFactory.CreateBuffer(StreamKind.Present);
_presentBuffer.MustBeIncluded = false;
_dataBuffer = bufferFactory.CreateBuffer(StreamKind.Data);
_lengthBuffer = bufferFactory.CreateBuffer(StreamKind.Length);
}
public DoubleWriter(bool isNullable, OrcCompressedBufferFactory bufferFactory, uint columnId)
{
_isNullable = isNullable;
ColumnId = columnId;
if (_isNullable)
{
_presentBuffer = bufferFactory.CreateBuffer(StreamKind.Present);
_presentBuffer.MustBeIncluded = false;
}
_dataBuffer = bufferFactory.CreateBuffer(StreamKind.Data);
}
public DateWriter(bool isNullable, bool shouldAlignEncodedValues, OrcCompressedBufferFactory bufferFactory, uint columnId)
{
_isNullable = isNullable;
_shouldAlignEncodedValues = shouldAlignEncodedValues;
ColumnId = columnId;
if (_isNullable)
{
_presentBuffer = bufferFactory.CreateBuffer(StreamKind.Present);
_presentBuffer.MustBeIncluded = false;
}
_dataBuffer = bufferFactory.CreateBuffer(StreamKind.Data);
}
public LongWriter(bool isNullable, bool shouldAlignEncodedValues, OrcCompressedBufferFactory bufferFactory, uint columnId)
{
_isNullable = isNullable;
_shouldAlignEncodedValues = shouldAlignEncodedValues;
ColumnId = columnId;
if (_isNullable)
{
_presentBuffer = bufferFactory.CreateBuffer(StreamKind.Present);
_presentBuffer.MustBeIncluded = false; //If we never have nulls, we won't write this stream
}
_dataBuffer = bufferFactory.CreateBuffer(StreamKind.Data);
}
public StringWriter(bool shouldAlignLengths, bool shouldAlignDictionaryLookup, double uniqueStringThresholdRatio, long strideLength, OrcCompressedBufferFactory bufferFactory, uint columnId)
{
_shouldAlignLengths = shouldAlignLengths;
_shouldAlignDictionaryLookup = shouldAlignDictionaryLookup;
_uniqueStringThresholdRatio = uniqueStringThresholdRatio;
_strideLength = strideLength;
ColumnId = columnId;
_presentBuffer = bufferFactory.CreateBuffer(StreamKind.Present);
_presentBuffer.MustBeIncluded = false;
_dataBuffer = bufferFactory.CreateBuffer(StreamKind.Data);
_lengthBuffer = bufferFactory.CreateBuffer(StreamKind.Length);
_dictionaryDataBuffer = bufferFactory.CreateBuffer(StreamKind.DictionaryData);
}
public static void SerializeAndCompressTo(this OrcCompressedBufferFactory bufferFactory, Stream outputStream, object instance, out long length)
{
var buffer = bufferFactory.CreateBuffer();
StaticProtoBuf.Serializer.Serialize(buffer, instance);
buffer.CopyTo(outputStream);
length = buffer.Length;
}
public DecimalWriter(bool isNullable, bool shouldAlignEncodedValues, int precision, int scale, OrcCompressedBufferFactory bufferFactory, uint columnId)
{
_isNullable = isNullable;
_shouldAlignEncodedValues = shouldAlignEncodedValues;
_scale = scale;
ColumnId = columnId;
if (precision > 18)
{
throw new NotSupportedException("This implementation of DecimalWriter does not support precision greater than 18 digits (2^63)");
}
if (_isNullable)
{
_presentBuffer = bufferFactory.CreateBuffer(StreamKind.Present);
_presentBuffer.MustBeIncluded = false;
}
_dataBuffer = bufferFactory.CreateBuffer(StreamKind.Data);
_secondaryBuffer = bufferFactory.CreateBuffer(StreamKind.Secondary);
}
private void CompleteStripe()
{
var stripeFooter = new StripeFooter();
var stripeStats = new StripeStatistics();
//Columns
foreach (var writer in _columnWriters)
{
writer.ColumnWriter.FlushBuffers();
var dictionaryLength =
(writer.ColumnWriter as StringWriter)?.DictionaryLength ??
0; //DictionaryLength is only used by StringWriter
stripeFooter.AddColumn(writer.ColumnWriter.ColumnEncoding, dictionaryLength);
}
var stripeInformation = new StripeInformation
{
Offset = (ulong)_outputStream.Position,
NumberOfRows = (ulong)_rowsInStripe
};
//Indexes
foreach (var writer in _columnWriters)
{
//Write the index buffer
var indexBuffer = _bufferFactory.CreateBuffer(StreamKind.RowIndex);
writer.ColumnWriter.Statistics.WriteToBuffer(indexBuffer);
indexBuffer.CopyTo(_outputStream);
//Add the index to the footer
stripeFooter.AddDataStream(writer.ColumnWriter.ColumnId, indexBuffer);
//Collect summary statistics
var columnStats = new ColumnStatistics();
foreach (var stats in writer.ColumnWriter.Statistics)
{
stats.FillColumnStatistics(columnStats);
stats.FillColumnStatistics(writer.FileStatistics);
}
stripeStats.ColStats.Add(columnStats);
}
_stripeStats.Add(stripeStats);
stripeInformation.IndexLength = (ulong)_outputStream.Position - stripeInformation.Offset;
//Data streams
foreach (var writer in _columnWriters)
{
foreach (var buffer in writer.ColumnWriter.Buffers)
{
if (!buffer.MustBeIncluded)
{
continue;
}
buffer.CopyTo(_outputStream);
stripeFooter.AddDataStream(writer.ColumnWriter.ColumnId, buffer);
}
}
stripeInformation.DataLength = (ulong)_outputStream.Position - stripeInformation.IndexLength -
stripeInformation.Offset;
//Footer
long footerLength;
_bufferFactory.SerializeAndCompressTo(_outputStream, stripeFooter, out footerLength);
stripeInformation.FooterLength = (ulong)footerLength;
_stripeInformations.Add(stripeInformation);
_rowsInFile += _rowsInStripe;
_rowsInStripe = 0;
foreach (var writer in _columnWriters)
{
writer.ColumnWriter.Reset();
}
}