void TestRoundTrip(long[] test, bool isSigned, bool aligned, int?expectedEncodeLength = null)
{
var stream = new MemoryStream();
var writer = new IntegerRunLengthEncodingV2Writer(stream);
writer.Write(test, isSigned, aligned);
//If we know the encode length, make sure it's correct
if (expectedEncodeLength.HasValue)
{
Assert.Equal(expectedEncodeLength.Value, stream.Length);
}
stream.Seek(0, SeekOrigin.Begin);
var reader = new IntegerRunLengthEncodingV2Reader(stream, isSigned);
var result = reader.Read().ToArray();
//Make sure all bytes in the written stream were consumed
Assert.Equal(stream.Length, stream.Position);
//Check the actual values
Assert.Equal(test.Length, result.Length);
for (int i = 0; i < test.Length; i++)
{
Assert.Equal(test[i], result[i]);
}
}
void TestRead(long[] expected, byte[] input, bool isSigned)
{
var stream = new MemoryStream(input);
var reader = new IntegerRunLengthEncodingV2Reader(stream, isSigned);
var actual = reader.Read().ToArray();
Assert.Equal(expected.Length, actual.Length);
for (int i = 0; i < expected.Length; i++)
{
Assert.Equal(expected[i], actual[i]);
}
}
protected long[] ReadNumericStream(Protocol.StreamKind streamKind, bool isSigned)
{
var stripeStream = GetStripeStream(streamKind);
if (stripeStream == null)
{
return(null);
}
if (stripeStream.ColumnEncodingKind != Protocol.ColumnEncodingKind.DirectV2 && stripeStream.ColumnEncodingKind != Protocol.ColumnEncodingKind.DictionaryV2)
{
throw new NotImplementedException($"Unimplemented Numeric {nameof(Protocol.ColumnEncodingKind)} {stripeStream.ColumnEncodingKind}");
}
var stream = stripeStream.GetDecompressedStream();
var reader = new IntegerRunLengthEncodingV2Reader(stream, isSigned);
return(reader.Read().ToArray());
}
public void ReadIntData()
{
var helper = new ProtocolHelper("demo-12-zlib.orc");
var postscriptLength = helper.GetPostscriptLength();
var postscriptStream = helper.GetPostscriptStream(postscriptLength);
var postScript = Serializer.Deserialize <PostScript>(postscriptStream);
var footerLength = postScript.FooterLength;
var footerStreamCompressed = helper.GetFooterCompressedStream(postscriptLength, footerLength);
var footerStream = OrcCompressedStream.GetDecompressingStream(footerStreamCompressed, CompressionKind.Zlib);
var footer = Serializer.Deserialize <Footer>(footerStream);
var stripeDetails = footer.Stripes[0];
var streamFooterStreamCompressed = helper.GetStripeFooterCompressedStream(stripeDetails.Offset, stripeDetails.IndexLength, stripeDetails.DataLength, stripeDetails.FooterLength);
var stripeFooterStream = OrcCompressedStream.GetDecompressingStream(streamFooterStreamCompressed, CompressionKind.Zlib);
var stripeFooter = Serializer.Deserialize <StripeFooter>(stripeFooterStream);
var offset = stripeDetails.Offset;
foreach (var stream in stripeFooter.Streams)
{
var columnInFooter = footer.Types[(int)stream.Column];
var columnInStripe = stripeFooter.Columns[(int)stream.Column];
if (columnInFooter.Kind == ColumnTypeKind.Int)
{
if (stream.Kind == StreamKind.Data)
{
Assert.Equal(ColumnEncodingKind.DirectV2, columnInStripe.Kind);
var dataStreamCompressed = helper.GetDataCompressedStream(offset, stream.Length);
var dataStream = OrcCompressedStream.GetDecompressingStream(dataStreamCompressed, CompressionKind.Zlib);
var reader = new IntegerRunLengthEncodingV2Reader(dataStream, true);
var result = reader.Read().ToArray();
for (int i = 0; i < result.Length; i++)
{
if (stream.Column == 1)
{
var expected = i + 1;
Assert.Equal(expected, result[i]);
}
else if (stream.Column == 5)
{
var expected = ((i / 70) * 500) % 10000 + 500;
Assert.Equal(expected, result[i]);
}
else if (stream.Column == 7)
{
var expected = (i / 5600) % 7;
Assert.Equal(expected, result[i]);
}
else if (stream.Column == 8)
{
var expected = (i / 39200) % 7;
Assert.Equal(expected, result[i]);
}
else if (stream.Column == 9)
{
var expected = (i / 274400);
Assert.Equal(expected, result[i]);
}
else
{
Assert.True(false, "Unexpected column");
}
}
}
}
offset += stream.Length;
}
}