/// <summary>
/// Deserializes to an array of nullable <see lang="double"/> from the specified reader.
/// </summary>
/// <param name="reader">The reader.</param>
/// <returns>An array of nullable <see lang="double"/>.</returns>
public static double?[] DeserializeToNullableDoubles(BinaryReader reader)
{
// version not in use yet.
reader.ReadByte();
var numOfItems = (int)SerializationUtils.ReadUInt32FromBase128(reader);
if (numOfItems == 0)
{
return(EmptyNullableDoubleArray);
}
var result = new double?[numOfItems];
var bitBinaryReader = new BitBinaryReader(reader);
var previousState = new DoubleValueState(0, -1, -1);
for (int i = 0; i < numOfItems; ++i)
{
var newState = ReadDouble(bitBinaryReader, previousState);
result[i] = double.IsNaN(newState.Value) ? (double?)null : newState.Value;
previousState = newState;
}
return(result);
}
public void VerifyEncoding_largecorpus()
{
string input = TestResources.RFC5_Text;
int len = input.Length;
var data = new MemoryStream();
var writer = new BitBinaryWriter(data);
var reader = new BitBinaryReader(data);
var compressor = new DynamicHuffman <char>(OriginalNYT);
for (int i = 0; i < len; ++i)
{
compressor.WriteCode(input[i], writer.Write, writer.Write);
}
writer.Flush();
data.Position = 0;
var decompressor = new DynamicHuffman <char>(OriginalNYT);
for (int i = 0; i < len; ++i)
{
Assert.AreEqual(input[i], decompressor.GetSymbol(reader.ReadBoolean, reader.ReadChar));
}
Assert.AreEqual(data.Position, data.Length);
}
public void BaseStream_NotBitStream()
{
var data = new byte[] { 0x23, 0x72, 0x64, 0xe3, 0x11, 0xd1, 0x4a, 0x9c, 0xb6, 0x02 };
var stream = new MemoryStream(data);
var reader = new BitBinaryReader(stream);
Assert.AreNotEqual(stream, reader.BaseStream);
}
private static unsafe void DeserializeValues(BinaryReader reader, double *values, int count)
{
var bitBinaryReader = new BitBinaryReader(reader);
var previousState = new DoubleValueState(0, -1, -1);
for (int i = 0; i < count; ++i)
{
var newState = ReadDouble(bitBinaryReader, previousState);
values[i] = newState.Value;
previousState = newState;
}
}
public void BinaryWriterCompatible_WithoutBits()
{
var stream = new MemoryStream();
var writer = new BinaryWriter(stream);
var reader = new BitBinaryReader(stream);
foreach (TestValue value in _testValues)
{
value.Write(writer);
}
writer.Flush();
stream.Position = 0;
foreach (TestValue value in _testValues)
{
value.Read(reader);
}
}
private static DoubleValueState ReadDouble(BitBinaryReader reader, DoubleValueState state)
{
var firstBit = reader.ReadBit();
if (!firstBit)
{
return(state);
}
var secondBit = reader.ReadBit();
long meaningFulBits;
if (!secondBit)
{
var numBitsToRead = BitAggregateMagic.NumBitsInLongInteger - state.LeadingZeros - state.TrailingZeros;
meaningFulBits = reader.ReadBits(numBitsToRead);
}
else
{
// a new block position was started since the number starts with "11".
state.LeadingZeros = (sbyte)reader.ReadBits(NumBitsToEncodeNumLeadingZeros);
var numBitsToRead = (sbyte)reader.ReadBits(NumBitsToEncodeNumMeaningfulBits);
if (numBitsToRead == 0)
{
// The block size is 64 bits which becomes 0 in writing into 6 bits - overflow.
// If the block size were indeed 0 bits, the xor value would be 0, and the actual value would be identical to the prior value,
// so we would have bailed out early on since firstBit would be 0.
numBitsToRead = (sbyte)BitAggregateMagic.NumBitsInLongInteger;
}
state.TrailingZeros = (sbyte)(BitAggregateMagic.NumBitsInLongInteger - state.LeadingZeros - numBitsToRead);
meaningFulBits = reader.ReadBits(numBitsToRead);
}
var xor = meaningFulBits << state.TrailingZeros;
state.Value = BitConverter.Int64BitsToDouble(xor ^ BitConverter.DoubleToInt64Bits(state.Value));
return(state);
}
public void VerifyEncoding_astrachan_()
{
string input = "astrachan_";
var data = new MemoryStream();
var writer = new BitBinaryWriter(data);
var reader = new BitBinaryReader(data);
var compressor = new DynamicHuffman <char>(OriginalNYT);
for (int i = 0; i < input.Length; ++i)
{
compressor.WriteCode(input[i], writer.Write, writer.Write);
}
writer.Flush();
data.Position = 0;
var decompressor = new DynamicHuffman <char>(OriginalNYT);
for (int i = 0; i < input.Length; ++i)
{
Assert.AreEqual(input[i], decompressor.GetSymbol(reader.ReadBoolean, reader.ReadChar));
}
Assert.AreEqual(data.Position, data.Length);
}
/// <summary>
/// Deserializes for one timestamp - v2.
/// </summary>
/// <param name="version">The serialization version.</param>
/// <param name="reader">The bit reader.</param>
/// <param name="definition">The definition.</param>
/// <param name="values">The values.</param>
/// <param name="dataPointIndex">Index of the data point.</param>
/// <param name="currentBlockLeadingZeros">The current block leading zeros.</param>
/// <param name="currentBlockTrailingZeros">The current block trailing zeros.</param>
/// <param name="priorValidValues">The prior valid values.</param>
private static void DeserializeForOneTimestampV2AndAbove(byte version, BitBinaryReader reader, TimeSeriesDefinition <MetricIdentifier> definition, List <List <double?> > values, int dataPointIndex, sbyte[] currentBlockLeadingZeros, sbyte[] currentBlockTrailingZeros, double[] priorValidValues)
{
var numSamplingTypes = values.Count;
for (var s = 0; s < numSamplingTypes; s++)
{
if (dataPointIndex == 0)
{
// very first value of the series
priorValidValues[s] = reader.BinaryReader.ReadDouble();
values[s].Add(GetNullableDouble(priorValidValues[s]));
}
else
{
var firstBit = reader.ReadBit();
if (!firstBit)
{
// first bit is 0
values[s].Add(GetNullableDouble(priorValidValues[s]));
}
else
{
var secondBit = reader.ReadBit();
long meaningfulBits;
if (!secondBit)
{
// 2nd bit is 0 while the first is 1.
if (currentBlockLeadingZeros[s] < 0)
{
throw new Exception("The block has not been set so it is a bug in serialization on server");
}
var numBitsToRead = BitAggregateMagic.NumBitsInLongInteger - currentBlockLeadingZeros[s] - currentBlockTrailingZeros[s];
meaningfulBits = reader.ReadBits(numBitsToRead);
}
else
{
// a new block position was started since the number starts with "11".
currentBlockLeadingZeros[s] = (sbyte)reader.ReadBits(GetNumBitsToEncodeNumLeadingZeros(version));
var numBitsToRead = (sbyte)reader.ReadBits(NumBitsToEncodeNumMeaningfulBits);
if (numBitsToRead == 0)
{
// The block size is 64 bits which becomes 0 in writing into 6 bits - overflow.
// If the block size were indeed 0 bits, the xor value would be 0, and the actual value would be identical to the prior value,
// so we would not have reached here since firstBit would be 0.
numBitsToRead = (sbyte)BitAggregateMagic.NumBitsInLongInteger;
}
currentBlockTrailingZeros[s] = (sbyte)(BitAggregateMagic.NumBitsInLongInteger - currentBlockLeadingZeros[s] - numBitsToRead);
meaningfulBits = reader.ReadBits(numBitsToRead);
}
long xor = meaningfulBits << currentBlockTrailingZeros[s];
priorValidValues[s] = BitConverter.Int64BitsToDouble(xor ^ BitConverter.DoubleToInt64Bits(priorValidValues[s]));
values[s].Add(GetNullableDouble(priorValidValues[s]));
}
}
}
}
/// <summary>
/// Deserializes one series.
/// </summary>
/// <param name="version">The serialization version.</param>
/// <param name="reader">The reader.</param>
/// <param name="definition">The definition.</param>
/// <param name="numSamplingTypesRequested">The number of sampling types.</param>
/// <returns>An <see cref="DeserializedRawData"/> object.</returns>
private static DeserializedRawData DeserializeOneSeries(byte version, BinaryReader reader, TimeSeriesDefinition <MetricIdentifier> definition, int numSamplingTypesRequested)
{
// Read the number of data points.
var totalNumberOfDatapoints = SerializationUtils.ReadInt32FromBase128(reader);
if (totalNumberOfDatapoints < 0)
{
// Failed to query the series. We reuse the same byte(s) for totalNumberOfDatapoints for error codes.
return(new DeserializedRawData(0, 0, null, (TimeSeriesErrorCode)totalNumberOfDatapoints));
}
// Read the number of missing data points or those with null values so that we can get the total size of data points next.
// in V2 and beyond, totalNumberMissingDatapoints is always to set to 0 since each missing data point in a very sparse series just uses a single bit for padding,
// and scaling will be done on the server side.
uint totalNumberMissingDatapoints = 0;
uint scalingFactor = 1;
if (version == 1)
{
totalNumberMissingDatapoints = SerializationUtils.ReadUInt32FromBase128(reader);
// We want to apply the scaling factor on the client so that server can return metric values of type long with variable length encoding.
scalingFactor = SerializationUtils.ReadUInt32FromBase128(reader);
}
var numberDatapointsToDeserialize = totalNumberOfDatapoints - (int)totalNumberMissingDatapoints;
var numSamplingTypes = definition?.SamplingTypes.Length ?? numSamplingTypesRequested;
var values = new List <List <double?> >(numSamplingTypes);
for (int index = 0; index < numSamplingTypes; index++)
{
values.Add(new List <double?>(totalNumberOfDatapoints));
}
// We use deltas/differences as compared with the prior data point.
// Although deltas/differences don't help values of type double, it will when we encode double type in the future.
var priorValidValues = new double[numSamplingTypes];
var sampleTyesWithMetricValueTypeLong = new bool[numSamplingTypes];
// Used for implementing the Gorilla algorithm
sbyte[] currentBlockLeadingZeros = new sbyte[numSamplingTypes];
sbyte[] currentBlockTrailingZeros = new sbyte[numSamplingTypes];
for (int index = 0; index < numSamplingTypes; index++)
{
priorValidValues[index] = 0;
currentBlockLeadingZeros[index] = -1;
currentBlockTrailingZeros[index] = -1;
if (definition == null || IsMetricValueTypeLong(definition.SamplingTypes[index], definition.SeriesResolutionInMinutes, definition.AggregationType))
{
sampleTyesWithMetricValueTypeLong[index] = true;
}
}
var sparseData = totalNumberMissingDatapoints > 0;
var bitReader = new BitBinaryReader(reader);
for (int d = 0; d < numberDatapointsToDeserialize; ++d)
{
if (version == 1)
{
DeserializeForOneTimestampV1(reader, definition, sparseData, values, sampleTyesWithMetricValueTypeLong, priorValidValues, scalingFactor);
}
else
{
DeserializeForOneTimestampV2AndAbove(version, bitReader, definition, values, d, currentBlockLeadingZeros, currentBlockTrailingZeros, priorValidValues);
}
}
// Fill the remaining missing data points at the tail of the series.
if (sparseData)
{
if (values[0].Count < totalNumberOfDatapoints)
{
var numNullsToFill = totalNumberOfDatapoints - values[0].Count;
FillNulls((uint)numNullsToFill, values);
}
}
// Start time can be adjusted for distinct count metric or when rollup serivce is enabled.
var deltaOfStartTimeInMinutes = SerializationUtils.ReadInt32FromBase128(reader);
// resolution window can be adjusted for distinct count metric or when rollup serivce is enabled.
var deltaOfResolutionWindowInMinutes = SerializationUtils.ReadInt32FromBase128(reader);
return(new DeserializedRawData(deltaOfStartTimeInMinutes, deltaOfResolutionWindowInMinutes, values, TimeSeriesErrorCode.Success));
}