/// <summary>
/// Determines whether it is count sampling type.
/// </summary>
/// <param name="definition">The definition.</param>
/// <param name="samplingTypeIndex">The sampling type index.</param>
/// <returns>True if it is count sampling type; false otherwise.</returns>
private static bool IsCountSamplingType(TimeSeriesDefinition <MetricIdentifier> definition, int samplingTypeIndex)
{
return((definition == null && samplingTypeIndex == 0) || (definition != null && definition.SamplingTypes[samplingTypeIndex].Equals(SamplingType.Count)));
}
/// <summary>
/// Deserializes for one timestamp - v2.
/// </summary>
/// <param name="version">The serialization version.</param>
/// <param name="reader">The bit reader.</param>
/// <param name="definition">The definition.</param>
/// <param name="values">The values.</param>
/// <param name="dataPointIndex">Index of the data point.</param>
/// <param name="currentBlockLeadingZeros">The current block leading zeros.</param>
/// <param name="currentBlockTrailingZeros">The current block trailing zeros.</param>
/// <param name="priorValidValues">The prior valid values.</param>
private static void DeserializeForOneTimestampV2AndAbove(byte version, BitBinaryReader reader, TimeSeriesDefinition <MetricIdentifier> definition, List <List <double?> > values, int dataPointIndex, sbyte[] currentBlockLeadingZeros, sbyte[] currentBlockTrailingZeros, double[] priorValidValues)
{
var numSamplingTypes = values.Count;
for (var s = 0; s < numSamplingTypes; s++)
{
if (dataPointIndex == 0)
{
// very first value of the series
priorValidValues[s] = reader.BinaryReader.ReadDouble();
values[s].Add(GetNullableDouble(priorValidValues[s]));
}
else
{
var firstBit = reader.ReadBit();
if (!firstBit)
{
// first bit is 0
values[s].Add(GetNullableDouble(priorValidValues[s]));
}
else
{
var secondBit = reader.ReadBit();
long meaningfulBits;
if (!secondBit)
{
// 2nd bit is 0 while the first is 1.
if (currentBlockLeadingZeros[s] < 0)
{
throw new Exception("The block has not been set so it is a bug in serialization on server");
}
var numBitsToRead = BitAggregateMagic.NumBitsInLongInteger - currentBlockLeadingZeros[s] - currentBlockTrailingZeros[s];
meaningfulBits = reader.ReadBits(numBitsToRead);
}
else
{
// a new block position was started since the number starts with "11".
currentBlockLeadingZeros[s] = (sbyte)reader.ReadBits(GetNumBitsToEncodeNumLeadingZeros(version));
var numBitsToRead = (sbyte)reader.ReadBits(NumBitsToEncodeNumMeaningfulBits);
if (numBitsToRead == 0)
{
// The block size is 64 bits which becomes 0 in writing into 6 bits - overflow.
// If the block size were indeed 0 bits, the xor value would be 0, and the actual value would be identical to the prior value,
// so we would not have reached here since firstBit would be 0.
numBitsToRead = (sbyte)BitAggregateMagic.NumBitsInLongInteger;
}
currentBlockTrailingZeros[s] = (sbyte)(BitAggregateMagic.NumBitsInLongInteger - currentBlockLeadingZeros[s] - numBitsToRead);
meaningfulBits = reader.ReadBits(numBitsToRead);
}
long xor = meaningfulBits << currentBlockTrailingZeros[s];
priorValidValues[s] = BitConverter.Int64BitsToDouble(xor ^ BitConverter.DoubleToInt64Bits(priorValidValues[s]));
values[s].Add(GetNullableDouble(priorValidValues[s]));
}
}
}
}
/// <summary>
/// Deserializes one series.
/// </summary>
/// <param name="version">The serialization version.</param>
/// <param name="reader">The reader.</param>
/// <param name="definition">The definition.</param>
/// <param name="numSamplingTypesRequested">The number of sampling types.</param>
/// <returns>An <see cref="DeserializedRawData"/> object.</returns>
private static DeserializedRawData DeserializeOneSeries(byte version, BinaryReader reader, TimeSeriesDefinition <MetricIdentifier> definition, int numSamplingTypesRequested)
{
// Read the number of data points.
var totalNumberOfDatapoints = SerializationUtils.ReadInt32FromBase128(reader);
if (totalNumberOfDatapoints < 0)
{
// Failed to query the series. We reuse the same byte(s) for totalNumberOfDatapoints for error codes.
return(new DeserializedRawData(0, 0, null, (TimeSeriesErrorCode)totalNumberOfDatapoints));
}
// Read the number of missing data points or those with null values so that we can get the total size of data points next.
// in V2 and beyond, totalNumberMissingDatapoints is always to set to 0 since each missing data point in a very sparse series just uses a single bit for padding,
// and scaling will be done on the server side.
uint totalNumberMissingDatapoints = 0;
uint scalingFactor = 1;
if (version == 1)
{
totalNumberMissingDatapoints = SerializationUtils.ReadUInt32FromBase128(reader);
// We want to apply the scaling factor on the client so that server can return metric values of type long with variable length encoding.
scalingFactor = SerializationUtils.ReadUInt32FromBase128(reader);
}
var numberDatapointsToDeserialize = totalNumberOfDatapoints - (int)totalNumberMissingDatapoints;
var numSamplingTypes = definition?.SamplingTypes.Length ?? numSamplingTypesRequested;
var values = new List <List <double?> >(numSamplingTypes);
for (int index = 0; index < numSamplingTypes; index++)
{
values.Add(new List <double?>(totalNumberOfDatapoints));
}
// We use deltas/differences as compared with the prior data point.
// Although deltas/differences don't help values of type double, it will when we encode double type in the future.
var priorValidValues = new double[numSamplingTypes];
var sampleTyesWithMetricValueTypeLong = new bool[numSamplingTypes];
// Used for implementing the Gorilla algorithm
sbyte[] currentBlockLeadingZeros = new sbyte[numSamplingTypes];
sbyte[] currentBlockTrailingZeros = new sbyte[numSamplingTypes];
for (int index = 0; index < numSamplingTypes; index++)
{
priorValidValues[index] = 0;
currentBlockLeadingZeros[index] = -1;
currentBlockTrailingZeros[index] = -1;
if (definition == null || IsMetricValueTypeLong(definition.SamplingTypes[index], definition.SeriesResolutionInMinutes, definition.AggregationType))
{
sampleTyesWithMetricValueTypeLong[index] = true;
}
}
var sparseData = totalNumberMissingDatapoints > 0;
var bitReader = new BitBinaryReader(reader);
for (int d = 0; d < numberDatapointsToDeserialize; ++d)
{
if (version == 1)
{
DeserializeForOneTimestampV1(reader, definition, sparseData, values, sampleTyesWithMetricValueTypeLong, priorValidValues, scalingFactor);
}
else
{
DeserializeForOneTimestampV2AndAbove(version, bitReader, definition, values, d, currentBlockLeadingZeros, currentBlockTrailingZeros, priorValidValues);
}
}
// Fill the remaining missing data points at the tail of the series.
if (sparseData)
{
if (values[0].Count < totalNumberOfDatapoints)
{
var numNullsToFill = totalNumberOfDatapoints - values[0].Count;
FillNulls((uint)numNullsToFill, values);
}
}
// Start time can be adjusted for distinct count metric or when rollup serivce is enabled.
var deltaOfStartTimeInMinutes = SerializationUtils.ReadInt32FromBase128(reader);
// resolution window can be adjusted for distinct count metric or when rollup serivce is enabled.
var deltaOfResolutionWindowInMinutes = SerializationUtils.ReadInt32FromBase128(reader);
return(new DeserializedRawData(deltaOfStartTimeInMinutes, deltaOfResolutionWindowInMinutes, values, TimeSeriesErrorCode.Success));
}
/// <summary>
/// Deserializes for one timestamp - v1.
/// </summary>
/// <param name="reader">The reader.</param>
/// <param name="definition">The definition.</param>
/// <param name="sparseData">if set to <c>true</c> [sparse data].</param>
/// <param name="values">The metric values.</param>
/// <param name="sampleTyesWithMetricValueTypeLong">The sample tyes with metric value type long.</param>
/// <param name="priorValidValues">The prior valid values.</param>
/// <param name="scalingFactor">The scaling factor.</param>
private static void DeserializeForOneTimestampV1(
BinaryReader reader,
TimeSeriesDefinition <MetricIdentifier> definition,
bool sparseData,
List <List <double?> > values,
bool[] sampleTyesWithMetricValueTypeLong,
double[] priorValidValues,
uint scalingFactor)
{
// For sparse data, although we don't serialize null values but we do serialize the number of null values between two serialized data points
// so that we can restore those null values on the client side.
if (sparseData)
{
var numMissingDatapointsSinceLastOne = SerializationUtils.ReadUInt32FromBase128(reader);
FillNulls(numMissingDatapointsSinceLastOne, values);
}
for (int samplingTypeIndex = 0; samplingTypeIndex < values.Count; samplingTypeIndex++)
{
if (sampleTyesWithMetricValueTypeLong[samplingTypeIndex])
{
// variable length encoding with differences.
var delta = SerializationUtils.ReadInt64FromBase128(reader);
if (delta == long.MaxValue)
{
// Padding
values[samplingTypeIndex].Add(null);
}
else
{
// We request unscaled values from server and apply the scaling factor on the client.
var unScaledValue = delta + priorValidValues[samplingTypeIndex];
if (IsCountSamplingType(definition, samplingTypeIndex))
{
values[samplingTypeIndex].Add(unScaledValue);
}
else
{
values[samplingTypeIndex].Add(unScaledValue / scalingFactor);
}
priorValidValues[samplingTypeIndex] = unScaledValue;
}
}
else
{
// We don't encode double in v1, so even a delta is still 8 bytes.
var delta = reader.ReadDouble();
if (double.IsNaN(delta))
{
// Padding
values[samplingTypeIndex].Add(null);
}
else
{
var unScaledValue = delta + priorValidValues[samplingTypeIndex];
if (IsCountSamplingType(definition, samplingTypeIndex))
{
values[samplingTypeIndex].Add(delta + priorValidValues[samplingTypeIndex]);
}
else
{
values[samplingTypeIndex].Add((delta + priorValidValues[samplingTypeIndex]) / scalingFactor);
}
priorValidValues[samplingTypeIndex] = unScaledValue;
}
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="TimeSeries{ID, TValue}" /> class.
/// </summary>
/// <param name="startTimeUtc">The start time in UTC.</param>
/// <param name="endTimeUtc">The end time in UTC.</param>
/// <param name="seriesResolutionInMinutes">The series resolution in minutes.</param>
/// <param name="definition">The time series definition.</param>
/// <param name="values">The time series values.</param>
/// <param name="errorCode">The error code.</param>
internal TimeSeries(DateTime startTimeUtc, DateTime endTimeUtc, int seriesResolutionInMinutes, TimeSeriesDefinition <TId> definition, List <List <TValue> > values, TimeSeriesErrorCode errorCode)
{
this.Definition = definition;
this.StartTimeUtc = startTimeUtc;
this.EndTimeUtc = endTimeUtc;
this.SeriesResolutionInMinutes = seriesResolutionInMinutes;
this.Values = values;
this.ErrorCode = errorCode;
}