/// <summary> /// Determines whether it is count sampling type. /// </summary> /// <param name="definition">The definition.</param> /// <param name="samplingTypeIndex">The sampling type index.</param> /// <returns>True if it is count sampling type; false otherwise.</returns> private static bool IsCountSamplingType(TimeSeriesDefinition <MetricIdentifier> definition, int samplingTypeIndex) { return((definition == null && samplingTypeIndex == 0) || (definition != null && definition.SamplingTypes[samplingTypeIndex].Equals(SamplingType.Count))); }
/// <summary> /// Deserializes for one timestamp - v2. /// </summary> /// <param name="version">The serialization version.</param> /// <param name="reader">The bit reader.</param> /// <param name="definition">The definition.</param> /// <param name="values">The values.</param> /// <param name="dataPointIndex">Index of the data point.</param> /// <param name="currentBlockLeadingZeros">The current block leading zeros.</param> /// <param name="currentBlockTrailingZeros">The current block trailing zeros.</param> /// <param name="priorValidValues">The prior valid values.</param> private static void DeserializeForOneTimestampV2AndAbove(byte version, BitBinaryReader reader, TimeSeriesDefinition <MetricIdentifier> definition, List <List <double?> > values, int dataPointIndex, sbyte[] currentBlockLeadingZeros, sbyte[] currentBlockTrailingZeros, double[] priorValidValues) { var numSamplingTypes = values.Count; for (var s = 0; s < numSamplingTypes; s++) { if (dataPointIndex == 0) { // very first value of the series priorValidValues[s] = reader.BinaryReader.ReadDouble(); values[s].Add(GetNullableDouble(priorValidValues[s])); } else { var firstBit = reader.ReadBit(); if (!firstBit) { // first bit is 0 values[s].Add(GetNullableDouble(priorValidValues[s])); } else { var secondBit = reader.ReadBit(); long meaningfulBits; if (!secondBit) { // 2nd bit is 0 while the first is 1. if (currentBlockLeadingZeros[s] < 0) { throw new Exception("The block has not been set so it is a bug in serialization on server"); } var numBitsToRead = BitAggregateMagic.NumBitsInLongInteger - currentBlockLeadingZeros[s] - currentBlockTrailingZeros[s]; meaningfulBits = reader.ReadBits(numBitsToRead); } else { // a new block position was started since the number starts with "11". currentBlockLeadingZeros[s] = (sbyte)reader.ReadBits(GetNumBitsToEncodeNumLeadingZeros(version)); var numBitsToRead = (sbyte)reader.ReadBits(NumBitsToEncodeNumMeaningfulBits); if (numBitsToRead == 0) { // The block size is 64 bits which becomes 0 in writing into 6 bits - overflow. // If the block size were indeed 0 bits, the xor value would be 0, and the actual value would be identical to the prior value, // so we would not have reached here since firstBit would be 0. numBitsToRead = (sbyte)BitAggregateMagic.NumBitsInLongInteger; } currentBlockTrailingZeros[s] = (sbyte)(BitAggregateMagic.NumBitsInLongInteger - currentBlockLeadingZeros[s] - numBitsToRead); meaningfulBits = reader.ReadBits(numBitsToRead); } long xor = meaningfulBits << currentBlockTrailingZeros[s]; priorValidValues[s] = BitConverter.Int64BitsToDouble(xor ^ BitConverter.DoubleToInt64Bits(priorValidValues[s])); values[s].Add(GetNullableDouble(priorValidValues[s])); } } } }
/// <summary> /// Deserializes one series. /// </summary> /// <param name="version">The serialization version.</param> /// <param name="reader">The reader.</param> /// <param name="definition">The definition.</param> /// <param name="numSamplingTypesRequested">The number of sampling types.</param> /// <returns>An <see cref="DeserializedRawData"/> object.</returns> private static DeserializedRawData DeserializeOneSeries(byte version, BinaryReader reader, TimeSeriesDefinition <MetricIdentifier> definition, int numSamplingTypesRequested) { // Read the number of data points. var totalNumberOfDatapoints = SerializationUtils.ReadInt32FromBase128(reader); if (totalNumberOfDatapoints < 0) { // Failed to query the series. We reuse the same byte(s) for totalNumberOfDatapoints for error codes. return(new DeserializedRawData(0, 0, null, (TimeSeriesErrorCode)totalNumberOfDatapoints)); } // Read the number of missing data points or those with null values so that we can get the total size of data points next. // in V2 and beyond, totalNumberMissingDatapoints is always to set to 0 since each missing data point in a very sparse series just uses a single bit for padding, // and scaling will be done on the server side. uint totalNumberMissingDatapoints = 0; uint scalingFactor = 1; if (version == 1) { totalNumberMissingDatapoints = SerializationUtils.ReadUInt32FromBase128(reader); // We want to apply the scaling factor on the client so that server can return metric values of type long with variable length encoding. scalingFactor = SerializationUtils.ReadUInt32FromBase128(reader); } var numberDatapointsToDeserialize = totalNumberOfDatapoints - (int)totalNumberMissingDatapoints; var numSamplingTypes = definition?.SamplingTypes.Length ?? numSamplingTypesRequested; var values = new List <List <double?> >(numSamplingTypes); for (int index = 0; index < numSamplingTypes; index++) { values.Add(new List <double?>(totalNumberOfDatapoints)); } // We use deltas/differences as compared with the prior data point. // Although deltas/differences don't help values of type double, it will when we encode double type in the future. var priorValidValues = new double[numSamplingTypes]; var sampleTyesWithMetricValueTypeLong = new bool[numSamplingTypes]; // Used for implementing the Gorilla algorithm sbyte[] currentBlockLeadingZeros = new sbyte[numSamplingTypes]; sbyte[] currentBlockTrailingZeros = new sbyte[numSamplingTypes]; for (int index = 0; index < numSamplingTypes; index++) { priorValidValues[index] = 0; currentBlockLeadingZeros[index] = -1; currentBlockTrailingZeros[index] = -1; if (definition == null || IsMetricValueTypeLong(definition.SamplingTypes[index], definition.SeriesResolutionInMinutes, definition.AggregationType)) { sampleTyesWithMetricValueTypeLong[index] = true; } } var sparseData = totalNumberMissingDatapoints > 0; var bitReader = new BitBinaryReader(reader); for (int d = 0; d < numberDatapointsToDeserialize; ++d) { if (version == 1) { DeserializeForOneTimestampV1(reader, definition, sparseData, values, sampleTyesWithMetricValueTypeLong, priorValidValues, scalingFactor); } else { DeserializeForOneTimestampV2AndAbove(version, bitReader, definition, values, d, currentBlockLeadingZeros, currentBlockTrailingZeros, priorValidValues); } } // Fill the remaining missing data points at the tail of the series. if (sparseData) { if (values[0].Count < totalNumberOfDatapoints) { var numNullsToFill = totalNumberOfDatapoints - values[0].Count; FillNulls((uint)numNullsToFill, values); } } // Start time can be adjusted for distinct count metric or when rollup serivce is enabled. var deltaOfStartTimeInMinutes = SerializationUtils.ReadInt32FromBase128(reader); // resolution window can be adjusted for distinct count metric or when rollup serivce is enabled. var deltaOfResolutionWindowInMinutes = SerializationUtils.ReadInt32FromBase128(reader); return(new DeserializedRawData(deltaOfStartTimeInMinutes, deltaOfResolutionWindowInMinutes, values, TimeSeriesErrorCode.Success)); }
/// <summary> /// Deserializes for one timestamp - v1. /// </summary> /// <param name="reader">The reader.</param> /// <param name="definition">The definition.</param> /// <param name="sparseData">if set to <c>true</c> [sparse data].</param> /// <param name="values">The metric values.</param> /// <param name="sampleTyesWithMetricValueTypeLong">The sample tyes with metric value type long.</param> /// <param name="priorValidValues">The prior valid values.</param> /// <param name="scalingFactor">The scaling factor.</param> private static void DeserializeForOneTimestampV1( BinaryReader reader, TimeSeriesDefinition <MetricIdentifier> definition, bool sparseData, List <List <double?> > values, bool[] sampleTyesWithMetricValueTypeLong, double[] priorValidValues, uint scalingFactor) { // For sparse data, although we don't serialize null values but we do serialize the number of null values between two serialized data points // so that we can restore those null values on the client side. if (sparseData) { var numMissingDatapointsSinceLastOne = SerializationUtils.ReadUInt32FromBase128(reader); FillNulls(numMissingDatapointsSinceLastOne, values); } for (int samplingTypeIndex = 0; samplingTypeIndex < values.Count; samplingTypeIndex++) { if (sampleTyesWithMetricValueTypeLong[samplingTypeIndex]) { // variable length encoding with differences. var delta = SerializationUtils.ReadInt64FromBase128(reader); if (delta == long.MaxValue) { // Padding values[samplingTypeIndex].Add(null); } else { // We request unscaled values from server and apply the scaling factor on the client. var unScaledValue = delta + priorValidValues[samplingTypeIndex]; if (IsCountSamplingType(definition, samplingTypeIndex)) { values[samplingTypeIndex].Add(unScaledValue); } else { values[samplingTypeIndex].Add(unScaledValue / scalingFactor); } priorValidValues[samplingTypeIndex] = unScaledValue; } } else { // We don't encode double in v1, so even a delta is still 8 bytes. var delta = reader.ReadDouble(); if (double.IsNaN(delta)) { // Padding values[samplingTypeIndex].Add(null); } else { var unScaledValue = delta + priorValidValues[samplingTypeIndex]; if (IsCountSamplingType(definition, samplingTypeIndex)) { values[samplingTypeIndex].Add(delta + priorValidValues[samplingTypeIndex]); } else { values[samplingTypeIndex].Add((delta + priorValidValues[samplingTypeIndex]) / scalingFactor); } priorValidValues[samplingTypeIndex] = unScaledValue; } } } }
/// <summary> /// Initializes a new instance of the <see cref="TimeSeries{ID, TValue}" /> class. /// </summary> /// <param name="startTimeUtc">The start time in UTC.</param> /// <param name="endTimeUtc">The end time in UTC.</param> /// <param name="seriesResolutionInMinutes">The series resolution in minutes.</param> /// <param name="definition">The time series definition.</param> /// <param name="values">The time series values.</param> /// <param name="errorCode">The error code.</param> internal TimeSeries(DateTime startTimeUtc, DateTime endTimeUtc, int seriesResolutionInMinutes, TimeSeriesDefinition <TId> definition, List <List <TValue> > values, TimeSeriesErrorCode errorCode) { this.Definition = definition; this.StartTimeUtc = startTimeUtc; this.EndTimeUtc = endTimeUtc; this.SeriesResolutionInMinutes = seriesResolutionInMinutes; this.Values = values; this.ErrorCode = errorCode; }