/// <summary>
/// Deserializes to an array of nullable <see lang="double"/> from the specified reader.
/// </summary>
/// <param name="reader">The reader.</param>
/// <returns>An array of nullable <see lang="double"/>.</returns>
public static double?[] DeserializeToNullableDoubles(BinaryReader reader)
{
// version not in use yet.
reader.ReadByte();
var numOfItems = (int)SerializationUtils.ReadUInt32FromBase128(reader);
if (numOfItems == 0)
{
return(EmptyNullableDoubleArray);
}
var result = new double?[numOfItems];
var bitBinaryReader = new BitBinaryReader(reader);
var previousState = new DoubleValueState(0, -1, -1);
for (int i = 0; i < numOfItems; ++i)
{
var newState = ReadDouble(bitBinaryReader, previousState);
result[i] = double.IsNaN(newState.Value) ? (double?)null : newState.Value;
previousState = newState;
}
return(result);
}
public static unsafe void Deserialize(BinaryReader reader, double *values, int expectedCount)
{
// version not in use yet.
reader.ReadByte();
var count = (int)SerializationUtils.ReadUInt32FromBase128(reader);
if (count != expectedCount)
{
throw new InvalidDataException($"Wrong count in serialized data: expected {expectedCount}, but was {count}");
}
DeserializeValues(reader, values, count);
}
public static double[] Deserialize(BinaryReader reader)
{
// version not in use yet.
reader.ReadByte();
var numOfItems = (int)SerializationUtils.ReadUInt32FromBase128(reader);
if (numOfItems == 0)
{
return(EmptyDoubleArray);
}
var result = new double[numOfItems];
unsafe
{
fixed(double *p = &result[0])
{
DeserializeValues(reader, p, numOfItems);
}
}
return(result);
}
private TMetadata ReadMetricMetadata(BinaryReader reader, IMetricBuilder <TMetadata> metricBuilder, ushort version, int maxMetricStringsLength, int maxMetricNamespaceStringsLength)
{
var metricNamespace = string.Empty;
// In versions 0-2 Metric Namespace was part of the Metric data, from version 3 it became a part of Metric Metadata
if (version >= 3)
{
metricNamespace = this.ReadStringByIndex(reader);
if (metricNamespace.Length > maxMetricNamespaceStringsLength)
{
throw new MetricSerializationException($"Namespace string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricNamespaceStringsLength}, Value:{metricNamespace}.", null);
}
}
var metricName = this.ReadStringByIndex(reader);
if (metricName.Length > maxMetricStringsLength)
{
throw new MetricSerializationException($"Metric name string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricStringsLength}, Value:{metricName}.", null);
}
var count = SerializationUtils.ReadUInt32FromBase128(reader);
var dimensionNames = new List <string>((int)count);
for (var i = 0; i < count; ++i)
{
var dimensionName = this.ReadStringByIndex(reader);
if (dimensionName.Length > maxMetricStringsLength)
{
throw new MetricSerializationException($"Dimension name string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricStringsLength}, Value:{dimensionName}.", null);
}
dimensionNames.Add(dimensionName);
}
return(metricBuilder.CreateMetadata(metricNamespace, metricName, dimensionNames));
}
private TMetadata ReadMetricMetadata(BinaryReader reader, IFrontEndMetricBuilder <TMetadata> metricBuilder, int maxMetricStringsLength, int maxMetricNamespaceStringsLength)
{
var metricNamespace = this.ReadStringByIndex(reader);
if (metricNamespace.Length > maxMetricNamespaceStringsLength)
{
throw new MetricSerializationException(
$"Namespace string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricNamespaceStringsLength}, Value:{metricNamespace}.", null, true);
}
var metricName = this.ReadStringByIndex(reader);
if (metricName.Length > maxMetricStringsLength)
{
throw new MetricSerializationException(
$"Metric name string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricStringsLength}, Value:{metricName}.", null, true);
}
var count = SerializationUtils.ReadUInt32FromBase128(reader);
for (var i = 0; i < count; ++i)
{
var dimensionName = this.ReadStringByIndex(reader);
if (dimensionName.Length > maxMetricStringsLength)
{
throw new MetricSerializationException(
$"Dimension name string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricStringsLength}, Value:{dimensionName}.", null, true);
}
this.reusableStringsList.Add(dimensionName);
}
var result = metricBuilder.CreateMetadata(metricNamespace, metricName, this.reusableStringsList);
this.reusableStringsList.Clear();
return(result);
}
/// <summary>
/// Deserializes counter (metric) data from the stream and adds all objects to provided collection.
/// </summary>
/// <param name="stream">Stream from which data should be deserialized. Stream should be readable and provide randon access.</param>
/// <param name="metricBuilder">An object responsible for creation and further consumption of deserialized data.</param>
/// <param name="maxMetricStringsLength">Maximum length of strings, which represent metric name, dimension names and values.</param>
/// <param name="maxMetricNamespaceStringsLength">Maximum length of metric namespace string.</param>
/// <param name="maxMetricDimensionValueStringsLength">Maximum length of metric dimension value string.</param>
public void Deserialize(Stream stream, IMetricBuilder <TMetadata> metricBuilder, int maxMetricStringsLength, int maxMetricNamespaceStringsLength, int maxMetricDimensionValueStringsLength)
{
if (!stream.CanRead || !stream.CanSeek)
{
throw new ArgumentException(@"Stream should be readable and provide random access.", nameof(stream));
}
try
{
using (var reader = new NoCloseBinaryReader(stream, Encoding.UTF8))
{
var startStreamPosition = stream.Position;
// Read version and type serializers info
var version = reader.ReadUInt16();
if (version > MaxVersion)
{
throw new VersionNotSupportedMetricSerializationException(
string.Format(
CultureInfo.InvariantCulture,
"Version is not supported. Read version:{0}, Max version:{1}.",
version,
MaxVersion));
}
if (version >= 5)
{
// Read CRC. CRC check is done in upper layers.
reader.ReadUInt32();
}
if (reader.ReadUInt32() != TypeSerializerFlags)
{
throw new VersionNotSupportedMetricSerializationException("Type serializers not supported.");
}
metricBuilder.SetSerializationVersion(version);
// Read strings
var deserializerDataPosition = stream.Position;
stream.Position += sizeof(long);
stream.Position = startStreamPosition + reader.ReadInt64();
var count = SerializationUtils.ReadUInt32FromBase128(reader);
for (uint i = 0; i < count; ++i)
{
this.stringsDictionary.Add(reader.ReadString());
}
// Read metrics metadata
stream.Position = deserializerDataPosition;
stream.Position = startStreamPosition + reader.ReadInt64();
count = SerializationUtils.ReadUInt32FromBase128(reader);
for (uint i = 0; i < count; ++i)
{
this.metadataDictionary.Add(this.ReadMetricMetadata(reader, metricBuilder, version, maxMetricStringsLength, maxMetricNamespaceStringsLength));
}
// Read metrics data
stream.Position = deserializerDataPosition + (2 * sizeof(long));
this.ReadMetricsData(reader, metricBuilder, version, maxMetricStringsLength, maxMetricNamespaceStringsLength, maxMetricDimensionValueStringsLength);
}
}
catch (IOException ioException)
{
throw new MetricSerializationException("Failed to deserialize data. Problem with input stream.", ioException);
}
catch (Exception exception)
{
throw new MetricSerializationException("Failed to deserialize data. Likely the incoming stream contains corrupted data.", exception);
}
finally
{
this.metadataDictionary.Clear();
this.stringsDictionary.Clear();
this.histogramBuffer.Clear();
}
}
private TMetadata ReadMetricMetadataByIndex(BinaryReader reader)
{
var index = (int)SerializationUtils.ReadUInt32FromBase128(reader);
return(this.metadataDictionary[index]);
}
private string ReadStringByIndex(BinaryReader reader)
{
var index = (int)SerializationUtils.ReadUInt32FromBase128(reader);
return(this.stringsDictionary[index]);
}
private void ReadMetricsData(BinaryReader reader, IMetricBuilder <TMetadata> metricBuilder, ushort version, int maxMetricStringsLength, int maxMetricNamespaceStringsLength, int maxMetricDimensionValueStringsLength)
{
long packetTime = 0;
if (version >= 5)
{
packetTime = (long)SerializationUtils.ReadUInt64FromBase128(reader);
}
// Versions before 2 used variable number of bytes to write number of serialized metrics data.
// From version 2 passing IEnumerable<IReadOnlyMetric> is supported, thus number of metrics data
// is unknown beforehand and we cannot use variable number anymore. Thus fixed 4 bytes uint is used.
var count = version >= 2 ? reader.ReadUInt32() : SerializationUtils.ReadUInt32FromBase128(reader);
for (var i = 0; i < count; ++i)
{
metricBuilder.BeginMetricCreation();
var metadata = this.ReadMetricMetadataByIndex(reader);
metricBuilder.AssignMetadata(metadata);
// In versions 0-2 Monitoring Account and Metric Namespace was part of the Metric data
// From version 3 Monitoring Account is removed and Metric Namespace became a part of Metric Metadata
if (version < 3)
{
var monitoringAccount = this.ReadStringByIndex(reader);
if (monitoringAccount.Length > maxMetricStringsLength)
{
throw new MetricSerializationException($"Monitoring account string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricStringsLength}, Value:{monitoringAccount}.", null);
}
var metricNamespace = this.ReadStringByIndex(reader);
if (metricNamespace.Length > maxMetricNamespaceStringsLength)
{
throw new MetricSerializationException($"Namespace string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricNamespaceStringsLength}, Value:{metricNamespace}.", null);
}
metricBuilder.AssignMonitoringAccount(monitoringAccount);
metricBuilder.AssignNamespace(metricNamespace);
}
if (version == 0)
{
// Skip event id
this.ReadStringByIndex(reader);
}
if (version >= 5)
{
var timeInTicks = (packetTime - SerializationUtils.ReadInt64FromBase128(reader)) * SerializationUtils.OneMinuteInterval;
metricBuilder.AssignTimeUtc(new DateTime(timeInTicks, DateTimeKind.Utc));
}
else
{
metricBuilder.AssignTimeUtc(new DateTime((long)SerializationUtils.ReadUInt64FromBase128(reader), DateTimeKind.Utc));
}
for (var j = 0; j < metadata.DimensionsCount; ++j)
{
var dimensionValue = this.ReadStringByIndex(reader);
if (dimensionValue.Length > maxMetricDimensionValueStringsLength)
{
throw new MetricSerializationException($"Dimension value string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricDimensionValueStringsLength}, Value:{dimensionValue}.", null);
}
metricBuilder.AddDimensionValue(dimensionValue);
}
var samplingTypes = (SamplingTypes)SerializationUtils.ReadUInt32FromBase128(reader);
metricBuilder.AssignSamplingTypes(samplingTypes);
if ((samplingTypes & SamplingTypes.Min) != 0)
{
metricBuilder.AssignMin(SerializationUtils.ReadUInt64FromBase128(reader));
}
if ((samplingTypes & SamplingTypes.Max) != 0)
{
metricBuilder.AssignMax(SerializationUtils.ReadUInt64FromBase128(reader));
}
if ((samplingTypes & SamplingTypes.Sum) != 0)
{
metricBuilder.AssignSum(SerializationUtils.ReadUInt64FromBase128(reader));
}
if ((samplingTypes & SamplingTypes.Count) != 0)
{
metricBuilder.AssignCount(SerializationUtils.ReadUInt32FromBase128(reader));
}
if ((samplingTypes & SamplingTypes.SumOfSquareDiffFromMean) != 0)
{
var sumOfSquareDiffFromMean = reader.ReadDouble();
metricBuilder.AssignSumOfSquareDiffFromMean(sumOfSquareDiffFromMean);
}
if ((samplingTypes & SamplingTypes.Histogram) != 0)
{
metricBuilder.AssignHistogram(this.ReadHistogram(reader, version));
}
if ((samplingTypes & SamplingTypes.HyperLogLogSketch) != 0)
{
var sizeOfHyperLogLogSketches = reader.ReadInt32();
metricBuilder.AssignHyperLogLogSketch(reader, sizeOfHyperLogLogSketches);
}
metricBuilder.EndMetricCreation();
}
}
private void ReadMetricsData(
BinaryReader reader,
IFrontEndMetricBuilder <TMetadata> metricBuilder,
ushort version,
int maxMetricDimensionValueStringsLength)
{
long packetTime = 0;
if (version >= 5)
{
packetTime = (long)SerializationUtils.ReadUInt64FromBase128(reader);
}
Stream readerStream = reader.BaseStream;
var metricsCount = reader.ReadUInt32();
for (var i = 0; i < metricsCount; ++i)
{
DateTime timeUtc;
var count = 0U;
var sum = default(MetricValueV2);
var min = default(MetricValueV2);
var max = default(MetricValueV2);
double sumOfSquareDiffFromMean = 0;
var metadata = this.ReadMetricMetadataByIndex(reader);
if (version >= 5)
{
var timeInTicks = (packetTime - SerializationUtils.ReadInt64FromBase128(reader)) * SerializationUtils.OneMinuteInterval;
timeUtc = new DateTime(timeInTicks, DateTimeKind.Utc);
}
else
{
timeUtc = new DateTime((long)SerializationUtils.ReadUInt64FromBase128(reader), DateTimeKind.Utc);
}
for (var j = 0; j < metadata.DimensionsCount; ++j)
{
var dimensionValue = this.ReadStringByIndex(reader);
if (dimensionValue.Length > maxMetricDimensionValueStringsLength)
{
throw new MetricSerializationException($"Dimension value string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricDimensionValueStringsLength}, Value:{dimensionValue}.", null, true);
}
this.reusableStringsList.Add(dimensionValue);
}
var samplingTypes = (SamplingTypes)SerializationUtils.ReadUInt32FromBase128(reader);
var isDouble = (samplingTypes & SamplingTypes.DoubleValueType) != 0;
var isDoubleStoredAslong = (samplingTypes & SamplingTypes.DoubleValueStoredAsLongType) != 0;
if ((samplingTypes & SamplingTypes.Min) != 0)
{
min = this.ReadMetricValue(reader, isDouble, isDoubleStoredAslong);
}
if ((samplingTypes & SamplingTypes.Max) != 0)
{
max = this.ReadMetricValue(reader, isDouble, isDoubleStoredAslong);
}
if ((samplingTypes & SamplingTypes.Sum) != 0)
{
sum = this.ReadMetricValue(reader, isDouble, isDoubleStoredAslong);
}
if ((samplingTypes & SamplingTypes.Count) != 0)
{
count = SerializationUtils.ReadUInt32FromBase128(reader);
}
if ((samplingTypes & SamplingTypes.SumOfSquareDiffFromMean) != 0)
{
sumOfSquareDiffFromMean = reader.ReadDouble();
}
bool haveHistogram = (samplingTypes & SamplingTypes.Histogram) != 0;
metricBuilder.BeginMetricCreation(metadata, this.reusableStringsList, timeUtc, samplingTypes, count, sum, min, max, sumOfSquareDiffFromMean);
this.reusableStringsList.Clear();
if (haveHistogram)
{
IEnumerable <KeyValuePair <ulong, uint> > histogramBuckets =
SerializationUtils.ReadHistogram(reader, hasHistogramSizePrefix: version > 3);
this.histogramBuffer.Clear();
this.histogramBuffer.AddRange(histogramBuckets);
metricBuilder.AssignHistogram(this.histogramBuffer);
}
if ((samplingTypes & SamplingTypes.HyperLogLogSketch) != 0)
{
var sizeOfHyperLogLogSketches = reader.ReadInt32();
metricBuilder.AssignHyperLogLogSketch(reader, sizeOfHyperLogLogSketches);
}
if (version >= 6)
{
bool haveTDigest = (samplingTypes & SamplingTypes.TDigest) != 0;
bool readTDigest = false;
// starting from version 6, there is a list of TLV-type
// (https://en.wikipedia.org/wiki/Type-length-value) tuples in the rest of the serialized metric.
// deserialize all of them ignoring the unknown ones.
// list TLV values is expected to contain a single end-of-list marker in the end with T = 0x00
uint type;
while ((type = SerializationUtils.ReadUInt32FromBase128(reader)) != 0x00)
{
int length = (int)SerializationUtils.ReadUInt32FromBase128(reader);
long nextPos = readerStream.Position + length;
switch (type)
{
case TDigestPrefixValue: // 0x74 == 't' (tDigest)
if (haveTDigest)
{
if (!readTDigest)
{
metricBuilder.AssignTDigest(reader, length);
readTDigest = true;
}
else
{
// if we already saw tDigest value and see it
// second time it is a sign of a protocol error.
throw new MetricSerializationException("Saw 2 TDigest values for the same metric", null, isInvalidData: true);
}
}
else
{
// if TLV list contains tDigest but the sampling types does not
// it is a protocol error
throw new MetricSerializationException("Sampling types do not contain tDigest, but TLV list contains it", null, isInvalidData: true);
}
break;
default:
// ignore unknown types
break;
}
// always set the position to point to the next entry.
// do not trust the deserializer code to leave the position set correctly
// this helps prevent compatibility problems and makes the deserializer
// more stable
readerStream.Position = nextPos;
}
if (haveTDigest && !readTDigest)
{
// if sampling types contain TDigest but we have not seen it in TLV this is
// a sign of protocol error
throw new MetricSerializationException("Sampling types contain tDigest, but TLV list does not contain it", null, isInvalidData: true);
}
}
metricBuilder.EndMetricCreation();
}
}