/// <summary>
/// Read metrics count and advances the stream.
/// </summary>
/// <param name="stream">Stream of incoming events data for one packet.</param>
/// <returns>Number of metrics in packet.</returns>
public int ReadMetricsCount(Stream stream)
{
using (var reader = new BinaryReader(stream, Encoding.UTF8))
{
var version = reader.ReadUInt16();
if (version >= 5)
{
reader.ReadUInt32(); // for CRC
}
if (reader.ReadUInt32() != TypeSerializerFlags)
{
throw new VersionNotSupportedMetricSerializationException("Type serializers not supported.");
}
stream.Position += 2 * sizeof(long); // skipping address of strings and address of metadata
if (version >= 5)
{
SerializationUtils.ReadInt64FromBase128(reader); // for packet time
}
return(reader.ReadInt32());
}
}
private MetricValueV2 ReadMetricValue(BinaryReader reader, bool isDouble, bool isDoubleStoredAsLong)
{
if (isDouble)
{
if (isDoubleStoredAsLong)
{
return(new MetricValueV2 {
ValueAsDouble = SerializationUtils.ReadInt64FromBase128(reader)
});
}
else
{
return(new MetricValueV2 {
ValueAsDouble = reader.ReadDouble()
});
}
}
else
{
return(new MetricValueV2 {
ValueAsULong = SerializationUtils.ReadUInt64FromBase128(reader)
});
}
}
private void ReadMetricsData(BinaryReader reader, IMetricBuilder <TMetadata> metricBuilder, ushort version, int maxMetricStringsLength, int maxMetricNamespaceStringsLength, int maxMetricDimensionValueStringsLength)
{
long packetTime = 0;
if (version >= 5)
{
packetTime = (long)SerializationUtils.ReadUInt64FromBase128(reader);
}
// Versions before 2 used variable number of bytes to write number of serialized metrics data.
// From version 2 passing IEnumerable<IReadOnlyMetric> is supported, thus number of metrics data
// is unknown beforehand and we cannot use variable number anymore. Thus fixed 4 bytes uint is used.
var count = version >= 2 ? reader.ReadUInt32() : SerializationUtils.ReadUInt32FromBase128(reader);
for (var i = 0; i < count; ++i)
{
metricBuilder.BeginMetricCreation();
var metadata = this.ReadMetricMetadataByIndex(reader);
metricBuilder.AssignMetadata(metadata);
// In versions 0-2 Monitoring Account and Metric Namespace was part of the Metric data
// From version 3 Monitoring Account is removed and Metric Namespace became a part of Metric Metadata
if (version < 3)
{
var monitoringAccount = this.ReadStringByIndex(reader);
if (monitoringAccount.Length > maxMetricStringsLength)
{
throw new MetricSerializationException($"Monitoring account string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricStringsLength}, Value:{monitoringAccount}.", null);
}
var metricNamespace = this.ReadStringByIndex(reader);
if (metricNamespace.Length > maxMetricNamespaceStringsLength)
{
throw new MetricSerializationException($"Namespace string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricNamespaceStringsLength}, Value:{metricNamespace}.", null);
}
metricBuilder.AssignMonitoringAccount(monitoringAccount);
metricBuilder.AssignNamespace(metricNamespace);
}
if (version == 0)
{
// Skip event id
this.ReadStringByIndex(reader);
}
if (version >= 5)
{
var timeInTicks = (packetTime - SerializationUtils.ReadInt64FromBase128(reader)) * SerializationUtils.OneMinuteInterval;
metricBuilder.AssignTimeUtc(new DateTime(timeInTicks, DateTimeKind.Utc));
}
else
{
metricBuilder.AssignTimeUtc(new DateTime((long)SerializationUtils.ReadUInt64FromBase128(reader), DateTimeKind.Utc));
}
for (var j = 0; j < metadata.DimensionsCount; ++j)
{
var dimensionValue = this.ReadStringByIndex(reader);
if (dimensionValue.Length > maxMetricDimensionValueStringsLength)
{
throw new MetricSerializationException($"Dimension value string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricDimensionValueStringsLength}, Value:{dimensionValue}.", null);
}
metricBuilder.AddDimensionValue(dimensionValue);
}
var samplingTypes = (SamplingTypes)SerializationUtils.ReadUInt32FromBase128(reader);
metricBuilder.AssignSamplingTypes(samplingTypes);
if ((samplingTypes & SamplingTypes.Min) != 0)
{
metricBuilder.AssignMin(SerializationUtils.ReadUInt64FromBase128(reader));
}
if ((samplingTypes & SamplingTypes.Max) != 0)
{
metricBuilder.AssignMax(SerializationUtils.ReadUInt64FromBase128(reader));
}
if ((samplingTypes & SamplingTypes.Sum) != 0)
{
metricBuilder.AssignSum(SerializationUtils.ReadUInt64FromBase128(reader));
}
if ((samplingTypes & SamplingTypes.Count) != 0)
{
metricBuilder.AssignCount(SerializationUtils.ReadUInt32FromBase128(reader));
}
if ((samplingTypes & SamplingTypes.SumOfSquareDiffFromMean) != 0)
{
var sumOfSquareDiffFromMean = reader.ReadDouble();
metricBuilder.AssignSumOfSquareDiffFromMean(sumOfSquareDiffFromMean);
}
if ((samplingTypes & SamplingTypes.Histogram) != 0)
{
metricBuilder.AssignHistogram(this.ReadHistogram(reader, version));
}
if ((samplingTypes & SamplingTypes.HyperLogLogSketch) != 0)
{
var sizeOfHyperLogLogSketches = reader.ReadInt32();
metricBuilder.AssignHyperLogLogSketch(reader, sizeOfHyperLogLogSketches);
}
metricBuilder.EndMetricCreation();
}
}
private void ReadMetricsData(
BinaryReader reader,
IFrontEndMetricBuilder <TMetadata> metricBuilder,
ushort version,
int maxMetricDimensionValueStringsLength)
{
long packetTime = 0;
if (version >= 5)
{
packetTime = (long)SerializationUtils.ReadUInt64FromBase128(reader);
}
Stream readerStream = reader.BaseStream;
var metricsCount = reader.ReadUInt32();
for (var i = 0; i < metricsCount; ++i)
{
DateTime timeUtc;
var count = 0U;
var sum = default(MetricValueV2);
var min = default(MetricValueV2);
var max = default(MetricValueV2);
double sumOfSquareDiffFromMean = 0;
var metadata = this.ReadMetricMetadataByIndex(reader);
if (version >= 5)
{
var timeInTicks = (packetTime - SerializationUtils.ReadInt64FromBase128(reader)) * SerializationUtils.OneMinuteInterval;
timeUtc = new DateTime(timeInTicks, DateTimeKind.Utc);
}
else
{
timeUtc = new DateTime((long)SerializationUtils.ReadUInt64FromBase128(reader), DateTimeKind.Utc);
}
for (var j = 0; j < metadata.DimensionsCount; ++j)
{
var dimensionValue = this.ReadStringByIndex(reader);
if (dimensionValue.Length > maxMetricDimensionValueStringsLength)
{
throw new MetricSerializationException($"Dimension value string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricDimensionValueStringsLength}, Value:{dimensionValue}.", null, true);
}
this.reusableStringsList.Add(dimensionValue);
}
var samplingTypes = (SamplingTypes)SerializationUtils.ReadUInt32FromBase128(reader);
var isDouble = (samplingTypes & SamplingTypes.DoubleValueType) != 0;
var isDoubleStoredAslong = (samplingTypes & SamplingTypes.DoubleValueStoredAsLongType) != 0;
if ((samplingTypes & SamplingTypes.Min) != 0)
{
min = this.ReadMetricValue(reader, isDouble, isDoubleStoredAslong);
}
if ((samplingTypes & SamplingTypes.Max) != 0)
{
max = this.ReadMetricValue(reader, isDouble, isDoubleStoredAslong);
}
if ((samplingTypes & SamplingTypes.Sum) != 0)
{
sum = this.ReadMetricValue(reader, isDouble, isDoubleStoredAslong);
}
if ((samplingTypes & SamplingTypes.Count) != 0)
{
count = SerializationUtils.ReadUInt32FromBase128(reader);
}
if ((samplingTypes & SamplingTypes.SumOfSquareDiffFromMean) != 0)
{
sumOfSquareDiffFromMean = reader.ReadDouble();
}
bool haveHistogram = (samplingTypes & SamplingTypes.Histogram) != 0;
metricBuilder.BeginMetricCreation(metadata, this.reusableStringsList, timeUtc, samplingTypes, count, sum, min, max, sumOfSquareDiffFromMean);
this.reusableStringsList.Clear();
if (haveHistogram)
{
IEnumerable <KeyValuePair <ulong, uint> > histogramBuckets =
SerializationUtils.ReadHistogram(reader, hasHistogramSizePrefix: version > 3);
this.histogramBuffer.Clear();
this.histogramBuffer.AddRange(histogramBuckets);
metricBuilder.AssignHistogram(this.histogramBuffer);
}
if ((samplingTypes & SamplingTypes.HyperLogLogSketch) != 0)
{
var sizeOfHyperLogLogSketches = reader.ReadInt32();
metricBuilder.AssignHyperLogLogSketch(reader, sizeOfHyperLogLogSketches);
}
if (version >= 6)
{
bool haveTDigest = (samplingTypes & SamplingTypes.TDigest) != 0;
bool readTDigest = false;
// starting from version 6, there is a list of TLV-type
// (https://en.wikipedia.org/wiki/Type-length-value) tuples in the rest of the serialized metric.
// deserialize all of them ignoring the unknown ones.
// list TLV values is expected to contain a single end-of-list marker in the end with T = 0x00
uint type;
while ((type = SerializationUtils.ReadUInt32FromBase128(reader)) != 0x00)
{
int length = (int)SerializationUtils.ReadUInt32FromBase128(reader);
long nextPos = readerStream.Position + length;
switch (type)
{
case TDigestPrefixValue: // 0x74 == 't' (tDigest)
if (haveTDigest)
{
if (!readTDigest)
{
metricBuilder.AssignTDigest(reader, length);
readTDigest = true;
}
else
{
// if we already saw tDigest value and see it
// second time it is a sign of a protocol error.
throw new MetricSerializationException("Saw 2 TDigest values for the same metric", null, isInvalidData: true);
}
}
else
{
// if TLV list contains tDigest but the sampling types does not
// it is a protocol error
throw new MetricSerializationException("Sampling types do not contain tDigest, but TLV list contains it", null, isInvalidData: true);
}
break;
default:
// ignore unknown types
break;
}
// always set the position to point to the next entry.
// do not trust the deserializer code to leave the position set correctly
// this helps prevent compatibility problems and makes the deserializer
// more stable
readerStream.Position = nextPos;
}
if (haveTDigest && !readTDigest)
{
// if sampling types contain TDigest but we have not seen it in TLV this is
// a sign of protocol error
throw new MetricSerializationException("Sampling types contain tDigest, but TLV list does not contain it", null, isInvalidData: true);
}
}
metricBuilder.EndMetricCreation();
}
}