private MetricValueV2 ReadMetricValue(BinaryReader reader, bool isDouble, bool isDoubleStoredAsLong) { if (isDouble) { if (isDoubleStoredAsLong) { return(new MetricValueV2 { ValueAsDouble = SerializationUtils.ReadInt64FromBase128(reader) }); } else { return(new MetricValueV2 { ValueAsDouble = reader.ReadDouble() }); } } else { return(new MetricValueV2 { ValueAsULong = SerializationUtils.ReadUInt64FromBase128(reader) }); } }
private void ReadMetricsData(BinaryReader reader, IMetricBuilder <TMetadata> metricBuilder, ushort version, int maxMetricStringsLength, int maxMetricNamespaceStringsLength, int maxMetricDimensionValueStringsLength) { long packetTime = 0; if (version >= 5) { packetTime = (long)SerializationUtils.ReadUInt64FromBase128(reader); } // Versions before 2 used variable number of bytes to write number of serialized metrics data. // From version 2 passing IEnumerable<IReadOnlyMetric> is supported, thus number of metrics data // is unknown beforehand and we cannot use variable number anymore. Thus fixed 4 bytes uint is used. var count = version >= 2 ? reader.ReadUInt32() : SerializationUtils.ReadUInt32FromBase128(reader); for (var i = 0; i < count; ++i) { metricBuilder.BeginMetricCreation(); var metadata = this.ReadMetricMetadataByIndex(reader); metricBuilder.AssignMetadata(metadata); // In versions 0-2 Monitoring Account and Metric Namespace was part of the Metric data // From version 3 Monitoring Account is removed and Metric Namespace became a part of Metric Metadata if (version < 3) { var monitoringAccount = this.ReadStringByIndex(reader); if (monitoringAccount.Length > maxMetricStringsLength) { throw new MetricSerializationException($"Monitoring account string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricStringsLength}, Value:{monitoringAccount}.", null); } var metricNamespace = this.ReadStringByIndex(reader); if (metricNamespace.Length > maxMetricNamespaceStringsLength) { throw new MetricSerializationException($"Namespace string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricNamespaceStringsLength}, Value:{metricNamespace}.", null); } metricBuilder.AssignMonitoringAccount(monitoringAccount); metricBuilder.AssignNamespace(metricNamespace); } if (version == 0) { // Skip event id this.ReadStringByIndex(reader); } if (version >= 5) { var timeInTicks = (packetTime - SerializationUtils.ReadInt64FromBase128(reader)) * SerializationUtils.OneMinuteInterval; metricBuilder.AssignTimeUtc(new DateTime(timeInTicks, DateTimeKind.Utc)); } else { metricBuilder.AssignTimeUtc(new DateTime((long)SerializationUtils.ReadUInt64FromBase128(reader), DateTimeKind.Utc)); } for (var j = 0; j < metadata.DimensionsCount; ++j) { var dimensionValue = this.ReadStringByIndex(reader); if (dimensionValue.Length > maxMetricDimensionValueStringsLength) { throw new MetricSerializationException($"Dimension value string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricDimensionValueStringsLength}, Value:{dimensionValue}.", null); } metricBuilder.AddDimensionValue(dimensionValue); } var samplingTypes = (SamplingTypes)SerializationUtils.ReadUInt32FromBase128(reader); metricBuilder.AssignSamplingTypes(samplingTypes); if ((samplingTypes & SamplingTypes.Min) != 0) { metricBuilder.AssignMin(SerializationUtils.ReadUInt64FromBase128(reader)); } if ((samplingTypes & SamplingTypes.Max) != 0) { metricBuilder.AssignMax(SerializationUtils.ReadUInt64FromBase128(reader)); } if ((samplingTypes & SamplingTypes.Sum) != 0) { metricBuilder.AssignSum(SerializationUtils.ReadUInt64FromBase128(reader)); } if ((samplingTypes & SamplingTypes.Count) != 0) { metricBuilder.AssignCount(SerializationUtils.ReadUInt32FromBase128(reader)); } if ((samplingTypes & SamplingTypes.SumOfSquareDiffFromMean) != 0) { var sumOfSquareDiffFromMean = reader.ReadDouble(); metricBuilder.AssignSumOfSquareDiffFromMean(sumOfSquareDiffFromMean); } if ((samplingTypes & SamplingTypes.Histogram) != 0) { metricBuilder.AssignHistogram(this.ReadHistogram(reader, version)); } if ((samplingTypes & SamplingTypes.HyperLogLogSketch) != 0) { var sizeOfHyperLogLogSketches = reader.ReadInt32(); metricBuilder.AssignHyperLogLogSketch(reader, sizeOfHyperLogLogSketches); } metricBuilder.EndMetricCreation(); } }
private void ReadMetricsData( BinaryReader reader, IFrontEndMetricBuilder <TMetadata> metricBuilder, ushort version, int maxMetricDimensionValueStringsLength) { long packetTime = 0; if (version >= 5) { packetTime = (long)SerializationUtils.ReadUInt64FromBase128(reader); } Stream readerStream = reader.BaseStream; var metricsCount = reader.ReadUInt32(); for (var i = 0; i < metricsCount; ++i) { DateTime timeUtc; var count = 0U; var sum = default(MetricValueV2); var min = default(MetricValueV2); var max = default(MetricValueV2); double sumOfSquareDiffFromMean = 0; var metadata = this.ReadMetricMetadataByIndex(reader); if (version >= 5) { var timeInTicks = (packetTime - SerializationUtils.ReadInt64FromBase128(reader)) * SerializationUtils.OneMinuteInterval; timeUtc = new DateTime(timeInTicks, DateTimeKind.Utc); } else { timeUtc = new DateTime((long)SerializationUtils.ReadUInt64FromBase128(reader), DateTimeKind.Utc); } for (var j = 0; j < metadata.DimensionsCount; ++j) { var dimensionValue = this.ReadStringByIndex(reader); if (dimensionValue.Length > maxMetricDimensionValueStringsLength) { throw new MetricSerializationException($"Dimension value string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricDimensionValueStringsLength}, Value:{dimensionValue}.", null, true); } this.reusableStringsList.Add(dimensionValue); } var samplingTypes = (SamplingTypes)SerializationUtils.ReadUInt32FromBase128(reader); var isDouble = (samplingTypes & SamplingTypes.DoubleValueType) != 0; var isDoubleStoredAslong = (samplingTypes & SamplingTypes.DoubleValueStoredAsLongType) != 0; if ((samplingTypes & SamplingTypes.Min) != 0) { min = this.ReadMetricValue(reader, isDouble, isDoubleStoredAslong); } if ((samplingTypes & SamplingTypes.Max) != 0) { max = this.ReadMetricValue(reader, isDouble, isDoubleStoredAslong); } if ((samplingTypes & SamplingTypes.Sum) != 0) { sum = this.ReadMetricValue(reader, isDouble, isDoubleStoredAslong); } if ((samplingTypes & SamplingTypes.Count) != 0) { count = SerializationUtils.ReadUInt32FromBase128(reader); } if ((samplingTypes & SamplingTypes.SumOfSquareDiffFromMean) != 0) { sumOfSquareDiffFromMean = reader.ReadDouble(); } bool haveHistogram = (samplingTypes & SamplingTypes.Histogram) != 0; metricBuilder.BeginMetricCreation(metadata, this.reusableStringsList, timeUtc, samplingTypes, count, sum, min, max, sumOfSquareDiffFromMean); this.reusableStringsList.Clear(); if (haveHistogram) { IEnumerable <KeyValuePair <ulong, uint> > histogramBuckets = SerializationUtils.ReadHistogram(reader, hasHistogramSizePrefix: version > 3); this.histogramBuffer.Clear(); this.histogramBuffer.AddRange(histogramBuckets); metricBuilder.AssignHistogram(this.histogramBuffer); } if ((samplingTypes & SamplingTypes.HyperLogLogSketch) != 0) { var sizeOfHyperLogLogSketches = reader.ReadInt32(); metricBuilder.AssignHyperLogLogSketch(reader, sizeOfHyperLogLogSketches); } if (version >= 6) { bool haveTDigest = (samplingTypes & SamplingTypes.TDigest) != 0; bool readTDigest = false; // starting from version 6, there is a list of TLV-type // (https://en.wikipedia.org/wiki/Type-length-value) tuples in the rest of the serialized metric. // deserialize all of them ignoring the unknown ones. // list TLV values is expected to contain a single end-of-list marker in the end with T = 0x00 uint type; while ((type = SerializationUtils.ReadUInt32FromBase128(reader)) != 0x00) { int length = (int)SerializationUtils.ReadUInt32FromBase128(reader); long nextPos = readerStream.Position + length; switch (type) { case TDigestPrefixValue: // 0x74 == 't' (tDigest) if (haveTDigest) { if (!readTDigest) { metricBuilder.AssignTDigest(reader, length); readTDigest = true; } else { // if we already saw tDigest value and see it // second time it is a sign of a protocol error. throw new MetricSerializationException("Saw 2 TDigest values for the same metric", null, isInvalidData: true); } } else { // if TLV list contains tDigest but the sampling types does not // it is a protocol error throw new MetricSerializationException("Sampling types do not contain tDigest, but TLV list contains it", null, isInvalidData: true); } break; default: // ignore unknown types break; } // always set the position to point to the next entry. // do not trust the deserializer code to leave the position set correctly // this helps prevent compatibility problems and makes the deserializer // more stable readerStream.Position = nextPos; } if (haveTDigest && !readTDigest) { // if sampling types contain TDigest but we have not seen it in TLV this is // a sign of protocol error throw new MetricSerializationException("Sampling types contain tDigest, but TLV list does not contain it", null, isInvalidData: true); } } metricBuilder.EndMetricCreation(); } }