/// <summary> /// Deserializes counter (metric) data from the stream and adds all objects to provided collection. /// </summary> /// <param name="stream">Stream from which data should be deserialized. Stream should be readable and provide randon access.</param> /// <param name="metricBuilder">An object responsible for creation and further consumption of deserialized data.</param> /// <param name="maxMetricStringsLength">Maximum length of strings, which represent metric name, dimension names and values.</param> /// <param name="maxMetricNamespaceStringsLength">Maximum length of metric namespace string.</param> /// <param name="maxMetricDimensionValueStringsLength">Maximum length of metric dimension value string.</param> public void Deserialize( Stream stream, IFrontEndMetricBuilder <TMetadata> metricBuilder, int maxMetricStringsLength, int maxMetricNamespaceStringsLength, int maxMetricDimensionValueStringsLength) { if (!stream.CanRead || !stream.CanSeek) { throw new ArgumentException(@"Stream should be readable and provide random access.", nameof(stream)); } try { using (var reader = new BinaryReader(stream, Encoding.UTF8, true)) { var startStreamPosition = stream.Position; // Read version and type serializers info var version = reader.ReadUInt16(); if (version < MinVersion || version > MaxVersion) { throw new VersionNotSupportedMetricSerializationException( string.Format( CultureInfo.InvariantCulture, "Version is not supported. ReadVersion:{0}, MinVersion:{1}, MaxVersion:{2}.", version, MinVersion, MaxVersion)); } if (version >= 5) { // Read CRC. CRC check is done in upper layers. reader.ReadUInt32(); } if (reader.ReadUInt32() != TypeSerializerFlags) { throw new VersionNotSupportedMetricSerializationException("Type serializers not supported."); } metricBuilder.SetSerializationVersion(version); // Read strings var deserializerDataPosition = stream.Position; stream.Position += sizeof(long); stream.Position = startStreamPosition + reader.ReadInt64(); var count = SerializationUtils.ReadUInt32FromBase128(reader); for (uint i = 0; i < count; ++i) { this.stringsDictionary.Add(metricBuilder.GetString(reader.ReadString())); } var endOfPacketStreamPosition = stream.Position; // Read metrics metadata stream.Position = deserializerDataPosition; stream.Position = startStreamPosition + reader.ReadInt64(); count = SerializationUtils.ReadUInt32FromBase128(reader); for (uint i = 0; i < count; ++i) { this.metadataDictionary.Add(this.ReadMetricMetadata(reader, metricBuilder, maxMetricStringsLength, maxMetricNamespaceStringsLength)); } // Read metrics data stream.Position = deserializerDataPosition + (2 * sizeof(long)); this.ReadMetricsData(reader, metricBuilder, version, maxMetricDimensionValueStringsLength); // Bring back the stream to total read data position stream.Position = endOfPacketStreamPosition; } } catch (IOException ioException) { throw new MetricSerializationException("Failed to deserialize data. Problem with input stream.", ioException); } catch (Exception exception) { var serializationException = exception as MetricSerializationException; bool isInvalidData = false; if (serializationException != null) { isInvalidData = serializationException.IsInvalidData; } throw new MetricSerializationException("Failed to deserialize data. Likely the incoming stream contains corrupted data.", exception, isInvalidData); } finally { this.metadataDictionary.Clear(); this.stringsDictionary.Clear(); this.histogramBuffer.Clear(); this.reusableStringsList.Clear(); } }
private string ReadStringByIndex(BinaryReader reader) { var index = (int)SerializationUtils.ReadUInt32FromBase128(reader); return(this.stringsDictionary[index]); }
private TMetadata ReadMetricMetadataByIndex(BinaryReader reader) { var index = (int)SerializationUtils.ReadUInt32FromBase128(reader); return(this.metadataDictionary[index]); }
private void ReadMetricsData( BinaryReader reader, IFrontEndMetricBuilder <TMetadata> metricBuilder, ushort version, int maxMetricDimensionValueStringsLength) { long packetTime = 0; if (version >= 5) { packetTime = (long)SerializationUtils.ReadUInt64FromBase128(reader); } Stream readerStream = reader.BaseStream; var metricsCount = reader.ReadUInt32(); for (var i = 0; i < metricsCount; ++i) { DateTime timeUtc; var count = 0U; var sum = default(MetricValueV2); var min = default(MetricValueV2); var max = default(MetricValueV2); double sumOfSquareDiffFromMean = 0; var metadata = this.ReadMetricMetadataByIndex(reader); if (version >= 5) { var timeInTicks = (packetTime - SerializationUtils.ReadInt64FromBase128(reader)) * SerializationUtils.OneMinuteInterval; timeUtc = new DateTime(timeInTicks, DateTimeKind.Utc); } else { timeUtc = new DateTime((long)SerializationUtils.ReadUInt64FromBase128(reader), DateTimeKind.Utc); } for (var j = 0; j < metadata.DimensionsCount; ++j) { var dimensionValue = this.ReadStringByIndex(reader); if (dimensionValue.Length > maxMetricDimensionValueStringsLength) { throw new MetricSerializationException($"Dimension value string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricDimensionValueStringsLength}, Value:{dimensionValue}.", null, true); } this.reusableStringsList.Add(dimensionValue); } var samplingTypes = (SamplingTypes)SerializationUtils.ReadUInt32FromBase128(reader); var isDouble = (samplingTypes & SamplingTypes.DoubleValueType) != 0; var isDoubleStoredAslong = (samplingTypes & SamplingTypes.DoubleValueStoredAsLongType) != 0; if ((samplingTypes & SamplingTypes.Min) != 0) { min = this.ReadMetricValue(reader, isDouble, isDoubleStoredAslong); } if ((samplingTypes & SamplingTypes.Max) != 0) { max = this.ReadMetricValue(reader, isDouble, isDoubleStoredAslong); } if ((samplingTypes & SamplingTypes.Sum) != 0) { sum = this.ReadMetricValue(reader, isDouble, isDoubleStoredAslong); } if ((samplingTypes & SamplingTypes.Count) != 0) { count = SerializationUtils.ReadUInt32FromBase128(reader); } if ((samplingTypes & SamplingTypes.SumOfSquareDiffFromMean) != 0) { sumOfSquareDiffFromMean = reader.ReadDouble(); } bool haveHistogram = (samplingTypes & SamplingTypes.Histogram) != 0; metricBuilder.BeginMetricCreation(metadata, this.reusableStringsList, timeUtc, samplingTypes, count, sum, min, max, sumOfSquareDiffFromMean); this.reusableStringsList.Clear(); if (haveHistogram) { IEnumerable <KeyValuePair <ulong, uint> > histogramBuckets = SerializationUtils.ReadHistogram(reader, hasHistogramSizePrefix: version > 3); this.histogramBuffer.Clear(); this.histogramBuffer.AddRange(histogramBuckets); metricBuilder.AssignHistogram(this.histogramBuffer); } if ((samplingTypes & SamplingTypes.HyperLogLogSketch) != 0) { var sizeOfHyperLogLogSketches = reader.ReadInt32(); metricBuilder.AssignHyperLogLogSketch(reader, sizeOfHyperLogLogSketches); } if (version >= 6) { bool haveTDigest = (samplingTypes & SamplingTypes.TDigest) != 0; bool readTDigest = false; // starting from version 6, there is a list of TLV-type // (https://en.wikipedia.org/wiki/Type-length-value) tuples in the rest of the serialized metric. // deserialize all of them ignoring the unknown ones. // list TLV values is expected to contain a single end-of-list marker in the end with T = 0x00 uint type; while ((type = SerializationUtils.ReadUInt32FromBase128(reader)) != 0x00) { int length = (int)SerializationUtils.ReadUInt32FromBase128(reader); long nextPos = readerStream.Position + length; switch (type) { case TDigestPrefixValue: // 0x74 == 't' (tDigest) if (haveTDigest) { if (!readTDigest) { metricBuilder.AssignTDigest(reader, length); readTDigest = true; } else { // if we already saw tDigest value and see it // second time it is a sign of a protocol error. throw new MetricSerializationException("Saw 2 TDigest values for the same metric", null, isInvalidData: true); } } else { // if TLV list contains tDigest but the sampling types does not // it is a protocol error throw new MetricSerializationException("Sampling types do not contain tDigest, but TLV list contains it", null, isInvalidData: true); } break; default: // ignore unknown types break; } // always set the position to point to the next entry. // do not trust the deserializer code to leave the position set correctly // this helps prevent compatibility problems and makes the deserializer // more stable readerStream.Position = nextPos; } if (haveTDigest && !readTDigest) { // if sampling types contain TDigest but we have not seen it in TLV this is // a sign of protocol error throw new MetricSerializationException("Sampling types contain tDigest, but TLV list does not contain it", null, isInvalidData: true); } } metricBuilder.EndMetricCreation(); } }