/// <summary> /// Deserializes to an array of nullable <see lang="double"/> from the specified reader. /// </summary> /// <param name="reader">The reader.</param> /// <returns>An array of nullable <see lang="double"/>.</returns> public static double?[] DeserializeToNullableDoubles(BinaryReader reader) { // version not in use yet. reader.ReadByte(); var numOfItems = (int)SerializationUtils.ReadUInt32FromBase128(reader); if (numOfItems == 0) { return(EmptyNullableDoubleArray); } var result = new double?[numOfItems]; var bitBinaryReader = new BitBinaryReader(reader); var previousState = new DoubleValueState(0, -1, -1); for (int i = 0; i < numOfItems; ++i) { var newState = ReadDouble(bitBinaryReader, previousState); result[i] = double.IsNaN(newState.Value) ? (double?)null : newState.Value; previousState = newState; } return(result); }
public static unsafe void Deserialize(BinaryReader reader, double *values, int expectedCount) { // version not in use yet. reader.ReadByte(); var count = (int)SerializationUtils.ReadUInt32FromBase128(reader); if (count != expectedCount) { throw new InvalidDataException($"Wrong count in serialized data: expected {expectedCount}, but was {count}"); } DeserializeValues(reader, values, count); }
public static double[] Deserialize(BinaryReader reader) { // version not in use yet. reader.ReadByte(); var numOfItems = (int)SerializationUtils.ReadUInt32FromBase128(reader); if (numOfItems == 0) { return(EmptyDoubleArray); } var result = new double[numOfItems]; unsafe { fixed(double *p = &result[0]) { DeserializeValues(reader, p, numOfItems); } } return(result); }
private TMetadata ReadMetricMetadata(BinaryReader reader, IMetricBuilder <TMetadata> metricBuilder, ushort version, int maxMetricStringsLength, int maxMetricNamespaceStringsLength) { var metricNamespace = string.Empty; // In versions 0-2 Metric Namespace was part of the Metric data, from version 3 it became a part of Metric Metadata if (version >= 3) { metricNamespace = this.ReadStringByIndex(reader); if (metricNamespace.Length > maxMetricNamespaceStringsLength) { throw new MetricSerializationException($"Namespace string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricNamespaceStringsLength}, Value:{metricNamespace}.", null); } } var metricName = this.ReadStringByIndex(reader); if (metricName.Length > maxMetricStringsLength) { throw new MetricSerializationException($"Metric name string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricStringsLength}, Value:{metricName}.", null); } var count = SerializationUtils.ReadUInt32FromBase128(reader); var dimensionNames = new List <string>((int)count); for (var i = 0; i < count; ++i) { var dimensionName = this.ReadStringByIndex(reader); if (dimensionName.Length > maxMetricStringsLength) { throw new MetricSerializationException($"Dimension name string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricStringsLength}, Value:{dimensionName}.", null); } dimensionNames.Add(dimensionName); } return(metricBuilder.CreateMetadata(metricNamespace, metricName, dimensionNames)); }
private TMetadata ReadMetricMetadata(BinaryReader reader, IFrontEndMetricBuilder <TMetadata> metricBuilder, int maxMetricStringsLength, int maxMetricNamespaceStringsLength) { var metricNamespace = this.ReadStringByIndex(reader); if (metricNamespace.Length > maxMetricNamespaceStringsLength) { throw new MetricSerializationException( $"Namespace string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricNamespaceStringsLength}, Value:{metricNamespace}.", null, true); } var metricName = this.ReadStringByIndex(reader); if (metricName.Length > maxMetricStringsLength) { throw new MetricSerializationException( $"Metric name string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricStringsLength}, Value:{metricName}.", null, true); } var count = SerializationUtils.ReadUInt32FromBase128(reader); for (var i = 0; i < count; ++i) { var dimensionName = this.ReadStringByIndex(reader); if (dimensionName.Length > maxMetricStringsLength) { throw new MetricSerializationException( $"Dimension name string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricStringsLength}, Value:{dimensionName}.", null, true); } this.reusableStringsList.Add(dimensionName); } var result = metricBuilder.CreateMetadata(metricNamespace, metricName, this.reusableStringsList); this.reusableStringsList.Clear(); return(result); }
/// <summary> /// Deserializes counter (metric) data from the stream and adds all objects to provided collection. /// </summary> /// <param name="stream">Stream from which data should be deserialized. Stream should be readable and provide randon access.</param> /// <param name="metricBuilder">An object responsible for creation and further consumption of deserialized data.</param> /// <param name="maxMetricStringsLength">Maximum length of strings, which represent metric name, dimension names and values.</param> /// <param name="maxMetricNamespaceStringsLength">Maximum length of metric namespace string.</param> /// <param name="maxMetricDimensionValueStringsLength">Maximum length of metric dimension value string.</param> public void Deserialize(Stream stream, IMetricBuilder <TMetadata> metricBuilder, int maxMetricStringsLength, int maxMetricNamespaceStringsLength, int maxMetricDimensionValueStringsLength) { if (!stream.CanRead || !stream.CanSeek) { throw new ArgumentException(@"Stream should be readable and provide random access.", nameof(stream)); } try { using (var reader = new NoCloseBinaryReader(stream, Encoding.UTF8)) { var startStreamPosition = stream.Position; // Read version and type serializers info var version = reader.ReadUInt16(); if (version > MaxVersion) { throw new VersionNotSupportedMetricSerializationException( string.Format( CultureInfo.InvariantCulture, "Version is not supported. Read version:{0}, Max version:{1}.", version, MaxVersion)); } if (version >= 5) { // Read CRC. CRC check is done in upper layers. reader.ReadUInt32(); } if (reader.ReadUInt32() != TypeSerializerFlags) { throw new VersionNotSupportedMetricSerializationException("Type serializers not supported."); } metricBuilder.SetSerializationVersion(version); // Read strings var deserializerDataPosition = stream.Position; stream.Position += sizeof(long); stream.Position = startStreamPosition + reader.ReadInt64(); var count = SerializationUtils.ReadUInt32FromBase128(reader); for (uint i = 0; i < count; ++i) { this.stringsDictionary.Add(reader.ReadString()); } // Read metrics metadata stream.Position = deserializerDataPosition; stream.Position = startStreamPosition + reader.ReadInt64(); count = SerializationUtils.ReadUInt32FromBase128(reader); for (uint i = 0; i < count; ++i) { this.metadataDictionary.Add(this.ReadMetricMetadata(reader, metricBuilder, version, maxMetricStringsLength, maxMetricNamespaceStringsLength)); } // Read metrics data stream.Position = deserializerDataPosition + (2 * sizeof(long)); this.ReadMetricsData(reader, metricBuilder, version, maxMetricStringsLength, maxMetricNamespaceStringsLength, maxMetricDimensionValueStringsLength); } } catch (IOException ioException) { throw new MetricSerializationException("Failed to deserialize data. Problem with input stream.", ioException); } catch (Exception exception) { throw new MetricSerializationException("Failed to deserialize data. Likely the incoming stream contains corrupted data.", exception); } finally { this.metadataDictionary.Clear(); this.stringsDictionary.Clear(); this.histogramBuffer.Clear(); } }
private TMetadata ReadMetricMetadataByIndex(BinaryReader reader) { var index = (int)SerializationUtils.ReadUInt32FromBase128(reader); return(this.metadataDictionary[index]); }
private string ReadStringByIndex(BinaryReader reader) { var index = (int)SerializationUtils.ReadUInt32FromBase128(reader); return(this.stringsDictionary[index]); }
private void ReadMetricsData(BinaryReader reader, IMetricBuilder <TMetadata> metricBuilder, ushort version, int maxMetricStringsLength, int maxMetricNamespaceStringsLength, int maxMetricDimensionValueStringsLength) { long packetTime = 0; if (version >= 5) { packetTime = (long)SerializationUtils.ReadUInt64FromBase128(reader); } // Versions before 2 used variable number of bytes to write number of serialized metrics data. // From version 2 passing IEnumerable<IReadOnlyMetric> is supported, thus number of metrics data // is unknown beforehand and we cannot use variable number anymore. Thus fixed 4 bytes uint is used. var count = version >= 2 ? reader.ReadUInt32() : SerializationUtils.ReadUInt32FromBase128(reader); for (var i = 0; i < count; ++i) { metricBuilder.BeginMetricCreation(); var metadata = this.ReadMetricMetadataByIndex(reader); metricBuilder.AssignMetadata(metadata); // In versions 0-2 Monitoring Account and Metric Namespace was part of the Metric data // From version 3 Monitoring Account is removed and Metric Namespace became a part of Metric Metadata if (version < 3) { var monitoringAccount = this.ReadStringByIndex(reader); if (monitoringAccount.Length > maxMetricStringsLength) { throw new MetricSerializationException($"Monitoring account string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricStringsLength}, Value:{monitoringAccount}.", null); } var metricNamespace = this.ReadStringByIndex(reader); if (metricNamespace.Length > maxMetricNamespaceStringsLength) { throw new MetricSerializationException($"Namespace string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricNamespaceStringsLength}, Value:{metricNamespace}.", null); } metricBuilder.AssignMonitoringAccount(monitoringAccount); metricBuilder.AssignNamespace(metricNamespace); } if (version == 0) { // Skip event id this.ReadStringByIndex(reader); } if (version >= 5) { var timeInTicks = (packetTime - SerializationUtils.ReadInt64FromBase128(reader)) * SerializationUtils.OneMinuteInterval; metricBuilder.AssignTimeUtc(new DateTime(timeInTicks, DateTimeKind.Utc)); } else { metricBuilder.AssignTimeUtc(new DateTime((long)SerializationUtils.ReadUInt64FromBase128(reader), DateTimeKind.Utc)); } for (var j = 0; j < metadata.DimensionsCount; ++j) { var dimensionValue = this.ReadStringByIndex(reader); if (dimensionValue.Length > maxMetricDimensionValueStringsLength) { throw new MetricSerializationException($"Dimension value string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricDimensionValueStringsLength}, Value:{dimensionValue}.", null); } metricBuilder.AddDimensionValue(dimensionValue); } var samplingTypes = (SamplingTypes)SerializationUtils.ReadUInt32FromBase128(reader); metricBuilder.AssignSamplingTypes(samplingTypes); if ((samplingTypes & SamplingTypes.Min) != 0) { metricBuilder.AssignMin(SerializationUtils.ReadUInt64FromBase128(reader)); } if ((samplingTypes & SamplingTypes.Max) != 0) { metricBuilder.AssignMax(SerializationUtils.ReadUInt64FromBase128(reader)); } if ((samplingTypes & SamplingTypes.Sum) != 0) { metricBuilder.AssignSum(SerializationUtils.ReadUInt64FromBase128(reader)); } if ((samplingTypes & SamplingTypes.Count) != 0) { metricBuilder.AssignCount(SerializationUtils.ReadUInt32FromBase128(reader)); } if ((samplingTypes & SamplingTypes.SumOfSquareDiffFromMean) != 0) { var sumOfSquareDiffFromMean = reader.ReadDouble(); metricBuilder.AssignSumOfSquareDiffFromMean(sumOfSquareDiffFromMean); } if ((samplingTypes & SamplingTypes.Histogram) != 0) { metricBuilder.AssignHistogram(this.ReadHistogram(reader, version)); } if ((samplingTypes & SamplingTypes.HyperLogLogSketch) != 0) { var sizeOfHyperLogLogSketches = reader.ReadInt32(); metricBuilder.AssignHyperLogLogSketch(reader, sizeOfHyperLogLogSketches); } metricBuilder.EndMetricCreation(); } }
private void ReadMetricsData( BinaryReader reader, IFrontEndMetricBuilder <TMetadata> metricBuilder, ushort version, int maxMetricDimensionValueStringsLength) { long packetTime = 0; if (version >= 5) { packetTime = (long)SerializationUtils.ReadUInt64FromBase128(reader); } Stream readerStream = reader.BaseStream; var metricsCount = reader.ReadUInt32(); for (var i = 0; i < metricsCount; ++i) { DateTime timeUtc; var count = 0U; var sum = default(MetricValueV2); var min = default(MetricValueV2); var max = default(MetricValueV2); double sumOfSquareDiffFromMean = 0; var metadata = this.ReadMetricMetadataByIndex(reader); if (version >= 5) { var timeInTicks = (packetTime - SerializationUtils.ReadInt64FromBase128(reader)) * SerializationUtils.OneMinuteInterval; timeUtc = new DateTime(timeInTicks, DateTimeKind.Utc); } else { timeUtc = new DateTime((long)SerializationUtils.ReadUInt64FromBase128(reader), DateTimeKind.Utc); } for (var j = 0; j < metadata.DimensionsCount; ++j) { var dimensionValue = this.ReadStringByIndex(reader); if (dimensionValue.Length > maxMetricDimensionValueStringsLength) { throw new MetricSerializationException($"Dimension value string in the packet exceeds preconfigured length. Packet is corrupted. MaxLength:{maxMetricDimensionValueStringsLength}, Value:{dimensionValue}.", null, true); } this.reusableStringsList.Add(dimensionValue); } var samplingTypes = (SamplingTypes)SerializationUtils.ReadUInt32FromBase128(reader); var isDouble = (samplingTypes & SamplingTypes.DoubleValueType) != 0; var isDoubleStoredAslong = (samplingTypes & SamplingTypes.DoubleValueStoredAsLongType) != 0; if ((samplingTypes & SamplingTypes.Min) != 0) { min = this.ReadMetricValue(reader, isDouble, isDoubleStoredAslong); } if ((samplingTypes & SamplingTypes.Max) != 0) { max = this.ReadMetricValue(reader, isDouble, isDoubleStoredAslong); } if ((samplingTypes & SamplingTypes.Sum) != 0) { sum = this.ReadMetricValue(reader, isDouble, isDoubleStoredAslong); } if ((samplingTypes & SamplingTypes.Count) != 0) { count = SerializationUtils.ReadUInt32FromBase128(reader); } if ((samplingTypes & SamplingTypes.SumOfSquareDiffFromMean) != 0) { sumOfSquareDiffFromMean = reader.ReadDouble(); } bool haveHistogram = (samplingTypes & SamplingTypes.Histogram) != 0; metricBuilder.BeginMetricCreation(metadata, this.reusableStringsList, timeUtc, samplingTypes, count, sum, min, max, sumOfSquareDiffFromMean); this.reusableStringsList.Clear(); if (haveHistogram) { IEnumerable <KeyValuePair <ulong, uint> > histogramBuckets = SerializationUtils.ReadHistogram(reader, hasHistogramSizePrefix: version > 3); this.histogramBuffer.Clear(); this.histogramBuffer.AddRange(histogramBuckets); metricBuilder.AssignHistogram(this.histogramBuffer); } if ((samplingTypes & SamplingTypes.HyperLogLogSketch) != 0) { var sizeOfHyperLogLogSketches = reader.ReadInt32(); metricBuilder.AssignHyperLogLogSketch(reader, sizeOfHyperLogLogSketches); } if (version >= 6) { bool haveTDigest = (samplingTypes & SamplingTypes.TDigest) != 0; bool readTDigest = false; // starting from version 6, there is a list of TLV-type // (https://en.wikipedia.org/wiki/Type-length-value) tuples in the rest of the serialized metric. // deserialize all of them ignoring the unknown ones. // list TLV values is expected to contain a single end-of-list marker in the end with T = 0x00 uint type; while ((type = SerializationUtils.ReadUInt32FromBase128(reader)) != 0x00) { int length = (int)SerializationUtils.ReadUInt32FromBase128(reader); long nextPos = readerStream.Position + length; switch (type) { case TDigestPrefixValue: // 0x74 == 't' (tDigest) if (haveTDigest) { if (!readTDigest) { metricBuilder.AssignTDigest(reader, length); readTDigest = true; } else { // if we already saw tDigest value and see it // second time it is a sign of a protocol error. throw new MetricSerializationException("Saw 2 TDigest values for the same metric", null, isInvalidData: true); } } else { // if TLV list contains tDigest but the sampling types does not // it is a protocol error throw new MetricSerializationException("Sampling types do not contain tDigest, but TLV list contains it", null, isInvalidData: true); } break; default: // ignore unknown types break; } // always set the position to point to the next entry. // do not trust the deserializer code to leave the position set correctly // this helps prevent compatibility problems and makes the deserializer // more stable readerStream.Position = nextPos; } if (haveTDigest && !readTDigest) { // if sampling types contain TDigest but we have not seen it in TLV this is // a sign of protocol error throw new MetricSerializationException("Sampling types contain tDigest, but TLV list does not contain it", null, isInvalidData: true); } } metricBuilder.EndMetricCreation(); } }