private void WriteMetricMetadataIndex(BinaryWriter writer, IMetricMetadata value) { uint index; if (!this.metadataIndexes.TryGetValue(value, out index)) { index = this.nextMetadataIndex++; this.metadataIndexes.Add(value, index); this.metadatas.Add(value); if (this.estimatePacketSize) { // In versions 0-2 Metric Namespace was part of the Metric data, from version 3 it became a part of Metric Metadata if (this.version >= 3) { this.currentMetadataDictionaryBlockSize += SerializationUtils.EstimateUInt32InBase128Size(this.RegisterString(value.MetricNamespace)); } this.currentMetadataDictionaryBlockSize += SerializationUtils.EstimateUInt32InBase128Size(this.RegisterString(value.MetricName)); this.currentMetadataDictionaryBlockSize += SerializationUtils.EstimateUInt32InBase128Size((uint)value.DimensionsCount); for (int i = 0; i < value.DimensionsCount; ++i) { this.currentMetadataDictionaryBlockSize += SerializationUtils.EstimateUInt32InBase128Size(this.RegisterString(value.GetDimensionName(i))); } } } SerializationUtils.WriteUInt32AsBase128(writer, index); }
private void WriteMetricMetadata(BinaryWriter writer, IMetricMetadata value) { // In versions 0-2 Metric Namespace was part of the Metric data, from version 3 it became a part of Metric Metadata if (this.version >= 3) { SerializationUtils.WriteUInt32AsBase128(writer, this.RegisterString(value.MetricNamespace)); } SerializationUtils.WriteUInt32AsBase128(writer, this.RegisterString(value.MetricName)); SerializationUtils.WriteUInt32AsBase128(writer, (uint)value.DimensionsCount); for (var i = 0; i < value.DimensionsCount; ++i) { SerializationUtils.WriteUInt32AsBase128(writer, this.RegisterString(value.GetDimensionName(i))); } }
public static unsafe void Serialize(BinaryWriter writer, double *values, int count) { // Reserve one byte for future versioning. writer.Write((byte)1); SerializationUtils.WriteUInt32AsBase128(writer, (uint)count); if (count > 0) { BitBinaryWriter bitWriter = new BitBinaryWriter(writer); var previousState = new DoubleValueState(0, -1, -1); for (int i = 0; i < count; ++i) { DoubleValueState newState; WriteDouble(bitWriter, values[i], previousState, out newState); previousState = newState; } bitWriter.Flush(); } }
private void WriteMetricsData(BinaryWriter writer, IEnumerable <IReadOnlyMetric> metricData) { Stream writerStream = writer.BaseStream; long currentTimeInMinutes = 0; if (this.version >= 5) { var currentTimeInTicks = DateTime.UtcNow.Ticks; currentTimeInMinutes = (currentTimeInTicks - (currentTimeInTicks % SerializationUtils.OneMinuteInterval)) / SerializationUtils.OneMinuteInterval; SerializationUtils.WriteUInt64AsBase128(writer, (ulong)currentTimeInMinutes); } var metricsCountPosition = writer.BaseStream.Position; writer.Write((uint)0); uint metricsCount = 0; foreach (var data in metricData) { ++metricsCount; var metadata = data.MetricMetadata; this.WriteMetricMetadataIndex(writer, metadata); // In versions 0-2 Monitoring Account and Metric Namespace was part of the Metric data // From version 3 Monitoring Account is removed and Metric Namespace became a part of Metric Metadata if (this.version < 3) { SerializationUtils.WriteUInt32AsBase128(writer, this.RegisterString(data.MonitoringAccount)); SerializationUtils.WriteUInt32AsBase128(writer, this.RegisterString(data.MetricNamespace)); } // In version 0 we had EventId, which was always passed as empty string if (this.version == 0) { SerializationUtils.WriteUInt32AsBase128(writer, this.RegisterString(string.Empty)); } if (this.version >= 5) { var timeTicks = data.TimeUtc.Ticks; long timeInMinutes = (timeTicks - (timeTicks % SerializationUtils.OneMinuteInterval)) / SerializationUtils.OneMinuteInterval; SerializationUtils.WriteInt64AsBase128(writer, currentTimeInMinutes - timeInMinutes); } else { SerializationUtils.WriteUInt64AsBase128(writer, (ulong)data.TimeUtc.Ticks); } for (byte j = 0; j < data.MetricMetadata.DimensionsCount; ++j) { SerializationUtils.WriteUInt32AsBase128(writer, this.RegisterString(data.GetDimensionValue(j))); } var samplingTypes = data.SamplingTypes; bool useDouble = false; bool storeDoubleAsLong = false; if ((samplingTypes & SamplingTypes.DoubleValueType) != 0) { useDouble = true; if (data.SumUnion.CanRepresentDoubleAsLong() && ((samplingTypes & SamplingTypes.Min) == 0 || (data.MinUnion.CanRepresentDoubleAsLong() && data.MaxUnion.CanRepresentDoubleAsLong()))) { samplingTypes = samplingTypes | SamplingTypes.DoubleValueStoredAsLongType; storeDoubleAsLong = true; } } SerializationUtils.WriteUInt32AsBase128(writer, (uint)samplingTypes); if ((data.SamplingTypes & SamplingTypes.Min) != 0) { this.WriteMetricValue(data.MinUnion, useDouble, storeDoubleAsLong, writer); } if ((data.SamplingTypes & SamplingTypes.Max) != 0) { this.WriteMetricValue(data.MaxUnion, useDouble, storeDoubleAsLong, writer); } if ((data.SamplingTypes & SamplingTypes.Sum) != 0) { this.WriteMetricValue(data.SumUnion, useDouble, storeDoubleAsLong, writer); } if ((data.SamplingTypes & SamplingTypes.Count) != 0) { SerializationUtils.WriteUInt32AsBase128(writer, data.Count); } if ((data.SamplingTypes & SamplingTypes.SumOfSquareDiffFromMean) != 0) { writer.Write(data.SumOfSquareDiffFromMean); } if ((data.SamplingTypes & SamplingTypes.Histogram) != 0) { if (data.Histogram == null) { var message = string.Format( CultureInfo.InvariantCulture, "Invalid input data. Declared sampling type contains Histogram, but Histogram data is null. Metric:({0},{1},{2}).", data.MonitoringAccount, data.MetricNamespace, data.MetricMetadata.MetricName); throw new MetricSerializationException(message, null); } SerializationUtils.WriteHistogramDataHistogram(writer, data.Histogram.SamplesCount, data.Histogram.Samples, this.version > 3); } if ((data.SamplingTypes & SamplingTypes.HyperLogLogSketch) != 0) { if (data.HyperLogLogSketchesStream == null && data.HyperLogLogSketches == null) { var message = string.Format( CultureInfo.InvariantCulture, "Invalid input data. Declared sampling type contains sketches, but sketches data is null. Metric:({0},{1},{2}).", data.MonitoringAccount, data.MetricNamespace, data.MetricMetadata.MetricName); throw new MetricSerializationException(message, null); } if (data.HyperLogLogSketchesStream != null) { if (this.tempBuffer == null) { this.tempBuffer = new byte[TempBufferSize]; } writer.Write((int)data.HyperLogLogSketchesStream.Length); var sketchStreamStartPosition = data.HyperLogLogSketchesStream.Position; SerializationUtils.ReadFromStream(data.HyperLogLogSketchesStream, writer.BaseStream, (int)data.HyperLogLogSketchesStream.Length, this.tempBuffer); data.HyperLogLogSketchesStream.Position = sketchStreamStartPosition; } else { if (this.hllSerializationVersion == 0) { SerializationUtils.WriteHyperLogLogSketches(writer, data.HyperLogLogSketches.HyperLogLogSketchesCount, data.HyperLogLogSketches.HyperLogLogSketches); } else { SerializationUtils.WriteHyperLogLogSketchesV2(writer, data.HyperLogLogSketches.HyperLogLogSketchesCount, data.HyperLogLogSketches.HyperLogLogSketches); } } } if (this.version >= 6) { if ((data.SamplingTypes & samplingTypes & SamplingTypes.TDigest) != 0) { SerializationUtils.WriteUInt32AsBase128(writer, FrontEndMetricDeserializer <IMetricMetadata> .TDigestPrefixValue); long pos = writerStream.Position; // placeholder for length encoded as 4 bytes writer.Write((ushort)0); writer.Write((ushort)0); data.TDigest.Serialize(writer); long tdigestSerializedLength = writerStream.Position - pos - 4; if (tdigestSerializedLength > ushort.MaxValue) { throw new ArgumentException("TDigest too big"); } writerStream.Position = pos; SerializationUtils.WriteUInt32InBase128AsFixed4Bytes(writer, (ushort)tdigestSerializedLength); writerStream.Position += tdigestSerializedLength; } SerializationUtils.WriteUInt32AsBase128(writer, 0); } this.currentMetricDataBlockSize = writer.BaseStream.Position; } var currentPosition = writer.BaseStream.Position; writer.BaseStream.Position = metricsCountPosition; // Versions before 2 used variable number of bytes to write number of serialized metrics data. // From version 2 passing IEnumerable<IReadOnlyMetric> is supported, thus number of metrics data // is unknown beforehand and we cannot use variable number anymore. Thus we use fixed 4 bytes // uint. To keep compatibility with previous versions, while still have ability to serialize // variable amount of data, we write uint number in variable manner but with fixed number of bytes. if (this.version >= 2) { writer.Write(metricsCount); } else { SerializationUtils.WriteUInt32InBase128AsFixed4Bytes(writer, metricsCount); } writer.BaseStream.Position = currentPosition; }
/// <summary> /// Serializes counter (metric) data to the stream. /// </summary> /// <param name="stream">Stream to which data should be serialized. Stream should be writable and provide random access.</param> /// <param name="metricData">Collection of metric data to be serialized.</param> public void Serialize(Stream stream, IEnumerable <IReadOnlyMetric> metricData) { if (!stream.CanWrite || !stream.CanSeek) { throw new ArgumentException("Stream should be writable and provide random access.", nameof(stream)); } try { using (var writer = new NoCloseBinaryWriter(stream, Encoding.UTF8)) { var startStreamPosition = stream.Position; // Write version and type serializers info writer.Write(this.version); long crcOffSet = 0; long crcBodyOffSet = 0; if (this.version >= 5) { // Add CRC crcOffSet = stream.Position; writer.Write((uint)0); crcBodyOffSet = stream.Position; } writer.Write(TypeSerializerFlags); // Reserve place to write type serializers data sections offsets var offsetsPosition = stream.Position; stream.Position += 2 * sizeof(long); // Write metrics data this.WriteMetricsData(writer, metricData); // Write cached metrics metadata and offset var serializerDataPosition = stream.Position; stream.Position = offsetsPosition; writer.Write(serializerDataPosition - startStreamPosition); offsetsPosition = stream.Position; stream.Position = serializerDataPosition; SerializationUtils.WriteUInt32AsBase128(writer, (uint)this.metadatas.Count); this.metadatas.ForEach(m => this.WriteMetricMetadata(writer, m)); // Write cached strings and offset serializerDataPosition = stream.Position; stream.Position = offsetsPosition; writer.Write(serializerDataPosition - startStreamPosition); stream.Position = serializerDataPosition; SerializationUtils.WriteUInt32AsBase128(writer, (uint)this.strings.Count); this.strings.ForEach(writer.Write); var endOfStream = stream.Position; if (this.version >= 5) { stream.Position = crcBodyOffSet; var crc = Crc.ComputeCrc(0, stream, stream.Length - crcBodyOffSet); stream.Position = crcOffSet; writer.Write(crc); } stream.Position = endOfStream; } } catch (IOException ioException) { throw new MetricSerializationException("Failed to serialize data.", ioException); } finally { this.nextStringIndex = 0; this.stringIndexes.Clear(); this.strings.Clear(); this.nextMetadataIndex = 0; this.metadataIndexes.Clear(); this.metadatas.Clear(); this.currentMetricDataBlockSize = 0; this.currentMetadataDictionaryBlockSize = 0; this.currentStringDictionaryBlockSize = 0; } }