/// <summary> /// Add the contents of another histogram to this one. /// </summary> /// <param name="fromHistogram">The other histogram.</param> /// <exception cref="System.IndexOutOfRangeException">if values in fromHistogram's are higher than highestTrackableValue.</exception> public virtual void Add(HistogramBase fromHistogram) { if (HighestTrackableValue < fromHistogram.HighestTrackableValue) { throw new ArgumentOutOfRangeException(nameof(fromHistogram), $"The other histogram covers a wider range ({fromHistogram.HighestTrackableValue} than this one ({HighestTrackableValue})."); } if ((BucketCount == fromHistogram.BucketCount) && (SubBucketCount == fromHistogram.SubBucketCount) && (_unitMagnitude == fromHistogram._unitMagnitude)) { // Counts arrays are of the same length and meaning, so we can just iterate and add directly: for (var i = 0; i < fromHistogram.CountsArrayLength; i++) { AddToCountAtIndex(i, fromHistogram.GetCountAtIndex(i)); } } else { // Arrays are not a direct match, so we can't just stream through and add them. // Instead, go through the array and add each non-zero value found at it's proper value: for (var i = 0; i < fromHistogram.CountsArrayLength; i++) { var count = fromHistogram.GetCountAtIndex(i); RecordValueWithCount(fromHistogram.ValueFromIndex(i), count); } } }
/// <summary> /// Copy this histogram, corrected for coordinated omission, into the target histogram, overwriting it's contents. /// </summary> /// <param name="targetHistogram">the histogram to copy into</param> /// <param name="expectedIntervalBetweenValueSamples">If <paramref name="expectedIntervalBetweenValueSamples"/> is larger than 0, add auto-generated value records as appropriate if value is larger than <paramref name="expectedIntervalBetweenValueSamples"/></param> /// <remarks> /// See <see cref="CopyCorrectedForCoordinatedOmission"/> for more detailed explanation about how correction is applied /// </remarks> public void CopyIntoCorrectedForCoordinatedOmission(HistogramBase targetHistogram, long expectedIntervalBetweenValueSamples) { targetHistogram.Reset(); targetHistogram.AddWhileCorrectingForCoordinatedOmission(this, expectedIntervalBetweenValueSamples); targetHistogram.StartTimeStamp = StartTimeStamp; targetHistogram.EndTimeStamp = EndTimeStamp; }
/// <summary> /// Add the contents of another histogram to this one. /// </summary> /// <param name="fromHistogram">The other histogram.</param> /// <exception cref="System.IndexOutOfRangeException">if values in fromHistogram's are higher than highestTrackableValue.</exception> public override void Add(HistogramBase fromHistogram) { //TODO: Review the use of locks on public instances. It seems that getting a copy in a thread safe manner, then adding that to this is better -LC // Synchronize add(). Avoid deadlocks by synchronizing in order of construction identity count. if (Identity < fromHistogram.Identity) { lock (UpdateLock) { lock (fromHistogram) { base.Add(fromHistogram); } } } else { lock (fromHistogram) { lock (UpdateLock) { base.Add(fromHistogram); } } } }
/// <summary> /// Copy this histogram into the target histogram, overwriting it's contents. /// </summary> /// <param name="source">The source histogram</param> /// <param name="targetHistogram">the histogram to copy into</param> public static void CopyInto(this HistogramBase source, HistogramBase targetHistogram) { targetHistogram.Reset(); targetHistogram.Add(source); targetHistogram.StartTimeStamp = source.StartTimeStamp; targetHistogram.EndTimeStamp = source.EndTimeStamp; }
/// <summary> /// Produce textual representation of the value distribution of histogram data by percentile. /// The distribution is output with exponentially increasing resolution, with each exponentially decreasing /// half-distance containing <paramref name="percentileTicksPerHalfDistance"/> percentile reporting tick points. /// </summary> /// <param name="histogram">The histogram to operate on</param> /// <param name="writer">The <see cref="TextWriter"/> into which the distribution will be output</param> /// <param name="percentileTicksPerHalfDistance"> /// The number of reporting points per exponentially decreasing half-distance /// </param> /// <param name="outputValueUnitScalingRatio"> /// The scaling factor by which to divide histogram recorded values units in output. /// Use the <see cref="OutputScalingFactor"/> constant values to help choose an appropriate output measurement. /// </param> /// <param name="useCsvFormat">Output in CSV (Comma Separated Values) format if <c>true</c>, else use plain text form.</param> public static void OutputPercentileDistribution(this HistogramBase histogram, TextWriter writer, int percentileTicksPerHalfDistance = 5, double outputValueUnitScalingRatio = OutputScalingFactor.None, bool useCsvFormat = false) { var formatter = useCsvFormat ? (IOutputFormatter) new CsvOutputFormatter(writer, histogram.NumberOfSignificantValueDigits, outputValueUnitScalingRatio) : (IOutputFormatter) new HgrmOutputFormatter(writer, histogram.NumberOfSignificantValueDigits, outputValueUnitScalingRatio); try { formatter.WriteHeader(); foreach (var iterationValue in histogram.Percentiles(percentileTicksPerHalfDistance)) { formatter.WriteValue(iterationValue); } formatter.WriteFooter(histogram); } catch (ArgumentOutOfRangeException) { // Overflow conditions on histograms can lead to ArgumentOutOfRangeException on iterations: if (histogram.HasOverflowed()) { writer.Write("# Histogram counts indicate OVERFLOW values"); } else { // Re-throw if reason is not a known overflow: throw; } } }
/// <summary> /// Copy this histogram into the target histogram, overwriting it's contents. /// </summary> /// <param name="source">The source histogram</param> /// <param name="targetHistogram">the histogram to copy into</param> public static void CopyInto(this HistogramBase source, HistogramBase targetHistogram) { targetHistogram.Reset(); targetHistogram.Add(source); targetHistogram.StartTimeStamp = source.StartTimeStamp; targetHistogram.EndTimeStamp = source.EndTimeStamp; }
/// <summary> /// Place a copy of the value counts accumulated since accumulated (since the last interval histogram was taken) into <paramref name="targetHistogram"/>. /// This will overwrite the existing data in <paramref name="targetHistogram"/>. /// Calling <see cref="GetIntervalHistogramInto(HistogramBase)"/> will reset the value counts, and start accumulating value counts for the next interval. /// </summary> /// <param name="targetHistogram">The histogram into which the interval histogram's data should be copied.</param> public void GetIntervalHistogramInto(HistogramBase targetHistogram) { lock (_gate) { PerformIntervalSample(); _inactiveHistogram.CopyInto(targetHistogram); } }
private void ValidateFitAsReplacementHistogram(HistogramBase replacementHistogram) { if (replacementHistogram != null && replacementHistogram.InstanceId != _activeHistogram.InstanceId) { throw new InvalidOperationException( $"Replacement histogram must have been obtained via a previous getIntervalHistogram() call from this {GetType().Name} instance"); } }
private static HistogramBase CompressedEncodeDecode(HistogramBase source) { var targetBuffer = ByteBuffer.Allocate(source.GetNeededByteBufferCapacity()); source.EncodeIntoCompressedByteBuffer(targetBuffer); targetBuffer.Position = 0; return(HistogramEncoding.DecodeFromCompressedByteBuffer(targetBuffer, 0)); }
/// <summary> /// Executes the action and records the time to complete the action. /// The time is recorded in ticks. /// Note this is a convenience method and can carry a cost. /// If the <paramref name="action"/> delegate is not cached, then it may incur an allocation cost for each invocation of <see cref="Record"/> /// </summary> /// <param name="histogram">The Histogram to record the latency in.</param> /// <param name="action">The functionality to execute and measure</param> /// <remarks> /// <para> /// Ticks are used as the unit of recording here as they are the smallest unit that .NET can measure /// and require no conversion at time of recording. Instead conversion (scaling) can be done at time /// of output to microseconds, milliseconds, seconds or other appropriate unit. /// </para> /// <para> /// If you are able to cache the <paramref name="action"/> delegate, then doing so is encouraged. /// <example> /// Here are two examples. /// The first does not cache the delegate /// /// <code> /// for (long i = 0; i < loopCount; i++) /// { /// histogram.Record(IncrementNumber); /// } /// </code> /// This second example does cache the delegate /// <code> /// Action incrementNumber = IncrementNumber; /// for (long i = 0; i < loopCount; i++) /// { /// histogram.Record(incrementNumber); /// } /// </code> /// In the second example, we will not be making allocations each time i.e. an allocation of an <seealso cref="Action"/> from <code>IncrementNumber</code>. /// This will reduce memory pressure and therefore garbage collection penalties. /// For performance sensitive applications, this method may not be suitable. /// As always, you are encouraged to test and measure the impact for your scenario. /// </example> /// </para> /// </remarks> public static void Record(this HistogramBase histogram, Action action) { var start = Stopwatch.GetTimestamp(); action(); var elapsedTicks = (Stopwatch.GetTimestamp() - start); histogram.RecordValue(elapsedTicks); }
/// <summary> /// Add the contents of another histogram to this one, while correcting the incoming data for coordinated omission. /// </summary> /// <param name="fromHistogram">The other histogram. highestTrackableValue and largestValueWithSingleUnitResolution must match.</param> /// <param name="expectedIntervalBetweenValueSamples">If <paramref name="expectedIntervalBetweenValueSamples"/> is larger than 0, add auto-generated value records as appropriate if value is larger than <paramref name="expectedIntervalBetweenValueSamples"/></param> /// <remarks> /// To compensate for the loss of sampled values when a recorded value is larger than the expected interval between value samples, the values added will include an auto-generated additional series of decreasingly-smaller(down to the expectedIntervalBetweenValueSamples) value records for each count found in the current histogram that is larger than the expectedIntervalBetweenValueSamples. /// /// Note: This is a post-recording correction method, as opposed to the at-recording correction method provided by {@link #recordValueWithExpectedInterval(long, long) recordValueWithExpectedInterval}. /// The two methods are mutually exclusive, and only one of the two should be be used on a given data set to correct for the same coordinated omission issue. /// See notes in the description of the Histogram calls for an illustration of why this corrective behavior is important. /// </remarks> /// <exception cref="System.IndexOutOfRangeException">if values exceed highestTrackableValue.</exception> public void AddWhileCorrectingForCoordinatedOmission(HistogramBase fromHistogram, long expectedIntervalBetweenValueSamples) { foreach (var v in fromHistogram.RecordedValues()) { RecordValueWithCountAndExpectedInterval( v.ValueIteratedTo, v.CountAtValueIteratedTo, expectedIntervalBetweenValueSamples); } }
/// <summary> /// Appends a Histogram to the log. /// </summary> /// <param name="histogram">The histogram to write to the log.</param> public void Append(HistogramBase histogram) { if (!_hasHeaderWritten) { Write(histogram.StartTimeStamp.ToDateFromMillisecondsSinceEpoch(), histogram); } else { WriteHistogram(histogram); } }
public Recorder( long lowestDiscernibleValue, long highestTrackableValue, int numberOfSignificantValueDigits, HistogramFactory histogramFactory) { _histogramFactory = histogramFactory; _activeHistogram = histogramFactory(_instanceId, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits); _inactiveHistogram = histogramFactory(_instanceId, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits); _activeHistogram.StartTimeStamp = DateTime.Now.MillisecondsSinceUnixEpoch(); }
/// <summary> /// Creates a recorder that will delegate recording to histograms created from these parameters. /// </summary> /// <param name="lowestDiscernibleValue">The lowest value that can be tracked (distinguished from 0) by the histogram. /// Must be a positive integer that is >= 1. /// May be internally rounded down to nearest power of 2. /// </param> /// <param name="highestTrackableValue">The highest value to be tracked by the histogram. /// Must be a positive integer that is >= (2 * lowestTrackableValue). /// </param> /// <param name="numberOfSignificantValueDigits"> /// The number of significant decimal digits to which the histogram will maintain value resolution and separation. /// Must be a non-negative integer between 0 and 5. /// </param> /// <param name="histogramFactory">The factory to be used to actually create instances of <seealso cref="HistogramBase"/>.</param> public Recorder( long lowestDiscernibleValue, long highestTrackableValue, int numberOfSignificantValueDigits, HistogramFactoryDelegate histogramFactory) { _histogramFactory = histogramFactory; _activeHistogram = histogramFactory(_instanceId, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits); _inactiveHistogram = histogramFactory(_instanceId, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits); _activeHistogram.StartTimeStamp = DateTime.Now.MillisecondsSinceUnixEpoch(); }
/// <summary> /// Get a new instance of an interval histogram, which will include a stable, consistent view of all value counts accumulated since the last interval histogram was taken. /// Calling <see cref="GetIntervalHistogram()"/> will reset the value counts, and start accumulating value counts for the next interval. /// </summary> /// <param name="histogramToRecycle">a previously returned interval histogram that may be recycled to avoid allocation and copy operations.</param> /// <returns>A histogram containing the value counts accumulated since the last interval histogram was taken.</returns> /// <remarks> /// <see cref="GetIntervalHistogram(HistogramBase)"/> accepts a previously returned interval histogram that can be recycled internally to avoid allocation and content copying operations. /// It is therefore significantly more efficient for repeated use than <see cref="GetIntervalHistogram()"/> and <see cref="GetIntervalHistogramInto(HistogramBase)"/>. /// The provided <paramref name="histogramToRecycle"/> must be either be null or an interval histogram returned by a previous call to <see cref="GetIntervalHistogram(HistogramBase)"/> or <see cref="GetIntervalHistogram()"/>. /// NOTE: The caller is responsible for not recycling the same returned interval histogram more than once. /// If the same interval histogram instance is recycled more than once, behavior is undefined. /// </remarks> public HistogramBase GetIntervalHistogram(HistogramBase histogramToRecycle) { lock (_gate) { // Verify that replacement histogram can validly be used as an inactive histogram replacement: ValidateFitAsReplacementHistogram(histogramToRecycle); _inactiveHistogram = histogramToRecycle; PerformIntervalSample(); var sampledHistogram = _inactiveHistogram; _inactiveHistogram = null; // Once we expose the sample, we can't reuse it internally until it is recycled return(sampledHistogram); } }
/// <summary> /// Get the computed standard deviation of all recorded values in the histogram /// </summary> /// <returns>the standard deviation (in value units) of the histogram data</returns> public static double GetStdDeviation(this HistogramBase histogram) { var mean = histogram.GetMean(); var geometricDeviationTotal = 0.0; foreach (var iterationValue in histogram.RecordedValues()) { double deviation = (histogram.MedianEquivalentValue(iterationValue.ValueIteratedTo) * 1.0) - mean; geometricDeviationTotal += (deviation * deviation) * iterationValue.CountAddedInThisIterationStep; } var stdDeviation = Math.Sqrt(geometricDeviationTotal / histogram.TotalCount); return(stdDeviation); }
public void Run(Stopwatch stopwatch, HistogramBase histogram) { var cancellationToken = new CancellationTokenSource(); var signal = new ManualResetEvent(false); var globalSignal = new CountdownEvent(3); _pinger.Reset(globalSignal, signal, histogram); _ponger.Reset(globalSignal, cancellationToken.Token); _executor.Execute(_pinger.Run); _executor.Execute(_ponger.Run); globalSignal.Signal(); globalSignal.Wait(); stopwatch.Start(); signal.WaitOne(); stopwatch.Stop(); cancellationToken.Cancel(); }
private void WriteHistogram(HistogramBase histogram) { var targetBuffer = ByteBuffer.Allocate(histogram.GetNeededByteBufferCapacity()); var compressedLength = histogram.EncodeIntoCompressedByteBuffer(targetBuffer); byte[] compressedArray = new byte[compressedLength]; targetBuffer.BlockGet(compressedArray, 0, 0, compressedLength); var startTimeStampSec = histogram.StartTimeStamp / 1000.0; var endTimeStampSec = histogram.EndTimeStamp / 1000.0; var intervalLength = endTimeStampSec - startTimeStampSec; var maxValueUnitRatio = 1000000.0; var intervalMax = histogram.GetMaxValue() / maxValueUnitRatio; var binary = Convert.ToBase64String(compressedArray); var payload = $"{startTimeStampSec:F3},{intervalLength:F3},{intervalMax:F3},{binary}"; _log.WriteLine(payload); _log.Flush(); }
/// <summary> /// Encode this histogram in compressed form into a <see cref="ByteBuffer"/>. /// </summary> /// <param name="source">The histogram to encode</param> /// <param name="targetBuffer">The buffer to write to</param> /// <returns>The number of bytes written to the buffer</returns> public static int EncodeIntoCompressedByteBuffer(this HistogramBase source, ByteBuffer targetBuffer) { int neededCapacity = source.GetNeededByteBufferCapacity(); var intermediateUncompressedByteBuffer = ByteBuffer.Allocate(neededCapacity); source.Encode(intermediateUncompressedByteBuffer, HistogramEncoderV2.Instance); int initialTargetPosition = targetBuffer.Position; targetBuffer.PutInt(GetCompressedEncodingCookie()); int compressedContentsHeaderPosition = targetBuffer.Position; targetBuffer.PutInt(0); // Placeholder for compressed contents length var compressedDataLength = targetBuffer.CompressedCopy(intermediateUncompressedByteBuffer, targetBuffer.Position); targetBuffer.PutInt(compressedContentsHeaderPosition, compressedDataLength); // Record the compressed length int bytesWritten = compressedDataLength + 8; targetBuffer.Position = (initialTargetPosition + bytesWritten); return(bytesWritten); }
private void PerformIntervalSample() { try { _recordingPhaser.ReaderLock(); // Make sure we have an inactive version to flip in: if (_inactiveHistogram == null) { _inactiveHistogram = _histogramFactory(_instanceId, _activeHistogram.LowestTrackableValue, _activeHistogram.HighestTrackableValue, _activeHistogram.NumberOfSignificantValueDigits); } _inactiveHistogram.Reset(); // Swap active and inactive histograms: var tempHistogram = _inactiveHistogram; _inactiveHistogram = _activeHistogram; _activeHistogram = tempHistogram; // Mark end time of previous interval and start time of new one: var now = DateTime.Now.MillisecondsSinceUnixEpoch(); _activeHistogram.StartTimeStamp = now; _inactiveHistogram.EndTimeStamp = now; // Make sure we are not in the middle of recording a value on the previously active histogram: // Flip phase to make sure no recordings that were in flight pre-flip are still active: _recordingPhaser.FlipPhase(TimeSpan.FromMilliseconds(0.5)); } finally { _recordingPhaser.ReaderUnlock(); } }
private void WriteHistogram(HistogramBase histogram) { var targetBuffer = ByteBuffer.Allocate(histogram.GetNeededByteBufferCapacity()); var compressedLength = histogram.EncodeIntoCompressedByteBuffer(targetBuffer); byte[] compressedArray = new byte[compressedLength]; targetBuffer.BlockGet(compressedArray, 0, 0, compressedLength); var startTimeStampSec = histogram.StartTimeStamp / 1000.0; var endTimeStampSec = histogram.EndTimeStamp / 1000.0; var intervalLength = endTimeStampSec - startTimeStampSec; var maxValueUnitRatio = 1000000.0; var intervalMax = histogram.GetMaxValue() / maxValueUnitRatio; var binary = Convert.ToBase64String(compressedArray); var payload = $"{startTimeStampSec:F3},{intervalLength:F3},{intervalMax:F3},{binary}"; _log.WriteLine(payload); _log.Flush(); }
public void Append(HistogramBase histogram) { WriteHistogram(histogram); }
/// <summary> /// Provide a means of iterating through histogram values using linear steps. The iteration is performed in /// steps of <paramref name="valueUnitsPerBucket"/> in size, terminating when all recorded histogram values /// are exhausted. /// </summary> /// <param name="histogram">The histogram to operate on</param> /// <param name="valueUnitsPerBucket">The size (in value units) of the linear buckets to use</param> /// <returns> /// An enumerator of <see cref="HistogramIterationValue"/> through the histogram using a /// <see cref="LinearEnumerator"/>. /// </returns> public static IEnumerable <HistogramIterationValue> LinearBucketValues(this HistogramBase histogram, int valueUnitsPerBucket) { return(new LinearBucketEnumerable(histogram, valueUnitsPerBucket)); }
/// <summary> /// Add the contents of another histogram to this one, while correcting the incoming data for coordinated omission. /// </summary> /// <param name="fromHistogram">The other histogram. highestTrackableValue and largestValueWithSingleUnitResolution must match.</param> /// <param name="expectedIntervalBetweenValueSamples">If <paramref name="expectedIntervalBetweenValueSamples"/> is larger than 0, add auto-generated value records as appropriate if value is larger than <paramref name="expectedIntervalBetweenValueSamples"/></param> /// <remarks> /// To compensate for the loss of sampled values when a recorded value is larger than the expected interval between value samples, the values added will include an auto-generated additional series of decreasingly-smaller(down to the expectedIntervalBetweenValueSamples) value records for each count found in the current histogram that is larger than the expectedIntervalBetweenValueSamples. /// /// Note: This is a post-recording correction method, as opposed to the at-recording correction method provided by {@link #recordValueWithExpectedInterval(long, long) recordValueWithExpectedInterval}. /// The two methods are mutually exclusive, and only one of the two should be be used on a given data set to correct for the same coordinated omission issue. /// See notes in the description of the Histogram calls for an illustration of why this corrective behavior is important. /// </remarks> /// <exception cref="System.IndexOutOfRangeException">if values exceed highestTrackableValue.</exception> public void AddWhileCorrectingForCoordinatedOmission(HistogramBase fromHistogram, long expectedIntervalBetweenValueSamples) { foreach (var v in fromHistogram.RecordedValues()) { RecordValueWithCountAndExpectedInterval( v.ValueIteratedTo, v.CountAtValueIteratedTo, expectedIntervalBetweenValueSamples); } }
/// <summary> /// Place a copy of the value counts accumulated since accumulated (since the last interval histogram was taken) into <paramref name="targetHistogram"/>. /// This will overwrite the existing data in <paramref name="targetHistogram"/>. /// Calling <see cref="GetIntervalHistogramInto(HistogramBase)"/> will reset the value counts, and start accumulating value counts for the next interval. /// </summary> /// <param name="targetHistogram">The histogram into which the interval histogram's data should be copied.</param> public void GetIntervalHistogramInto(HistogramBase targetHistogram) { lock (_gate) { PerformIntervalSample(); _inactiveHistogram.CopyInto(targetHistogram); } }
/// <summary> /// Get the lowest recorded value level in the histogram /// </summary> /// <returns>the Min value recorded in the histogram</returns> public static long GetMinValue(this HistogramBase histogram) { var min = histogram.RecordedValues().Select(hiv => hiv.ValueIteratedTo).FirstOrDefault(); return(histogram.LowestEquivalentValue(min)); }
/// <summary> /// Gets the encoding cookie for a Histogram. /// </summary> /// <param name="histogram">The histogram to get the cookie for</param> /// <returns>The integer cookie value for the histogram.</returns> public static int GetEncodingCookie(this HistogramBase histogram) { //return EncodingCookieBase + (histogram.WordSizeInBytes << 4); return(EncodingCookieBaseV2 | 0x10); // LSBit of wordsize byte indicates TLZE Encoding }
/// <summary> /// Add the contents of another histogram to this one. /// </summary> /// <param name="fromHistogram">The other histogram.</param> /// <exception cref="System.IndexOutOfRangeException">if values in fromHistogram's are higher than highestTrackableValue.</exception> public override void Add(HistogramBase fromHistogram) { //TODO: Review the use of locks on public instances. It seems that getting a copy in a thread safe manner, then adding that to this is better -LC // Synchronize add(). Avoid deadlocks by synchronizing in order of construction identity count. if (Identity < fromHistogram.Identity) { lock (UpdateLock) { lock (fromHistogram) { base.Add(fromHistogram); } } } else { lock (fromHistogram) { lock (UpdateLock) { base.Add(fromHistogram); } } } }
private void ValidateFitAsReplacementHistogram(HistogramBase replacementHistogram) { if(replacementHistogram !=null && replacementHistogram.InstanceId != _activeHistogram.InstanceId) { throw new InvalidOperationException( $"Replacement histogram must have been obtained via a previous getIntervalHistogram() call from this {GetType().Name} instance"); } }
private void PerformIntervalSample() { try { _recordingPhaser.ReaderLock(); // Make sure we have an inactive version to flip in: if (_inactiveHistogram == null) { _inactiveHistogram = _histogramFactory(_instanceId, _activeHistogram.LowestTrackableValue, _activeHistogram.HighestTrackableValue, _activeHistogram.NumberOfSignificantValueDigits); } _inactiveHistogram.Reset(); // Swap active and inactive histograms: var tempHistogram = _inactiveHistogram; _inactiveHistogram = _activeHistogram; _activeHistogram = tempHistogram; // Mark end time of previous interval and start time of new one: var now = DateTime.Now.MillisecondsSinceUnixEpoch(); _activeHistogram.StartTimeStamp = now; _inactiveHistogram.EndTimeStamp = now; // Make sure we are not in the middle of recording a value on the previously active histogram: // Flip phase to make sure no recordings that were in flight pre-flip are still active: _recordingPhaser.FlipPhase(TimeSpan.FromMilliseconds(0.5)); } finally { _recordingPhaser.ReaderUnlock(); } }
private static int GetCompressedEncodingCookie(this HistogramBase histogram) { //return CompressedEncodingCookieBaseV2 + (histogram.WordSizeInBytes << 4); return(CompressedEncodingCookieBaseV2 | 0x10); // LSBit of wordsize byte indicates TLZE Encoding }
/// <summary> /// Produce textual representation of the value distribution of histogram data by percentile. /// The distribution is output with exponentially increasing resolution, with each exponentially decreasing /// half-distance containing <paramref name="percentileTicksPerHalfDistance"/> percentile reporting tick points. /// </summary> /// <param name="histogram">The histogram to operate on</param> /// <param name="printStream">Stream into which the distribution will be output</param> /// <param name="percentileTicksPerHalfDistance"> /// The number of reporting points per exponentially decreasing half-distance /// </param> /// <param name="outputValueUnitScalingRatio"> /// The scaling factor by which to divide histogram recorded values units in output. /// Use the <see cref="OutputScalingFactor"/> constant values to help choose an appropriate output measurement. /// </param> /// <param name="useCsvFormat">Output in CSV (Comma Separated Values) format if <c>true</c>, else use plain text form.</param> public static void OutputPercentileDistribution(this HistogramBase histogram, TextWriter printStream, int percentileTicksPerHalfDistance = 5, double outputValueUnitScalingRatio = OutputScalingFactor.TicksToMilliseconds, bool useCsvFormat = false) { if (useCsvFormat) { printStream.Write("\"Value\",\"Percentile\",\"TotalCount\",\"1/(1-Percentile)\"\n"); } else { printStream.Write("{0,12} {1,14} {2,10} {3,14}\n\n", "Value", "Percentile", "TotalCount", "1/(1-Percentile)"); } string percentileFormatString; string lastLinePercentileFormatString; if (useCsvFormat) { percentileFormatString = "{0:F" + histogram.NumberOfSignificantValueDigits + "},{1:F12},{2},{3:F2}\n"; lastLinePercentileFormatString = "{0:F" + histogram.NumberOfSignificantValueDigits + "},{1:F12},{2},Infinity\n"; } else { percentileFormatString = "{0,12:F" + histogram.NumberOfSignificantValueDigits + "}" + " {1,2:F12} {2,10} {3,14:F2}\n"; lastLinePercentileFormatString = "{0,12:F" + histogram.NumberOfSignificantValueDigits + "} {1,2:F12} {2,10}\n"; } try { foreach (var iterationValue in histogram.Percentiles(percentileTicksPerHalfDistance)) { if (iterationValue.PercentileLevelIteratedTo != 100.0D) { printStream.Write(percentileFormatString, iterationValue.ValueIteratedTo / outputValueUnitScalingRatio, iterationValue.PercentileLevelIteratedTo / 100.0D, iterationValue.TotalCountToThisValue, 1 / (1.0D - (iterationValue.PercentileLevelIteratedTo / 100.0D))); } else { printStream.Write(lastLinePercentileFormatString, iterationValue.ValueIteratedTo / outputValueUnitScalingRatio, iterationValue.PercentileLevelIteratedTo / 100.0D, iterationValue.TotalCountToThisValue); } } if (!useCsvFormat) { // Calculate and output mean and std. deviation. // Note: mean/std. deviation numbers are very often completely irrelevant when // data is extremely non-normal in distribution (e.g. in cases of strong multi-modal // response time distribution associated with GC pauses). However, reporting these numbers // can be very useful for contrasting with the detailed percentile distribution // reported by outputPercentileDistribution(). It is not at all surprising to find // percentile distributions where results fall many tens or even hundreds of standard // deviations away from the mean - such results simply indicate that the data sampled // exhibits a very non-normal distribution, highlighting situations for which the std. // deviation metric is a useless indicator. var mean = histogram.GetMean() / outputValueUnitScalingRatio; var stdDeviation = histogram.GetStdDeviation() / outputValueUnitScalingRatio; printStream.Write("#[Mean = {0,12:F" + histogram.NumberOfSignificantValueDigits + "}, " + "StdDeviation = {1,12:F" + histogram.NumberOfSignificantValueDigits + "}]\n", mean, stdDeviation); printStream.Write("#[Max = {0,12:F" + histogram.NumberOfSignificantValueDigits + "}, Total count = {1,12}]\n", histogram.GetMaxValue() / outputValueUnitScalingRatio, histogram.TotalCount); printStream.Write("#[Buckets = {0,12}, SubBuckets = {1,12}]\n", histogram.BucketCount, histogram.SubBucketCount); } } catch (ArgumentOutOfRangeException) { // Overflow conditions on histograms can lead to ArgumentOutOfRangeException on iterations: if (histogram.HasOverflowed()) { printStream.Write("# Histogram counts indicate OVERFLOW values"); } else { // Re-throw if reason is not a known overflow: throw; } } }
/// <summary> /// Get the computed mean value of all recorded values in the histogram /// </summary> /// <returns>the mean value (in value units) of the histogram data</returns> public static double GetMean(this HistogramBase histogram) { var totalValue = histogram.RecordedValues().Select(hiv => hiv.TotalValueToThisValue).LastOrDefault(); return((totalValue * 1.0) / histogram.TotalCount); }
/// <summary> /// Provide a means of iterating through histogram values according to percentile levels. /// The iteration is performed in steps that start at 0% and reduce their distance to 100% according to the /// <paramref name="percentileTicksPerHalfDistance"/> parameter, ultimately reaching 100% when all recorded /// histogram values are exhausted. /// </summary> /// <param name="histogram">The histogram to operate on</param> /// <param name="percentileTicksPerHalfDistance"> /// The number of iteration steps per half-distance to 100%. /// </param> /// <returns> /// An enumerator of <see cref="HistogramIterationValue"/> through the histogram using a /// <see cref="PercentileEnumerator"/>. /// </returns> public static IEnumerable <HistogramIterationValue> Percentiles(this HistogramBase histogram, int percentileTicksPerHalfDistance) { return(new PercentileEnumerable(histogram, percentileTicksPerHalfDistance)); }
/// <summary> /// Get the highest value that is equivalent to the given value within the histogram's resolution. /// Where "equivalent" means that value samples recorded for any two equivalent values are counted in a common /// total count. /// </summary> /// <param name="histogram">The histogram to operate on</param> /// <param name="value">The given value</param> /// <returns>The highest value that is equivalent to the given value within the histogram's resolution.</returns> public static long HighestEquivalentValue(this HistogramBase histogram, long value) { return(histogram.NextNonEquivalentValue(value) - 1); }
/// <summary> /// Get the highest recorded value level in the histogram /// </summary> /// <returns>the Max value recorded in the histogram</returns> public static long GetMaxValue(this HistogramBase histogram) { var max = histogram.RecordedValues().Select(hiv => hiv.ValueIteratedTo).LastOrDefault(); return(histogram.HighestEquivalentValue(max)); }
/// <summary> /// Get a new instance of an interval histogram, which will include a stable, consistent view of all value counts accumulated since the last interval histogram was taken. /// Calling <see cref="GetIntervalHistogram()"/> will reset the value counts, and start accumulating value counts for the next interval. /// </summary> /// <param name="histogramToRecycle">a previously returned interval histogram that may be recycled to avoid allocation and copy operations.</param> /// <returns>A histogram containing the value counts accumulated since the last interval histogram was taken.</returns> /// <remarks> /// <see cref="GetIntervalHistogram(HistogramBase)"/> accepts a previously returned interval histogram that can be recycled internally to avoid allocation and content copying operations. /// It is therefore significantly more efficient for repeated use than <see cref="GetIntervalHistogram()"/> and <see cref="GetIntervalHistogramInto(HistogramBase)"/>. /// The provided <paramref name="histogramToRecycle"/> must be either be null or an interval histogram returned by a previous call to <see cref="GetIntervalHistogram(HistogramBase)"/> or <see cref="GetIntervalHistogram()"/>. /// NOTE: The caller is responsible for not recycling the same returned interval histogram more than once. /// If the same interval histogram instance is recycled more than once, behavior is undefined. /// </remarks> public HistogramBase GetIntervalHistogram(HistogramBase histogramToRecycle) { lock (_gate) { // Verify that replacement histogram can validly be used as an inactive histogram replacement: ValidateFitAsReplacementHistogram(histogramToRecycle); _inactiveHistogram = histogramToRecycle; PerformIntervalSample(); var sampledHistogram = _inactiveHistogram; _inactiveHistogram = null; // Once we expose the sample, we can't reuse it internally until it is recycled return sampledHistogram; } }
/// <summary> /// Provide a means of iterating through histogram values at logarithmically increasing levels. /// The iteration is performed in steps that start at<i>valueUnitsInFirstBucket</i> and increase exponentially /// according to <paramref name="logBase"/>, terminating when all recorded histogram values are exhausted. /// </summary> /// <param name="histogram">The histogram to operate on</param> /// <param name="valueUnitsInFirstBucket">The size (in value units) of the first bucket in the iteration</param> /// <param name="logBase">The multiplier by which bucket sizes will grow in each iteration step</param> /// <returns> /// An enumerator of <see cref="HistogramIterationValue"/> through the histogram using a /// <see cref="LogarithmicEnumerator"/>. /// </returns> public static IEnumerable <HistogramIterationValue> LogarithmicBucketValues(this HistogramBase histogram, int valueUnitsInFirstBucket, double logBase) { return(new LogarithmicBucketEnumerable(histogram, valueUnitsInFirstBucket, logBase)); }
/// <summary> /// Add the contents of another histogram to this one. /// </summary> /// <param name="fromHistogram">The other histogram.</param> /// <exception cref="System.IndexOutOfRangeException">if values in fromHistogram's are higher than highestTrackableValue.</exception> public virtual void Add(HistogramBase fromHistogram) { if (HighestTrackableValue < fromHistogram.HighestTrackableValue) { throw new ArgumentOutOfRangeException(nameof(fromHistogram), $"The other histogram covers a wider range ({fromHistogram.HighestTrackableValue} than this one ({HighestTrackableValue})."); } if ((BucketCount == fromHistogram.BucketCount) && (SubBucketCount == fromHistogram.SubBucketCount) && (_unitMagnitude == fromHistogram._unitMagnitude)) { // Counts arrays are of the same length and meaning, so we can just iterate and add directly: for (var i = 0; i < fromHistogram.CountsArrayLength; i++) { AddToCountAtIndex(i, fromHistogram.GetCountAtIndex(i)); } } else { // Arrays are not a direct match, so we can't just stream through and add them. // Instead, go through the array and add each non-zero value found at it's proper value: for (var i = 0; i < fromHistogram.CountsArrayLength; i++) { var count = fromHistogram.GetCountAtIndex(i); RecordValueWithCount(fromHistogram.ValueFromIndex(i), count); } } }
public void Reset(CountdownEvent globalSignal, ManualResetEvent signal, HistogramBase histogram) { _globalSignal = globalSignal; _signal = signal; _histogram = histogram; _counter = 0; }
/// <summary> /// Copy this histogram, corrected for coordinated omission, into the target histogram, overwriting it's contents. /// </summary> /// <param name="targetHistogram">the histogram to copy into</param> /// <param name="expectedIntervalBetweenValueSamples">If <paramref name="expectedIntervalBetweenValueSamples"/> is larger than 0, add auto-generated value records as appropriate if value is larger than <paramref name="expectedIntervalBetweenValueSamples"/></param> /// <remarks> /// See <see cref="CopyCorrectedForCoordinatedOmission"/> for more detailed explanation about how correction is applied /// </remarks> public void CopyIntoCorrectedForCoordinatedOmission(HistogramBase targetHistogram, long expectedIntervalBetweenValueSamples) { targetHistogram.Reset(); targetHistogram.AddWhileCorrectingForCoordinatedOmission(this, expectedIntervalBetweenValueSamples); targetHistogram.StartTimeStamp = StartTimeStamp; targetHistogram.EndTimeStamp = EndTimeStamp; }