/// <summary>
/// Add the contents of another histogram to this one.
/// </summary>
/// <param name="fromHistogram">The other histogram.</param>
/// <exception cref="System.IndexOutOfRangeException">if values in fromHistogram's are higher than highestTrackableValue.</exception>
public virtual void Add(HistogramBase fromHistogram)
{
if (HighestTrackableValue < fromHistogram.HighestTrackableValue)
{
throw new ArgumentOutOfRangeException(nameof(fromHistogram), $"The other histogram covers a wider range ({fromHistogram.HighestTrackableValue} than this one ({HighestTrackableValue}).");
}
if ((BucketCount == fromHistogram.BucketCount) &&
(SubBucketCount == fromHistogram.SubBucketCount) &&
(_unitMagnitude == fromHistogram._unitMagnitude))
{
// Counts arrays are of the same length and meaning, so we can just iterate and add directly:
for (var i = 0; i < fromHistogram.CountsArrayLength; i++)
{
AddToCountAtIndex(i, fromHistogram.GetCountAtIndex(i));
}
}
else
{
// Arrays are not a direct match, so we can't just stream through and add them.
// Instead, go through the array and add each non-zero value found at it's proper value:
for (var i = 0; i < fromHistogram.CountsArrayLength; i++)
{
var count = fromHistogram.GetCountAtIndex(i);
RecordValueWithCount(fromHistogram.ValueFromIndex(i), count);
}
}
}
/// <summary>
/// Copy this histogram, corrected for coordinated omission, into the target histogram, overwriting it's contents.
/// </summary>
/// <param name="targetHistogram">the histogram to copy into</param>
/// <param name="expectedIntervalBetweenValueSamples">If <paramref name="expectedIntervalBetweenValueSamples"/> is larger than 0, add auto-generated value records as appropriate if value is larger than <paramref name="expectedIntervalBetweenValueSamples"/></param>
/// <remarks>
/// See <see cref="CopyCorrectedForCoordinatedOmission"/> for more detailed explanation about how correction is applied
/// </remarks>
public void CopyIntoCorrectedForCoordinatedOmission(HistogramBase targetHistogram, long expectedIntervalBetweenValueSamples)
{
targetHistogram.Reset();
targetHistogram.AddWhileCorrectingForCoordinatedOmission(this, expectedIntervalBetweenValueSamples);
targetHistogram.StartTimeStamp = StartTimeStamp;
targetHistogram.EndTimeStamp = EndTimeStamp;
}
/// <summary>
/// Add the contents of another histogram to this one.
/// </summary>
/// <param name="fromHistogram">The other histogram.</param>
/// <exception cref="System.IndexOutOfRangeException">if values in fromHistogram's are higher than highestTrackableValue.</exception>
public override void Add(HistogramBase fromHistogram)
{
//TODO: Review the use of locks on public instances. It seems that getting a copy in a thread safe manner, then adding that to this is better -LC
// Synchronize add(). Avoid deadlocks by synchronizing in order of construction identity count.
if (Identity < fromHistogram.Identity)
{
lock (UpdateLock)
{
lock (fromHistogram)
{
base.Add(fromHistogram);
}
}
}
else
{
lock (fromHistogram)
{
lock (UpdateLock)
{
base.Add(fromHistogram);
}
}
}
}
/// <summary>
/// Copy this histogram into the target histogram, overwriting it's contents.
/// </summary>
/// <param name="source">The source histogram</param>
/// <param name="targetHistogram">the histogram to copy into</param>
public static void CopyInto(this HistogramBase source, HistogramBase targetHistogram)
{
targetHistogram.Reset();
targetHistogram.Add(source);
targetHistogram.StartTimeStamp = source.StartTimeStamp;
targetHistogram.EndTimeStamp = source.EndTimeStamp;
}
/// <summary>
/// Produce textual representation of the value distribution of histogram data by percentile.
/// The distribution is output with exponentially increasing resolution, with each exponentially decreasing
/// half-distance containing <paramref name="percentileTicksPerHalfDistance"/> percentile reporting tick points.
/// </summary>
/// <param name="histogram">The histogram to operate on</param>
/// <param name="writer">The <see cref="TextWriter"/> into which the distribution will be output</param>
/// <param name="percentileTicksPerHalfDistance">
/// The number of reporting points per exponentially decreasing half-distance
/// </param>
/// <param name="outputValueUnitScalingRatio">
/// The scaling factor by which to divide histogram recorded values units in output.
/// Use the <see cref="OutputScalingFactor"/> constant values to help choose an appropriate output measurement.
/// </param>
/// <param name="useCsvFormat">Output in CSV (Comma Separated Values) format if <c>true</c>, else use plain text form.</param>
public static void OutputPercentileDistribution(this HistogramBase histogram,
TextWriter writer,
int percentileTicksPerHalfDistance = 5,
double outputValueUnitScalingRatio = OutputScalingFactor.None,
bool useCsvFormat = false)
{
var formatter = useCsvFormat
? (IOutputFormatter) new CsvOutputFormatter(writer, histogram.NumberOfSignificantValueDigits, outputValueUnitScalingRatio)
: (IOutputFormatter) new HgrmOutputFormatter(writer, histogram.NumberOfSignificantValueDigits, outputValueUnitScalingRatio);
try
{
formatter.WriteHeader();
foreach (var iterationValue in histogram.Percentiles(percentileTicksPerHalfDistance))
{
formatter.WriteValue(iterationValue);
}
formatter.WriteFooter(histogram);
}
catch (ArgumentOutOfRangeException)
{
// Overflow conditions on histograms can lead to ArgumentOutOfRangeException on iterations:
if (histogram.HasOverflowed())
{
writer.Write("# Histogram counts indicate OVERFLOW values");
}
else
{
// Re-throw if reason is not a known overflow:
throw;
}
}
}
/// <summary>
/// Copy this histogram into the target histogram, overwriting it's contents.
/// </summary>
/// <param name="source">The source histogram</param>
/// <param name="targetHistogram">the histogram to copy into</param>
public static void CopyInto(this HistogramBase source, HistogramBase targetHistogram)
{
targetHistogram.Reset();
targetHistogram.Add(source);
targetHistogram.StartTimeStamp = source.StartTimeStamp;
targetHistogram.EndTimeStamp = source.EndTimeStamp;
}
/// <summary>
/// Place a copy of the value counts accumulated since accumulated (since the last interval histogram was taken) into <paramref name="targetHistogram"/>.
/// This will overwrite the existing data in <paramref name="targetHistogram"/>.
/// Calling <see cref="GetIntervalHistogramInto(HistogramBase)"/> will reset the value counts, and start accumulating value counts for the next interval.
/// </summary>
/// <param name="targetHistogram">The histogram into which the interval histogram's data should be copied.</param>
public void GetIntervalHistogramInto(HistogramBase targetHistogram)
{
lock (_gate)
{
PerformIntervalSample();
_inactiveHistogram.CopyInto(targetHistogram);
}
}
private void ValidateFitAsReplacementHistogram(HistogramBase replacementHistogram)
{
if (replacementHistogram != null && replacementHistogram.InstanceId != _activeHistogram.InstanceId)
{
throw new InvalidOperationException(
$"Replacement histogram must have been obtained via a previous getIntervalHistogram() call from this {GetType().Name} instance");
}
}
private static HistogramBase CompressedEncodeDecode(HistogramBase source)
{
var targetBuffer = ByteBuffer.Allocate(source.GetNeededByteBufferCapacity());
source.EncodeIntoCompressedByteBuffer(targetBuffer);
targetBuffer.Position = 0;
return(HistogramEncoding.DecodeFromCompressedByteBuffer(targetBuffer, 0));
}
/// <summary>
/// Executes the action and records the time to complete the action.
/// The time is recorded in ticks.
/// Note this is a convenience method and can carry a cost.
/// If the <paramref name="action"/> delegate is not cached, then it may incur an allocation cost for each invocation of <see cref="Record"/>
/// </summary>
/// <param name="histogram">The Histogram to record the latency in.</param>
/// <param name="action">The functionality to execute and measure</param>
/// <remarks>
/// <para>
/// Ticks are used as the unit of recording here as they are the smallest unit that .NET can measure
/// and require no conversion at time of recording. Instead conversion (scaling) can be done at time
/// of output to microseconds, milliseconds, seconds or other appropriate unit.
/// </para>
/// <para>
/// If you are able to cache the <paramref name="action"/> delegate, then doing so is encouraged.
/// <example>
/// Here are two examples.
/// The first does not cache the delegate
///
/// <code>
/// for (long i = 0; i < loopCount; i++)
/// {
/// histogram.Record(IncrementNumber);
/// }
/// </code>
/// This second example does cache the delegate
/// <code>
/// Action incrementNumber = IncrementNumber;
/// for (long i = 0; i < loopCount; i++)
/// {
/// histogram.Record(incrementNumber);
/// }
/// </code>
/// In the second example, we will not be making allocations each time i.e. an allocation of an <seealso cref="Action"/> from <code>IncrementNumber</code>.
/// This will reduce memory pressure and therefore garbage collection penalties.
/// For performance sensitive applications, this method may not be suitable.
/// As always, you are encouraged to test and measure the impact for your scenario.
/// </example>
/// </para>
/// </remarks>
public static void Record(this HistogramBase histogram, Action action)
{
var start = Stopwatch.GetTimestamp();
action();
var elapsedTicks = (Stopwatch.GetTimestamp() - start);
histogram.RecordValue(elapsedTicks);
}
/// <summary>
/// Add the contents of another histogram to this one, while correcting the incoming data for coordinated omission.
/// </summary>
/// <param name="fromHistogram">The other histogram. highestTrackableValue and largestValueWithSingleUnitResolution must match.</param>
/// <param name="expectedIntervalBetweenValueSamples">If <paramref name="expectedIntervalBetweenValueSamples"/> is larger than 0, add auto-generated value records as appropriate if value is larger than <paramref name="expectedIntervalBetweenValueSamples"/></param>
/// <remarks>
/// To compensate for the loss of sampled values when a recorded value is larger than the expected interval between value samples, the values added will include an auto-generated additional series of decreasingly-smaller(down to the expectedIntervalBetweenValueSamples) value records for each count found in the current histogram that is larger than the expectedIntervalBetweenValueSamples.
///
/// Note: This is a post-recording correction method, as opposed to the at-recording correction method provided by {@link #recordValueWithExpectedInterval(long, long) recordValueWithExpectedInterval}.
/// The two methods are mutually exclusive, and only one of the two should be be used on a given data set to correct for the same coordinated omission issue.
/// See notes in the description of the Histogram calls for an illustration of why this corrective behavior is important.
/// </remarks>
/// <exception cref="System.IndexOutOfRangeException">if values exceed highestTrackableValue.</exception>
public void AddWhileCorrectingForCoordinatedOmission(HistogramBase fromHistogram, long expectedIntervalBetweenValueSamples)
{
foreach (var v in fromHistogram.RecordedValues())
{
RecordValueWithCountAndExpectedInterval(
v.ValueIteratedTo,
v.CountAtValueIteratedTo,
expectedIntervalBetweenValueSamples);
}
}
/// <summary>
/// Appends a Histogram to the log.
/// </summary>
/// <param name="histogram">The histogram to write to the log.</param>
public void Append(HistogramBase histogram)
{
if (!_hasHeaderWritten)
{
Write(histogram.StartTimeStamp.ToDateFromMillisecondsSinceEpoch(), histogram);
}
else
{
WriteHistogram(histogram);
}
}
public Recorder(
long lowestDiscernibleValue,
long highestTrackableValue,
int numberOfSignificantValueDigits,
HistogramFactory histogramFactory)
{
_histogramFactory = histogramFactory;
_activeHistogram = histogramFactory(_instanceId, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
_inactiveHistogram = histogramFactory(_instanceId, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
_activeHistogram.StartTimeStamp = DateTime.Now.MillisecondsSinceUnixEpoch();
}
/// <summary>
/// Creates a recorder that will delegate recording to histograms created from these parameters.
/// </summary>
/// <param name="lowestDiscernibleValue">The lowest value that can be tracked (distinguished from 0) by the histogram.
/// Must be a positive integer that is >= 1.
/// May be internally rounded down to nearest power of 2.
/// </param>
/// <param name="highestTrackableValue">The highest value to be tracked by the histogram.
/// Must be a positive integer that is >= (2 * lowestTrackableValue).
/// </param>
/// <param name="numberOfSignificantValueDigits">
/// The number of significant decimal digits to which the histogram will maintain value resolution and separation.
/// Must be a non-negative integer between 0 and 5.
/// </param>
/// <param name="histogramFactory">The factory to be used to actually create instances of <seealso cref="HistogramBase"/>.</param>
public Recorder(
long lowestDiscernibleValue,
long highestTrackableValue,
int numberOfSignificantValueDigits,
HistogramFactoryDelegate histogramFactory)
{
_histogramFactory = histogramFactory;
_activeHistogram = histogramFactory(_instanceId, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
_inactiveHistogram = histogramFactory(_instanceId, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
_activeHistogram.StartTimeStamp = DateTime.Now.MillisecondsSinceUnixEpoch();
}
/// <summary>
/// Get a new instance of an interval histogram, which will include a stable, consistent view of all value counts accumulated since the last interval histogram was taken.
/// Calling <see cref="GetIntervalHistogram()"/> will reset the value counts, and start accumulating value counts for the next interval.
/// </summary>
/// <param name="histogramToRecycle">a previously returned interval histogram that may be recycled to avoid allocation and copy operations.</param>
/// <returns>A histogram containing the value counts accumulated since the last interval histogram was taken.</returns>
/// <remarks>
/// <see cref="GetIntervalHistogram(HistogramBase)"/> accepts a previously returned interval histogram that can be recycled internally to avoid allocation and content copying operations.
/// It is therefore significantly more efficient for repeated use than <see cref="GetIntervalHistogram()"/> and <see cref="GetIntervalHistogramInto(HistogramBase)"/>.
/// The provided <paramref name="histogramToRecycle"/> must be either be null or an interval histogram returned by a previous call to <see cref="GetIntervalHistogram(HistogramBase)"/> or <see cref="GetIntervalHistogram()"/>.
/// NOTE: The caller is responsible for not recycling the same returned interval histogram more than once.
/// If the same interval histogram instance is recycled more than once, behavior is undefined.
/// </remarks>
public HistogramBase GetIntervalHistogram(HistogramBase histogramToRecycle)
{
lock (_gate)
{
// Verify that replacement histogram can validly be used as an inactive histogram replacement:
ValidateFitAsReplacementHistogram(histogramToRecycle);
_inactiveHistogram = histogramToRecycle;
PerformIntervalSample();
var sampledHistogram = _inactiveHistogram;
_inactiveHistogram = null; // Once we expose the sample, we can't reuse it internally until it is recycled
return(sampledHistogram);
}
}
/// <summary>
/// Get the computed standard deviation of all recorded values in the histogram
/// </summary>
/// <returns>the standard deviation (in value units) of the histogram data</returns>
public static double GetStdDeviation(this HistogramBase histogram)
{
var mean = histogram.GetMean();
var geometricDeviationTotal = 0.0;
foreach (var iterationValue in histogram.RecordedValues())
{
double deviation = (histogram.MedianEquivalentValue(iterationValue.ValueIteratedTo) * 1.0) - mean;
geometricDeviationTotal += (deviation * deviation) * iterationValue.CountAddedInThisIterationStep;
}
var stdDeviation = Math.Sqrt(geometricDeviationTotal / histogram.TotalCount);
return(stdDeviation);
}
public void Run(Stopwatch stopwatch, HistogramBase histogram)
{
var cancellationToken = new CancellationTokenSource();
var signal = new ManualResetEvent(false);
var globalSignal = new CountdownEvent(3);
_pinger.Reset(globalSignal, signal, histogram);
_ponger.Reset(globalSignal, cancellationToken.Token);
_executor.Execute(_pinger.Run);
_executor.Execute(_ponger.Run);
globalSignal.Signal();
globalSignal.Wait();
stopwatch.Start();
signal.WaitOne();
stopwatch.Stop();
cancellationToken.Cancel();
}
private void WriteHistogram(HistogramBase histogram)
{
var targetBuffer = ByteBuffer.Allocate(histogram.GetNeededByteBufferCapacity());
var compressedLength = histogram.EncodeIntoCompressedByteBuffer(targetBuffer);
byte[] compressedArray = new byte[compressedLength];
targetBuffer.BlockGet(compressedArray, 0, 0, compressedLength);
var startTimeStampSec = histogram.StartTimeStamp / 1000.0;
var endTimeStampSec = histogram.EndTimeStamp / 1000.0;
var intervalLength = endTimeStampSec - startTimeStampSec;
var maxValueUnitRatio = 1000000.0;
var intervalMax = histogram.GetMaxValue() / maxValueUnitRatio;
var binary = Convert.ToBase64String(compressedArray);
var payload = $"{startTimeStampSec:F3},{intervalLength:F3},{intervalMax:F3},{binary}";
_log.WriteLine(payload);
_log.Flush();
}
/// <summary>
/// Encode this histogram in compressed form into a <see cref="ByteBuffer"/>.
/// </summary>
/// <param name="source">The histogram to encode</param>
/// <param name="targetBuffer">The buffer to write to</param>
/// <returns>The number of bytes written to the buffer</returns>
public static int EncodeIntoCompressedByteBuffer(this HistogramBase source, ByteBuffer targetBuffer)
{
int neededCapacity = source.GetNeededByteBufferCapacity();
var intermediateUncompressedByteBuffer = ByteBuffer.Allocate(neededCapacity);
source.Encode(intermediateUncompressedByteBuffer, HistogramEncoderV2.Instance);
int initialTargetPosition = targetBuffer.Position;
targetBuffer.PutInt(GetCompressedEncodingCookie());
int compressedContentsHeaderPosition = targetBuffer.Position;
targetBuffer.PutInt(0); // Placeholder for compressed contents length
var compressedDataLength = targetBuffer.CompressedCopy(intermediateUncompressedByteBuffer, targetBuffer.Position);
targetBuffer.PutInt(compressedContentsHeaderPosition, compressedDataLength); // Record the compressed length
int bytesWritten = compressedDataLength + 8;
targetBuffer.Position = (initialTargetPosition + bytesWritten);
return(bytesWritten);
}
private void PerformIntervalSample()
{
try
{
_recordingPhaser.ReaderLock();
// Make sure we have an inactive version to flip in:
if (_inactiveHistogram == null)
{
_inactiveHistogram = _histogramFactory(_instanceId,
_activeHistogram.LowestTrackableValue,
_activeHistogram.HighestTrackableValue,
_activeHistogram.NumberOfSignificantValueDigits);
}
_inactiveHistogram.Reset();
// Swap active and inactive histograms:
var tempHistogram = _inactiveHistogram;
_inactiveHistogram = _activeHistogram;
_activeHistogram = tempHistogram;
// Mark end time of previous interval and start time of new one:
var now = DateTime.Now.MillisecondsSinceUnixEpoch();
_activeHistogram.StartTimeStamp = now;
_inactiveHistogram.EndTimeStamp = now;
// Make sure we are not in the middle of recording a value on the previously active histogram:
// Flip phase to make sure no recordings that were in flight pre-flip are still active:
_recordingPhaser.FlipPhase(TimeSpan.FromMilliseconds(0.5));
}
finally
{
_recordingPhaser.ReaderUnlock();
}
}
private void WriteHistogram(HistogramBase histogram)
{
var targetBuffer = ByteBuffer.Allocate(histogram.GetNeededByteBufferCapacity());
var compressedLength = histogram.EncodeIntoCompressedByteBuffer(targetBuffer);
byte[] compressedArray = new byte[compressedLength];
targetBuffer.BlockGet(compressedArray, 0, 0, compressedLength);
var startTimeStampSec = histogram.StartTimeStamp / 1000.0;
var endTimeStampSec = histogram.EndTimeStamp / 1000.0;
var intervalLength = endTimeStampSec - startTimeStampSec;
var maxValueUnitRatio = 1000000.0;
var intervalMax = histogram.GetMaxValue() / maxValueUnitRatio;
var binary = Convert.ToBase64String(compressedArray);
var payload = $"{startTimeStampSec:F3},{intervalLength:F3},{intervalMax:F3},{binary}";
_log.WriteLine(payload);
_log.Flush();
}
public void Append(HistogramBase histogram)
{
WriteHistogram(histogram);
}
/// <summary>
/// Provide a means of iterating through histogram values using linear steps. The iteration is performed in
/// steps of <paramref name="valueUnitsPerBucket"/> in size, terminating when all recorded histogram values
/// are exhausted.
/// </summary>
/// <param name="histogram">The histogram to operate on</param>
/// <param name="valueUnitsPerBucket">The size (in value units) of the linear buckets to use</param>
/// <returns>
/// An enumerator of <see cref="HistogramIterationValue"/> through the histogram using a
/// <see cref="LinearEnumerator"/>.
/// </returns>
public static IEnumerable <HistogramIterationValue> LinearBucketValues(this HistogramBase histogram, int valueUnitsPerBucket)
{
return(new LinearBucketEnumerable(histogram, valueUnitsPerBucket));
}
/// <summary>
/// Add the contents of another histogram to this one, while correcting the incoming data for coordinated omission.
/// </summary>
/// <param name="fromHistogram">The other histogram. highestTrackableValue and largestValueWithSingleUnitResolution must match.</param>
/// <param name="expectedIntervalBetweenValueSamples">If <paramref name="expectedIntervalBetweenValueSamples"/> is larger than 0, add auto-generated value records as appropriate if value is larger than <paramref name="expectedIntervalBetweenValueSamples"/></param>
/// <remarks>
/// To compensate for the loss of sampled values when a recorded value is larger than the expected interval between value samples, the values added will include an auto-generated additional series of decreasingly-smaller(down to the expectedIntervalBetweenValueSamples) value records for each count found in the current histogram that is larger than the expectedIntervalBetweenValueSamples.
///
/// Note: This is a post-recording correction method, as opposed to the at-recording correction method provided by {@link #recordValueWithExpectedInterval(long, long) recordValueWithExpectedInterval}.
/// The two methods are mutually exclusive, and only one of the two should be be used on a given data set to correct for the same coordinated omission issue.
/// See notes in the description of the Histogram calls for an illustration of why this corrective behavior is important.
/// </remarks>
/// <exception cref="System.IndexOutOfRangeException">if values exceed highestTrackableValue.</exception>
public void AddWhileCorrectingForCoordinatedOmission(HistogramBase fromHistogram, long expectedIntervalBetweenValueSamples)
{
foreach (var v in fromHistogram.RecordedValues())
{
RecordValueWithCountAndExpectedInterval(
v.ValueIteratedTo,
v.CountAtValueIteratedTo,
expectedIntervalBetweenValueSamples);
}
}
/// <summary>
/// Place a copy of the value counts accumulated since accumulated (since the last interval histogram was taken) into <paramref name="targetHistogram"/>.
/// This will overwrite the existing data in <paramref name="targetHistogram"/>.
/// Calling <see cref="GetIntervalHistogramInto(HistogramBase)"/> will reset the value counts, and start accumulating value counts for the next interval.
/// </summary>
/// <param name="targetHistogram">The histogram into which the interval histogram's data should be copied.</param>
public void GetIntervalHistogramInto(HistogramBase targetHistogram)
{
lock (_gate)
{
PerformIntervalSample();
_inactiveHistogram.CopyInto(targetHistogram);
}
}
/// <summary>
/// Get the lowest recorded value level in the histogram
/// </summary>
/// <returns>the Min value recorded in the histogram</returns>
public static long GetMinValue(this HistogramBase histogram)
{
var min = histogram.RecordedValues().Select(hiv => hiv.ValueIteratedTo).FirstOrDefault();
return(histogram.LowestEquivalentValue(min));
}
/// <summary>
/// Gets the encoding cookie for a Histogram.
/// </summary>
/// <param name="histogram">The histogram to get the cookie for</param>
/// <returns>The integer cookie value for the histogram.</returns>
public static int GetEncodingCookie(this HistogramBase histogram)
{
//return EncodingCookieBase + (histogram.WordSizeInBytes << 4);
return(EncodingCookieBaseV2 | 0x10); // LSBit of wordsize byte indicates TLZE Encoding
}
/// <summary>
/// Add the contents of another histogram to this one.
/// </summary>
/// <param name="fromHistogram">The other histogram.</param>
/// <exception cref="System.IndexOutOfRangeException">if values in fromHistogram's are higher than highestTrackableValue.</exception>
public override void Add(HistogramBase fromHistogram)
{
//TODO: Review the use of locks on public instances. It seems that getting a copy in a thread safe manner, then adding that to this is better -LC
// Synchronize add(). Avoid deadlocks by synchronizing in order of construction identity count.
if (Identity < fromHistogram.Identity)
{
lock (UpdateLock)
{
lock (fromHistogram)
{
base.Add(fromHistogram);
}
}
}
else
{
lock (fromHistogram)
{
lock (UpdateLock)
{
base.Add(fromHistogram);
}
}
}
}
private void ValidateFitAsReplacementHistogram(HistogramBase replacementHistogram)
{
if(replacementHistogram !=null && replacementHistogram.InstanceId != _activeHistogram.InstanceId)
{
throw new InvalidOperationException(
$"Replacement histogram must have been obtained via a previous getIntervalHistogram() call from this {GetType().Name} instance");
}
}
private void PerformIntervalSample()
{
try
{
_recordingPhaser.ReaderLock();
// Make sure we have an inactive version to flip in:
if (_inactiveHistogram == null)
{
_inactiveHistogram = _histogramFactory(_instanceId,
_activeHistogram.LowestTrackableValue,
_activeHistogram.HighestTrackableValue,
_activeHistogram.NumberOfSignificantValueDigits);
}
_inactiveHistogram.Reset();
// Swap active and inactive histograms:
var tempHistogram = _inactiveHistogram;
_inactiveHistogram = _activeHistogram;
_activeHistogram = tempHistogram;
// Mark end time of previous interval and start time of new one:
var now = DateTime.Now.MillisecondsSinceUnixEpoch();
_activeHistogram.StartTimeStamp = now;
_inactiveHistogram.EndTimeStamp = now;
// Make sure we are not in the middle of recording a value on the previously active histogram:
// Flip phase to make sure no recordings that were in flight pre-flip are still active:
_recordingPhaser.FlipPhase(TimeSpan.FromMilliseconds(0.5));
}
finally
{
_recordingPhaser.ReaderUnlock();
}
}
private static int GetCompressedEncodingCookie(this HistogramBase histogram)
{
//return CompressedEncodingCookieBaseV2 + (histogram.WordSizeInBytes << 4);
return(CompressedEncodingCookieBaseV2 | 0x10); // LSBit of wordsize byte indicates TLZE Encoding
}
/// <summary>
/// Produce textual representation of the value distribution of histogram data by percentile.
/// The distribution is output with exponentially increasing resolution, with each exponentially decreasing
/// half-distance containing <paramref name="percentileTicksPerHalfDistance"/> percentile reporting tick points.
/// </summary>
/// <param name="histogram">The histogram to operate on</param>
/// <param name="printStream">Stream into which the distribution will be output</param>
/// <param name="percentileTicksPerHalfDistance">
/// The number of reporting points per exponentially decreasing half-distance
/// </param>
/// <param name="outputValueUnitScalingRatio">
/// The scaling factor by which to divide histogram recorded values units in output.
/// Use the <see cref="OutputScalingFactor"/> constant values to help choose an appropriate output measurement.
/// </param>
/// <param name="useCsvFormat">Output in CSV (Comma Separated Values) format if <c>true</c>, else use plain text form.</param>
public static void OutputPercentileDistribution(this HistogramBase histogram,
TextWriter printStream,
int percentileTicksPerHalfDistance = 5,
double outputValueUnitScalingRatio = OutputScalingFactor.TicksToMilliseconds,
bool useCsvFormat = false)
{
if (useCsvFormat)
{
printStream.Write("\"Value\",\"Percentile\",\"TotalCount\",\"1/(1-Percentile)\"\n");
}
else
{
printStream.Write("{0,12} {1,14} {2,10} {3,14}\n\n", "Value", "Percentile", "TotalCount", "1/(1-Percentile)");
}
string percentileFormatString;
string lastLinePercentileFormatString;
if (useCsvFormat)
{
percentileFormatString = "{0:F" + histogram.NumberOfSignificantValueDigits + "},{1:F12},{2},{3:F2}\n";
lastLinePercentileFormatString = "{0:F" + histogram.NumberOfSignificantValueDigits + "},{1:F12},{2},Infinity\n";
}
else
{
percentileFormatString = "{0,12:F" + histogram.NumberOfSignificantValueDigits + "}" + " {1,2:F12} {2,10} {3,14:F2}\n";
lastLinePercentileFormatString = "{0,12:F" + histogram.NumberOfSignificantValueDigits + "} {1,2:F12} {2,10}\n";
}
try
{
foreach (var iterationValue in histogram.Percentiles(percentileTicksPerHalfDistance))
{
if (iterationValue.PercentileLevelIteratedTo != 100.0D)
{
printStream.Write(percentileFormatString,
iterationValue.ValueIteratedTo / outputValueUnitScalingRatio,
iterationValue.PercentileLevelIteratedTo / 100.0D,
iterationValue.TotalCountToThisValue,
1 / (1.0D - (iterationValue.PercentileLevelIteratedTo / 100.0D)));
}
else
{
printStream.Write(lastLinePercentileFormatString,
iterationValue.ValueIteratedTo / outputValueUnitScalingRatio,
iterationValue.PercentileLevelIteratedTo / 100.0D,
iterationValue.TotalCountToThisValue);
}
}
if (!useCsvFormat)
{
// Calculate and output mean and std. deviation.
// Note: mean/std. deviation numbers are very often completely irrelevant when
// data is extremely non-normal in distribution (e.g. in cases of strong multi-modal
// response time distribution associated with GC pauses). However, reporting these numbers
// can be very useful for contrasting with the detailed percentile distribution
// reported by outputPercentileDistribution(). It is not at all surprising to find
// percentile distributions where results fall many tens or even hundreds of standard
// deviations away from the mean - such results simply indicate that the data sampled
// exhibits a very non-normal distribution, highlighting situations for which the std.
// deviation metric is a useless indicator.
var mean = histogram.GetMean() / outputValueUnitScalingRatio;
var stdDeviation = histogram.GetStdDeviation() / outputValueUnitScalingRatio;
printStream.Write("#[Mean = {0,12:F" + histogram.NumberOfSignificantValueDigits + "}, " +
"StdDeviation = {1,12:F" + histogram.NumberOfSignificantValueDigits + "}]\n", mean, stdDeviation);
printStream.Write("#[Max = {0,12:F" + histogram.NumberOfSignificantValueDigits + "}, Total count = {1,12}]\n",
histogram.GetMaxValue() / outputValueUnitScalingRatio, histogram.TotalCount);
printStream.Write("#[Buckets = {0,12}, SubBuckets = {1,12}]\n",
histogram.BucketCount, histogram.SubBucketCount);
}
}
catch (ArgumentOutOfRangeException)
{
// Overflow conditions on histograms can lead to ArgumentOutOfRangeException on iterations:
if (histogram.HasOverflowed())
{
printStream.Write("# Histogram counts indicate OVERFLOW values");
}
else
{
// Re-throw if reason is not a known overflow:
throw;
}
}
}
/// <summary>
/// Get the computed mean value of all recorded values in the histogram
/// </summary>
/// <returns>the mean value (in value units) of the histogram data</returns>
public static double GetMean(this HistogramBase histogram)
{
var totalValue = histogram.RecordedValues().Select(hiv => hiv.TotalValueToThisValue).LastOrDefault();
return((totalValue * 1.0) / histogram.TotalCount);
}
/// <summary>
/// Provide a means of iterating through histogram values according to percentile levels.
/// The iteration is performed in steps that start at 0% and reduce their distance to 100% according to the
/// <paramref name="percentileTicksPerHalfDistance"/> parameter, ultimately reaching 100% when all recorded
/// histogram values are exhausted.
/// </summary>
/// <param name="histogram">The histogram to operate on</param>
/// <param name="percentileTicksPerHalfDistance">
/// The number of iteration steps per half-distance to 100%.
/// </param>
/// <returns>
/// An enumerator of <see cref="HistogramIterationValue"/> through the histogram using a
/// <see cref="PercentileEnumerator"/>.
/// </returns>
public static IEnumerable <HistogramIterationValue> Percentiles(this HistogramBase histogram, int percentileTicksPerHalfDistance)
{
return(new PercentileEnumerable(histogram, percentileTicksPerHalfDistance));
}
/// <summary>
/// Get the highest value that is equivalent to the given value within the histogram's resolution.
/// Where "equivalent" means that value samples recorded for any two equivalent values are counted in a common
/// total count.
/// </summary>
/// <param name="histogram">The histogram to operate on</param>
/// <param name="value">The given value</param>
/// <returns>The highest value that is equivalent to the given value within the histogram's resolution.</returns>
public static long HighestEquivalentValue(this HistogramBase histogram, long value)
{
return(histogram.NextNonEquivalentValue(value) - 1);
}
/// <summary>
/// Get the highest recorded value level in the histogram
/// </summary>
/// <returns>the Max value recorded in the histogram</returns>
public static long GetMaxValue(this HistogramBase histogram)
{
var max = histogram.RecordedValues().Select(hiv => hiv.ValueIteratedTo).LastOrDefault();
return(histogram.HighestEquivalentValue(max));
}
/// <summary>
/// Get a new instance of an interval histogram, which will include a stable, consistent view of all value counts accumulated since the last interval histogram was taken.
/// Calling <see cref="GetIntervalHistogram()"/> will reset the value counts, and start accumulating value counts for the next interval.
/// </summary>
/// <param name="histogramToRecycle">a previously returned interval histogram that may be recycled to avoid allocation and copy operations.</param>
/// <returns>A histogram containing the value counts accumulated since the last interval histogram was taken.</returns>
/// <remarks>
/// <see cref="GetIntervalHistogram(HistogramBase)"/> accepts a previously returned interval histogram that can be recycled internally to avoid allocation and content copying operations.
/// It is therefore significantly more efficient for repeated use than <see cref="GetIntervalHistogram()"/> and <see cref="GetIntervalHistogramInto(HistogramBase)"/>.
/// The provided <paramref name="histogramToRecycle"/> must be either be null or an interval histogram returned by a previous call to <see cref="GetIntervalHistogram(HistogramBase)"/> or <see cref="GetIntervalHistogram()"/>.
/// NOTE: The caller is responsible for not recycling the same returned interval histogram more than once.
/// If the same interval histogram instance is recycled more than once, behavior is undefined.
/// </remarks>
public HistogramBase GetIntervalHistogram(HistogramBase histogramToRecycle)
{
lock (_gate)
{
// Verify that replacement histogram can validly be used as an inactive histogram replacement:
ValidateFitAsReplacementHistogram(histogramToRecycle);
_inactiveHistogram = histogramToRecycle;
PerformIntervalSample();
var sampledHistogram = _inactiveHistogram;
_inactiveHistogram = null; // Once we expose the sample, we can't reuse it internally until it is recycled
return sampledHistogram;
}
}
/// <summary>
/// Provide a means of iterating through histogram values at logarithmically increasing levels.
/// The iteration is performed in steps that start at<i>valueUnitsInFirstBucket</i> and increase exponentially
/// according to <paramref name="logBase"/>, terminating when all recorded histogram values are exhausted.
/// </summary>
/// <param name="histogram">The histogram to operate on</param>
/// <param name="valueUnitsInFirstBucket">The size (in value units) of the first bucket in the iteration</param>
/// <param name="logBase">The multiplier by which bucket sizes will grow in each iteration step</param>
/// <returns>
/// An enumerator of <see cref="HistogramIterationValue"/> through the histogram using a
/// <see cref="LogarithmicEnumerator"/>.
/// </returns>
public static IEnumerable <HistogramIterationValue> LogarithmicBucketValues(this HistogramBase histogram, int valueUnitsInFirstBucket, double logBase)
{
return(new LogarithmicBucketEnumerable(histogram, valueUnitsInFirstBucket, logBase));
}
/// <summary>
/// Add the contents of another histogram to this one.
/// </summary>
/// <param name="fromHistogram">The other histogram.</param>
/// <exception cref="System.IndexOutOfRangeException">if values in fromHistogram's are higher than highestTrackableValue.</exception>
public virtual void Add(HistogramBase fromHistogram)
{
if (HighestTrackableValue < fromHistogram.HighestTrackableValue)
{
throw new ArgumentOutOfRangeException(nameof(fromHistogram), $"The other histogram covers a wider range ({fromHistogram.HighestTrackableValue} than this one ({HighestTrackableValue}).");
}
if ((BucketCount == fromHistogram.BucketCount) &&
(SubBucketCount == fromHistogram.SubBucketCount) &&
(_unitMagnitude == fromHistogram._unitMagnitude))
{
// Counts arrays are of the same length and meaning, so we can just iterate and add directly:
for (var i = 0; i < fromHistogram.CountsArrayLength; i++)
{
AddToCountAtIndex(i, fromHistogram.GetCountAtIndex(i));
}
}
else
{
// Arrays are not a direct match, so we can't just stream through and add them.
// Instead, go through the array and add each non-zero value found at it's proper value:
for (var i = 0; i < fromHistogram.CountsArrayLength; i++)
{
var count = fromHistogram.GetCountAtIndex(i);
RecordValueWithCount(fromHistogram.ValueFromIndex(i), count);
}
}
}
public void Reset(CountdownEvent globalSignal, ManualResetEvent signal, HistogramBase histogram)
{
_globalSignal = globalSignal;
_signal = signal;
_histogram = histogram;
_counter = 0;
}
/// <summary>
/// Copy this histogram, corrected for coordinated omission, into the target histogram, overwriting it's contents.
/// </summary>
/// <param name="targetHistogram">the histogram to copy into</param>
/// <param name="expectedIntervalBetweenValueSamples">If <paramref name="expectedIntervalBetweenValueSamples"/> is larger than 0, add auto-generated value records as appropriate if value is larger than <paramref name="expectedIntervalBetweenValueSamples"/></param>
/// <remarks>
/// See <see cref="CopyCorrectedForCoordinatedOmission"/> for more detailed explanation about how correction is applied
/// </remarks>
public void CopyIntoCorrectedForCoordinatedOmission(HistogramBase targetHistogram, long expectedIntervalBetweenValueSamples)
{
targetHistogram.Reset();
targetHistogram.AddWhileCorrectingForCoordinatedOmission(this, expectedIntervalBetweenValueSamples);
targetHistogram.StartTimeStamp = StartTimeStamp;
targetHistogram.EndTimeStamp = EndTimeStamp;
}
