public void TestAdd()
{
var longHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
var other = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
longHistogram.RecordValue(TestValueLevel);
longHistogram.RecordValue(TestValueLevel * 1000);
other.RecordValue(TestValueLevel);
other.RecordValue(TestValueLevel * 1000);
longHistogram.Add(other);
Assert.AreEqual(2L, longHistogram.GetCountAtValue(TestValueLevel));
Assert.AreEqual(2L, longHistogram.GetCountAtValue(TestValueLevel * 1000));
Assert.AreEqual(4L, longHistogram.TotalCount);
var biggerOther = new LongHistogram(HighestTrackableValue * 2, NumberOfSignificantValueDigits);
biggerOther.RecordValue(TestValueLevel);
biggerOther.RecordValue(TestValueLevel * 1000);
// Adding the smaller histogram to the bigger one should work:
biggerOther.Add(longHistogram);
Assert.AreEqual(3L, biggerOther.GetCountAtValue(TestValueLevel));
Assert.AreEqual(3L, biggerOther.GetCountAtValue(TestValueLevel * 1000));
Assert.AreEqual(6L, biggerOther.TotalCount);
// But trying to add a larger histogram into a smaller one should throw an AIOOB:
Assert.Throws <ArgumentOutOfRangeException>(() => { longHistogram.Add(biggerOther); });
}
private void WriteToDisk(Recorder recorder)
{
//Sample every second until flagged as completed.
var accumulatingHistogram = new LongHistogram(TimeStamp.Hours(1), 3);
while (_isCompleted == 0)
{
Thread.Sleep(1000);
var histogram = recorder.GetIntervalHistogram();
accumulatingHistogram.Add(histogram);
_logWriter.Append(histogram);
Console.WriteLine($"{DateTime.Now:o} Interval.TotalCount = {histogram.TotalCount,10:G}. Accumulated.TotalCount = {accumulatingHistogram.TotalCount,10:G}.");
}
_logWriter.Dispose();
_outputStream.Dispose();
Console.WriteLine("Log contents");
Console.WriteLine(File.ReadAllText(LogPath));
Console.WriteLine();
Console.WriteLine("Percentile distribution (values reported in milliseconds)");
accumulatingHistogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToMilliseconds);
Console.WriteLine("Output thread finishing.");
}
private void WriteToDisk(Recorder recorder)
{
//Sample every second until flagged as completed.
var accumulatingHistogram = new LongHistogram(TimeStamp.Hours(1), 3);
while (_isCompleted == 0)
{
Thread.Sleep(1000);
var histogram = recorder.GetIntervalHistogram();
accumulatingHistogram.Add(histogram);
_logWriter.Append(histogram);
Console.WriteLine($"{DateTime.Now:o} Interval.TotalCount = {histogram.TotalCount,10:G}. Accumulated.TotalCount = {accumulatingHistogram.TotalCount,10:G}.");
}
_logWriter.Dispose();
_outputStream.Dispose();
Console.WriteLine("Log contents");
Console.WriteLine(File.ReadAllText(LogPath));
Console.WriteLine();
Console.WriteLine("Percentile distribution (values reported in milliseconds)");
accumulatingHistogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToMilliseconds);
Console.WriteLine("Output thread finishing.");
}
public void CanReadv1Logs_Skip_PreStart(string logPath, int skip, int take,
int expectedHistogramCount, int expectedCombinedValueCount,
int expectedCombined999, long expectedCombinedMaxLength,
double expectedStartTime)
{
var readerStream = GetEmbeddedFileStream(logPath);
var reader = new HistogramLogReader(readerStream);
int histogramCount = 0;
long totalCount = 0;
HistogramBase accumulatedHistogram = new LongHistogram(3600L * 1000 * 1000 * 1000, 3);
var histograms = reader.ReadHistograms()
.Where(h => h.StartTimeStamp >= reader.GetStartTime().MillisecondsSinceUnixEpoch())
.Skip(skip)
.Take(take);
foreach (var histogram in histograms)
{
histogramCount++;
totalCount += histogram.TotalCount;
accumulatedHistogram.Add(histogram);
}
Assert.AreEqual(expectedHistogramCount, histogramCount);
Assert.AreEqual(expectedCombinedValueCount, totalCount);
Assert.AreEqual(expectedCombined999, accumulatedHistogram.GetValueAtPercentile(99.9));
Assert.AreEqual(expectedCombinedMaxLength, accumulatedHistogram.GetMaxValue());
Assert.AreEqual(expectedStartTime, reader.GetStartTime().SecondsSinceUnixEpoch());
}
static void DocumentResults(LongHistogram accumulatingHistogram, Recorder recorder)
{
recorder?.RecordValue(GC.GetTotalMemory(false));
var histogram = recorder?.GetIntervalHistogram();
accumulatingHistogram?.Add(histogram);
_logWriter?.Append(histogram);
RILogManager.Default?.SendDebug($"Accumulated.TotalCount = {accumulatingHistogram.TotalCount,10:G}.");
RILogManager.Default?.SendDebug("Mean: " + BytesToString(accumulatingHistogram.GetMean()) + ", StdDev: " +
BytesToString(accumulatingHistogram.GetStdDeviation()));
}
public void CanReadv2Logs(string logPath)
{
var readerStream = GetEmbeddedFileStream(logPath);
var reader = new HistogramLogReader(readerStream);
int histogramCount = 0;
long totalCount = 0;
var accumulatedHistogram = new LongHistogram(85899345920838, 3);
foreach (var histogram in reader.ReadHistograms())
{
histogramCount++;
Assert.IsInstanceOf <HistogramBase>(histogram, "Expected integer value histograms in log file");
totalCount += histogram.TotalCount;
accumulatedHistogram.Add(histogram);
}
Assert.AreEqual(62, histogramCount);
Assert.AreEqual(48761, totalCount);
Assert.AreEqual(1745879039, accumulatedHistogram.GetValueAtPercentile(99.9));
Assert.AreEqual(1796210687, accumulatedHistogram.GetMaxValue());
Assert.AreEqual(1441812279.474, reader.GetStartTime().SecondsSinceUnixEpoch());
}
public void TestAdd()
{
var longHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
var other = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
longHistogram.RecordValue(TestValueLevel);
longHistogram.RecordValue(TestValueLevel * 1000);
other.RecordValue(TestValueLevel);
other.RecordValue(TestValueLevel * 1000);
longHistogram.Add(other);
Assert.AreEqual(2L, longHistogram.GetCountAtValue(TestValueLevel));
Assert.AreEqual(2L, longHistogram.GetCountAtValue(TestValueLevel * 1000));
Assert.AreEqual(4L, longHistogram.TotalCount);
var biggerOther = new LongHistogram(HighestTrackableValue * 2, NumberOfSignificantValueDigits);
biggerOther.RecordValue(TestValueLevel);
biggerOther.RecordValue(TestValueLevel * 1000);
// Adding the smaller histogram to the bigger one should work:
biggerOther.Add(longHistogram);
Assert.AreEqual(3L, biggerOther.GetCountAtValue(TestValueLevel));
Assert.AreEqual(3L, biggerOther.GetCountAtValue(TestValueLevel * 1000));
Assert.AreEqual(6L, biggerOther.TotalCount);
// But trying to add a larger histogram into a smaller one should throw an AIOOB:
Assert.Throws<ArgumentOutOfRangeException>(() => { longHistogram.Add(biggerOther); });
}
public void CanReadv2Logs(string logPath)
{
var readerStream = GetEmbeddedFileStream(logPath);
var reader = new HistogramLogReader(readerStream);
int histogramCount = 0;
long totalCount = 0;
var accumulatedHistogram = new LongHistogram(85899345920838, 3);
foreach (var histogram in reader.ReadHistograms())
{
histogramCount++;
Assert.IsInstanceOf<HistogramBase>(histogram, "Expected integer value histograms in log file");
totalCount += histogram.TotalCount;
accumulatedHistogram.Add(histogram);
}
Assert.AreEqual(62, histogramCount);
Assert.AreEqual(48761, totalCount);
Assert.AreEqual(1745879039, accumulatedHistogram.GetValueAtPercentile(99.9));
Assert.AreEqual(1796210687, accumulatedHistogram.GetMaxValue());
Assert.AreEqual(1441812279.474, reader.GetStartTime().SecondsSinceUnixEpoch());
}
public void CanReadv1Logs_Skip_PreStart(string logPath, int skip, int take,
int expectedHistogramCount, int expectedCombinedValueCount,
int expectedCombined999, long expectedCombinedMaxLength,
double expectedStartTime)
{
var readerStream = GetEmbeddedFileStream(logPath);
var reader = new HistogramLogReader(readerStream);
int histogramCount = 0;
long totalCount = 0;
HistogramBase accumulatedHistogram = new LongHistogram(3600L * 1000 * 1000 * 1000, 3);
var histograms = reader.ReadHistograms()
.Where(h => h.StartTimeStamp >= reader.GetStartTime().MillisecondsSinceUnixEpoch())
.Skip(skip)
.Take(take);
foreach (var histogram in histograms)
{
histogramCount++;
totalCount += histogram.TotalCount;
accumulatedHistogram.Add(histogram);
}
Assert.AreEqual(expectedHistogramCount, histogramCount);
Assert.AreEqual(expectedCombinedValueCount, totalCount);
Assert.AreEqual(expectedCombined999, accumulatedHistogram.GetValueAtPercentile(99.9));
Assert.AreEqual(expectedCombinedMaxLength, accumulatedHistogram.GetMaxValue());
Assert.AreEqual(expectedStartTime, reader.GetStartTime().SecondsSinceUnixEpoch());
}
public void CouldReadSortedMapNewValuesWhileTheyAreAddedUsingCursor_NoSemaphore()
{
var cts = new CancellationTokenSource();
var ct = CancellationToken.None; // cts.Token; //
var count = 10000000;
var sw = new Stopwatch();
sw.Start();
var sm = new SortedMap <DateTime, double>();
sm.IsSynchronized = true;
//var sm = new SortedChunkedMap<DateTime, double>();
//sm.Add(DateTime.UtcNow.Date.AddSeconds(-2), 0);
for (int i = 0; i < 5; i++)
{
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
var histogram = new LongHistogram(TimeSpan.TicksPerMillisecond * 100 * 1000, 3);
double sum = 0;
var cnt = 0;
var histogram1 = new LongHistogram(TimeSpan.TicksPerMillisecond * 100 * 1000, 3);
var sumTask = Task.Run(async() =>
{
var c = sm.GetCursor();
var startTick = sw.ElapsedTicks;
while (await c.MoveNext(ct))
{
sum += c.CurrentValue;
if ((int)c.CurrentValue != cnt)
{
//Console.WriteLine("Wrong sequence");
//Assert.Fail($"Wrong sequence: {c.CurrentValue} != {cnt}");
Trace.WriteLine($"Wrong sequence1: {c.CurrentValue} != {cnt}; thread {Thread.CurrentThread.ManagedThreadId}");
}
else
{
//Console.WriteLine("Async move");
}
cnt++;
var ticks = sw.ElapsedTicks - startTick;
var nanos = (long)(1000000000.0 * (double)ticks / Stopwatch.Frequency);
try
{
histogram1.RecordValue(nanos);
}
catch (Exception e)
{
Console.WriteLine($"Nanos: {nanos}; " + e.Message);
}
startTick = sw.ElapsedTicks;
}
});
double sum2 = 0;
var cnt2 = 0;
var histogram2 = new LongHistogram(TimeSpan.TicksPerMillisecond * 100 * 1000, 3);
var sumTask2 = Task.Run(async() =>
{
var c = sm.GetCursor();
var startTick = sw.ElapsedTicks;
while (await c.MoveNext(ct))
{
sum2 += c.CurrentValue;
if ((int)c.CurrentValue != cnt2)
{
//Console.WriteLine("Wrong sequence");
//Assert.Fail($"Wrong sequence: {c.CurrentValue} != {cnt}");
Trace.WriteLine($"Wrong sequence2: {c.CurrentValue} != {cnt2}; thread {Thread.CurrentThread.ManagedThreadId}");
}
else
{
//Console.WriteLine("Async move");
}
cnt2++;
var ticks = sw.ElapsedTicks - startTick;
var nanos = (long)(1000000000.0 * (double)ticks / Stopwatch.Frequency);
try
{
histogram2.RecordValue(nanos);
}
catch (Exception e)
{
Console.WriteLine($"Nanos: {nanos}; " + e.Message);
}
startTick = sw.ElapsedTicks;
}
});
double sum3 = 0;
var cnt3 = 0;
var histogram3 = new LongHistogram(TimeSpan.TicksPerMillisecond * 100 * 1000, 3);
var sumTask3 = Task.Run(async() =>
{
var c = sm.GetCursor();
var startTick = sw.ElapsedTicks;
while (await c.MoveNext(ct))
{
sum3 += c.CurrentValue;
if ((int)c.CurrentValue != cnt3)
{
//Console.WriteLine("Wrong sequence");
//Assert.Fail($"Wrong sequence: {c.CurrentValue} != {cnt}");
Trace.WriteLine($"Wrong sequence3: {c.CurrentValue} != {cnt3}; thread {Thread.CurrentThread.ManagedThreadId}");
}
else
{
//Console.WriteLine("Async move");
}
cnt3++;
var ticks = sw.ElapsedTicks - startTick;
var nanos = (long)(1000000000.0 * (double)ticks / Stopwatch.Frequency);
try
{
histogram3.RecordValue(nanos);
}
catch (Exception e)
{
Console.WriteLine($"Nanos: {nanos}; " + e.Message);
}
startTick = sw.ElapsedTicks;
}
});
Thread.Sleep(1);
var addTask = Task.Run(() =>
{
//Console.WriteLine($"Adding from thread {Thread.CurrentThread.ManagedThreadId}");
try
{
for (int i = 5; i < count; i++)
{
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
sm.Complete();
}
catch (Exception ex)
{
Console.WriteLine(ex);
Environment.FailFast(ex.Message, ex);
}
});
while (!sumTask.Wait(2000))
{
OptimizationSettings.Verbose = true;
Trace.WriteLine($"cnt: {cnt}");
}
while (!sumTask2.Wait(2000))
{
//OptimizationSettings.Verbose = true;
Trace.WriteLine($"cnt2: {cnt2}");
}
while (!sumTask3.Wait(2000))
{
//OptimizationSettings.Verbose = true;
Trace.WriteLine($"cnt3: {cnt3}");
}
addTask.Wait();
histogram.Add(histogram1);
histogram.Add(histogram2);
histogram.Add(histogram3);
histogram.OutputPercentileDistribution(
writer: Console.Out,
percentileTicksPerHalfDistance: 3,
outputValueUnitScalingRatio: OutputScalingFactor.None);
sw.Stop();
Trace.Write($"Elapsed msec: {sw.ElapsedMilliseconds}; ");
Trace.WriteLine($"Ops: {Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2)}");
double expectedSum = 0.0;
for (int i = 0; i < count; i++)
{
expectedSum += i;
}
if (expectedSum != sum)
{
Trace.WriteLine("Sum 1 is wrong");
}
if (expectedSum != sum2)
{
Trace.WriteLine("Sum 2 is wrong");
}
if (expectedSum != sum3)
{
Trace.WriteLine("Sum 3 is wrong");
}
Assert.AreEqual(expectedSum, sum, "Sum 1");
Assert.AreEqual(expectedSum, sum2, "Sum 2");
Assert.AreEqual(expectedSum, sum3, "Sum 3");
//sm.Dispose();
}
