static HistogramDataAccessTest()
{
LongHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
ScaledHistogram = new LongHistogram(1000, HighestTrackableValue * 512, NumberOfSignificantValueDigits);
RawHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
var scaledRawHistogram = new LongHistogram(1000, HighestTrackableValue * 512, NumberOfSignificantValueDigits);
// Log hypothetical scenario: 100 seconds of "perfect" 1msec results, sampled
// 100 times per second (10,000 results), followed by a 100 second pause with
// a single (100 second) recorded result. Recording is done indicating an expected
// interval between samples of 10 msec:
for (var i = 0; i < 10000; i++)
{
LongHistogram.RecordValueWithExpectedInterval(1000 /* 1 msec */, 10000 /* 10 msec expected interval */);
ScaledHistogram.RecordValueWithExpectedInterval(1000 * 512 /* 1 msec */, 10000 * 512 /* 10 msec expected interval */);
RawHistogram.RecordValue(1000 /* 1 msec */);
scaledRawHistogram.RecordValue(1000 * 512/* 1 msec */);
}
LongHistogram.RecordValueWithExpectedInterval(100000000L /* 100 sec */, 10000 /* 10 msec expected interval */);
ScaledHistogram.RecordValueWithExpectedInterval(100000000L * 512 /* 100 sec */, 10000 * 512 /* 10 msec expected interval */);
RawHistogram.RecordValue(100000000L /* 100 sec */);
scaledRawHistogram.RecordValue(100000000L * 512 /* 100 sec */);
PostCorrectedHistogram = RawHistogram.CopyCorrectedForCoordinatedOmission(10000 /* 10 msec expected interval */);
PostCorrectedScaledHistogram = scaledRawHistogram.CopyCorrectedForCoordinatedOmission(10000 * 512 /* 10 msec expected interval */);
}
static HistogramDataAccessTest()
{
LongHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
ScaledHistogram = new LongHistogram(1000, HighestTrackableValue * 512, NumberOfSignificantValueDigits);
RawHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
var scaledRawHistogram = new LongHistogram(1000, HighestTrackableValue * 512, NumberOfSignificantValueDigits);
// Log hypothetical scenario: 100 seconds of "perfect" 1msec results, sampled
// 100 times per second (10,000 results), followed by a 100 second pause with
// a single (100 second) recorded result. Recording is done indicating an expected
// interval between samples of 10 msec:
for (var i = 0; i < 10000; i++)
{
LongHistogram.RecordValueWithExpectedInterval(1000 /* 1 msec */, 10000 /* 10 msec expected interval */);
ScaledHistogram.RecordValueWithExpectedInterval(1000 * 512 /* 1 msec */, 10000 * 512 /* 10 msec expected interval */);
RawHistogram.RecordValue(1000 /* 1 msec */);
scaledRawHistogram.RecordValue(1000 * 512 /* 1 msec */);
}
LongHistogram.RecordValueWithExpectedInterval(100000000L /* 100 sec */, 10000 /* 10 msec expected interval */);
ScaledHistogram.RecordValueWithExpectedInterval(100000000L * 512 /* 100 sec */, 10000 * 512 /* 10 msec expected interval */);
RawHistogram.RecordValue(100000000L /* 100 sec */);
scaledRawHistogram.RecordValue(100000000L * 512 /* 100 sec */);
PostCorrectedHistogram = RawHistogram.CopyCorrectedForCoordinatedOmission(10000 /* 10 msec expected interval */);
PostCorrectedScaledHistogram = scaledRawHistogram.CopyCorrectedForCoordinatedOmission(10000 * 512 /* 10 msec expected interval */);
}
public void TestAdd()
{
var longHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
var other = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
longHistogram.RecordValue(TestValueLevel);
longHistogram.RecordValue(TestValueLevel * 1000);
other.RecordValue(TestValueLevel);
other.RecordValue(TestValueLevel * 1000);
longHistogram.Add(other);
Assert.AreEqual(2L, longHistogram.GetCountAtValue(TestValueLevel));
Assert.AreEqual(2L, longHistogram.GetCountAtValue(TestValueLevel * 1000));
Assert.AreEqual(4L, longHistogram.TotalCount);
var biggerOther = new LongHistogram(HighestTrackableValue * 2, NumberOfSignificantValueDigits);
biggerOther.RecordValue(TestValueLevel);
biggerOther.RecordValue(TestValueLevel * 1000);
// Adding the smaller histogram to the bigger one should work:
biggerOther.Add(longHistogram);
Assert.AreEqual(3L, biggerOther.GetCountAtValue(TestValueLevel));
Assert.AreEqual(3L, biggerOther.GetCountAtValue(TestValueLevel * 1000));
Assert.AreEqual(6L, biggerOther.TotalCount);
// But trying to add a larger histogram into a smaller one should throw an AIOOB:
Assert.Throws <ArgumentOutOfRangeException>(() => { longHistogram.Add(biggerOther); });
}
private static void Load(LongHistogram source)
{
for (long i = 0L; i < 10000L; i++)
{
source.RecordValue(1000L * i);
}
}
public void TestRecordValue()
{
var longHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
longHistogram.RecordValue(TestValueLevel);
Assert.AreEqual(1L, longHistogram.GetCountAtValue(TestValueLevel));
Assert.AreEqual(1L, longHistogram.TotalCount);
}
public void TestRecordValue()
{
var longHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
longHistogram.RecordValue(TestValueLevel);
Assert.AreEqual(1L, longHistogram.GetCountAtValue(TestValueLevel));
Assert.AreEqual(1L, longHistogram.TotalCount);
}
private static void PongHandler(IDirectBuffer buffer, int offset, int length, Header header)
{
var pingTimestamp = buffer.GetLong(offset);
var rttNs = Stopwatch.GetTimestamp() - pingTimestamp;
var b = rttNs * 1000 * 1000 * 1000d / Stopwatch.Frequency;
Histogram.RecordValue((long)b);
}
public void TestGetValueAtPercentileForLargeHistogram()
{
const long largestValue = 1000000000000L;
var h = new LongHistogram(largestValue, 5);
h.RecordValue(largestValue);
Assert.That(h.GetValueAtPercentile(100.0) > 0);
}
private static LongHistogram CreatePopulatedHistogram(long multiplier)
{
var histogram = new LongHistogram(3600L * 1000 * 1000, 3);
for (int i = 0; i < 10000; i++)
{
histogram.RecordValue(i * multiplier);
}
return histogram;
}
private static LongHistogram CreatePopulatedHistogram(long multiplier)
{
var histogram = new LongHistogram(3600L * 1000 * 1000, 3);
for (int i = 0; i < 10000; i++)
{
histogram.RecordValue(i * multiplier);
}
return(histogram);
}
public async void AmazonIotStreamer([FromQuery] int messageToSend = 10, [FromQuery] int delayBtwMessageInMs = 50)
{
var rd = new Random(100);
var histogram = new LongHistogram(TimeStamp.Hours(1), 3);
var writer = new StringWriter();
var messages = messageToSend;
string[] currencyPairs = new string[] { "EUR/USD", "EUR/JPY", "EUR/GBP", "USD/JPY", "USD/GBP" };
for (int currIndex = 0; currIndex < currencyPairs.Length; currIndex++)
{
var pair = currencyPairs[currIndex];
Task.Run(async() =>
{
var publisher = new AmazonIotDataClient(IotEndpoint, new BasicAWSCredentials(IotAccessKey, IotSecret));
for (var i = 1; i <= messages; i++)
{
long startTimestamp = Stopwatch.GetTimestamp();
long timestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds();
var currency = new Currency()
{
Id = i,
CurrencyType = pair,
Price = rd.NextDouble(),
Timestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds().ToString(),
Ladders = new LadderFactory().Build(10)
};
var cur = JsonConvert.SerializeObject(currency);
publisher.PublishAsync(new Amazon.IotData.Model.PublishRequest()
{
Topic = pair,
Payload = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(cur)),
Qos = 0
});
long elapsed = Stopwatch.GetTimestamp() - startTimestamp;
histogram.RecordValue(elapsed);
await Task.Delay(delayBtwMessageInMs).ConfigureAwait(false);
}
var scalingRatio = OutputScalingFactor.TimeStampToMilliseconds;
histogram.OutputPercentileDistribution(
writer,
outputValueUnitScalingRatio: scalingRatio);
System.IO.File.WriteAllText(@"d:\cloud\appsync.txt", writer.ToString());
});
}
}
public long LongHistogramRecording()
{
long counter = 0L;
for (int i = 0; i < _testValues.Length; i++)
{
var value = _testValues[i];
_longHistogram.RecordValue(value);
counter += value;
}
return(counter);
}
private static void RunClient(IFeedClient client)
{
var histogram = new LongHistogram(TimeSpan.FromSeconds(10).Ticks, 2);
var receivedMessageCount = 0;
void OnMessageReceived(ReadOnlySpan <byte> message)
{
var now = Stopwatch.GetTimestamp();
var start = Unsafe.ReadUnaligned <long>(ref MemoryMarshal.GetReference(message));
var rrt = now - start;
var latencyInMicroseconds = unchecked (rrt * 1_000_000 / (double)Stopwatch.Frequency);
receivedMessageCount++;
histogram.RecordValue((long)latencyInMicroseconds);
}
using (client)
{
var connectedSignal = new AutoResetEvent(false);
client.Connected += () => Console.WriteLine($"Connected {connectedSignal.Set()}");
client.MessageReceived += OnMessageReceived;
client.Start("client " + Guid.NewGuid());
connectedSignal.WaitOne();
do
{
Console.WriteLine("Bench? (<message count> <message size> <delay in micro> <burst>, eg.: 100000 128 10 1)");
var benchArgs = Console.ReadLine();
if (!TryParseBenchArgs(benchArgs, out var args))
{
break;
}
histogram.Reset();
RunBench(client, args, ref receivedMessageCount);
histogram.OutputPercentileDistribution(Console.Out, 1);
Console.WriteLine("FailSends : " + Counters.FailedReceivingNextCount);
Console.WriteLine("FailReceives : " + Counters.FailedReceivingNextCount);
Counters.Reset();
} while (true);
client.Stop();
}
}
public void TestScaledCopy()
{
var longHistogram = new LongHistogram(1000, HighestTrackableValue, NumberOfSignificantValueDigits);
longHistogram.RecordValue(TestValueLevel);
longHistogram.RecordValue(TestValueLevel * 10);
longHistogram.RecordValueWithExpectedInterval(longHistogram.HighestTrackableValue - 1, 31000);
Console.WriteLine("Testing copy of scaled Histogram:");
AssertEqual(longHistogram, longHistogram.Copy());
var intHistogram = new IntHistogram(1000, HighestTrackableValue, NumberOfSignificantValueDigits);
intHistogram.RecordValue(TestValueLevel);
intHistogram.RecordValue(TestValueLevel * 10);
intHistogram.RecordValueWithExpectedInterval(intHistogram.HighestTrackableValue - 1, 31000);
Console.WriteLine("Testing copy of scaled IntHistogram:");
AssertEqual(intHistogram, intHistogram.Copy());
var shortHistogram = new ShortHistogram(1000, HighestTrackableValue, NumberOfSignificantValueDigits);
shortHistogram.RecordValue(TestValueLevel);
shortHistogram.RecordValue(TestValueLevel * 10);
shortHistogram.RecordValueWithExpectedInterval(shortHistogram.HighestTrackableValue - 1, 31000);
Console.WriteLine("Testing copy of scaled ShortHistogram:");
AssertEqual(shortHistogram, shortHistogram.Copy());
var syncHistogram = new SynchronizedHistogram(1000, HighestTrackableValue, NumberOfSignificantValueDigits);
syncHistogram.RecordValue(TestValueLevel);
syncHistogram.RecordValue(TestValueLevel * 10);
syncHistogram.RecordValueWithExpectedInterval(syncHistogram.HighestTrackableValue - 1, 31000);
Console.WriteLine("Testing copy of scaled SynchronizedHistogram:");
AssertEqual(syncHistogram, syncHistogram.Copy());
}
public void GetIntervalHistogramInto_copies_data_over_provided_Histogram()
{
var originalStart = DateTime.Today.AddDays(-1).MillisecondsSinceUnixEpoch();
var originalEnd = DateTime.Today.MillisecondsSinceUnixEpoch();
var targetHistogram = new LongHistogram(DefautltLowestDiscernibleValue, DefaultHighestTrackableValue, DefaultSignificantFigures);
targetHistogram.StartTimeStamp = originalStart;
targetHistogram.RecordValue(1);
targetHistogram.RecordValue(10);
targetHistogram.RecordValue(100);
targetHistogram.EndTimeStamp = originalEnd;
Assert.AreEqual(3, targetHistogram.TotalCount);
Assert.AreEqual(1, targetHistogram.GetCountAtValue(1));
Assert.AreEqual(1, targetHistogram.GetCountAtValue(10));
Assert.AreEqual(1, targetHistogram.GetCountAtValue(100));
var recorder = Create(DefautltLowestDiscernibleValue, DefaultHighestTrackableValue, DefaultSignificantFigures);
recorder.RecordValue(1000);
recorder.RecordValue(10000);
recorder.RecordValue(100000);
recorder.GetIntervalHistogramInto(targetHistogram);
Assert.AreEqual(3, targetHistogram.TotalCount);
Assert.AreEqual(0, targetHistogram.GetCountAtValue(1));
Assert.AreEqual(0, targetHistogram.GetCountAtValue(10));
Assert.AreEqual(0, targetHistogram.GetCountAtValue(100));
Assert.AreEqual(1, targetHistogram.GetCountAtValue(1000));
Assert.AreEqual(1, targetHistogram.GetCountAtValue(10000));
Assert.AreEqual(1, targetHistogram.GetCountAtValue(100000));
Assert.AreNotEqual(originalStart, targetHistogram.StartTimeStamp);
Assert.AreNotEqual(originalEnd, targetHistogram.EndTimeStamp);
}
public static void Stream(Func <string, string, Random, string> createQuery, int messageToSend, int delayBtwMessageInMs, string apiKey, string endpointUrl)
{
var client = new HttpClient();
System.Net.ServicePointManager.SecurityProtocol =
SecurityProtocolType.Tls12 | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls;
var rd = new Random(100);
var histogram = new LongHistogram(TimeStamp.Hours(1), 3);
var writer = new StringWriter();
var messages = messageToSend;
Task.Run(async() =>
{
for (var i = 1; i <= messages; i++)
{
var req = new HttpRequestMessage();
req.Headers.TryAddWithoutValidation("x-api-key", apiKey);
req.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
req.Headers.TryAddWithoutValidation("Content-Type", "application/json");
req.Method = HttpMethod.Post;
req.RequestUri = new Uri(endpointUrl);
long startTimestamp = Stopwatch.GetTimestamp();
long timestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds();
req.Content = new StringContent(createQuery(i.ToString(), timestamp.ToString(), rd), Encoding.UTF8, "application/json");
await client.SendAsync((req)).ConfigureAwait(false);
long elapsed = Stopwatch.GetTimestamp() - startTimestamp;
histogram.RecordValue(elapsed);
await Task.Delay(delayBtwMessageInMs).ConfigureAwait(false);
}
var scalingRatio = OutputScalingFactor.TimeStampToMilliseconds;
histogram.OutputPercentileDistribution(
writer,
outputValueUnitScalingRatio: scalingRatio);
System.IO.File.WriteAllText(@"d:\cloud\appsync.txt", writer.ToString());
});
}
public void SignalRStreamer([FromQuery] int messageToSend = 10, [FromQuery] int delayBtwMessageInMs = 50)
{
var histogram = new LongHistogram(TimeStamp.Hours(1), 3);
var writer = new StringWriter();
string[] currencyPairs = new string[] { "EUR/USD", "EUR/JPY", "EUR/GBP", "USD/JPY", "USD/GBP" };
for (int currIndex = 0; currIndex < currencyPairs.Length; currIndex++)
{
var pair = currencyPairs[currIndex];
Task.Run(async() =>
{
var rd = new Random(100);
for (var i = 1; i <= messageToSend; i++)
{
long startTimestamp = Stopwatch.GetTimestamp();
var currency = new Currency()
{
Id = i,
CurrencyType = pair,
Price = rd.NextDouble(),
Timestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds().ToString(),
Ladders = new LadderFactory().Build(10)
};
var cur = JsonConvert.SerializeObject(currency);
await context.Clients.Group(pair).SendAsync("broadcastMessage", "currency", cur).ConfigureAwait(false);
long elapsed = Stopwatch.GetTimestamp() - startTimestamp;
histogram.RecordValue(elapsed);
await Task.Delay(delayBtwMessageInMs).ConfigureAwait(false);
}
var scalingRatio = OutputScalingFactor.TimeStampToMilliseconds;
histogram.OutputPercentileDistribution(
writer,
outputValueUnitScalingRatio: scalingRatio);
System.IO.File.WriteAllText(@"d:\cloud\signalr.txt", writer.ToString());
});
}
}
public void TestRecordValueWithExpectedInterval()
{
var longHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
longHistogram.RecordValueWithExpectedInterval(TestValueLevel, TestValueLevel / 4);
var rawHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
rawHistogram.RecordValue(TestValueLevel);
// The data will include corrected samples:
Assert.AreEqual(1L, longHistogram.GetCountAtValue((TestValueLevel * 1) / 4));
Assert.AreEqual(1L, longHistogram.GetCountAtValue((TestValueLevel * 2) / 4));
Assert.AreEqual(1L, longHistogram.GetCountAtValue((TestValueLevel * 3) / 4));
Assert.AreEqual(1L, longHistogram.GetCountAtValue((TestValueLevel * 4) / 4));
Assert.AreEqual(4L, longHistogram.TotalCount);
// But the raw data will not:
Assert.AreEqual(0L, rawHistogram.GetCountAtValue((TestValueLevel * 1) / 4));
Assert.AreEqual(0L, rawHistogram.GetCountAtValue((TestValueLevel * 2) / 4));
Assert.AreEqual(0L, rawHistogram.GetCountAtValue((TestValueLevel * 3) / 4));
Assert.AreEqual(1L, rawHistogram.GetCountAtValue((TestValueLevel * 4) / 4));
Assert.AreEqual(1L, rawHistogram.TotalCount);
}
public void TestRecordValueWithExpectedInterval()
{
var longHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
longHistogram.RecordValueWithExpectedInterval(TestValueLevel, TestValueLevel / 4);
var rawHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
rawHistogram.RecordValue(TestValueLevel);
// The data will include corrected samples:
Assert.AreEqual(1L, longHistogram.GetCountAtValue((TestValueLevel * 1) / 4));
Assert.AreEqual(1L, longHistogram.GetCountAtValue((TestValueLevel * 2) / 4));
Assert.AreEqual(1L, longHistogram.GetCountAtValue((TestValueLevel * 3) / 4));
Assert.AreEqual(1L, longHistogram.GetCountAtValue((TestValueLevel * 4) / 4));
Assert.AreEqual(4L, longHistogram.TotalCount);
// But the raw data will not:
Assert.AreEqual(0L, rawHistogram.GetCountAtValue((TestValueLevel * 1) / 4));
Assert.AreEqual(0L, rawHistogram.GetCountAtValue((TestValueLevel * 2) / 4));
Assert.AreEqual(0L, rawHistogram.GetCountAtValue((TestValueLevel * 3) / 4));
Assert.AreEqual(1L, rawHistogram.GetCountAtValue((TestValueLevel * 4) / 4));
Assert.AreEqual(1L, rawHistogram.TotalCount);
}
public void RecordValueWithExpectedInterval()
{
var TestValueLevel = 4L;
var recorder = Create(DefautltLowestDiscernibleValue, DefaultHighestTrackableValue, DefaultSignificantFigures);
var valueHistogram = new LongHistogram(DefaultHighestTrackableValue, DefaultSignificantFigures);
recorder.RecordValueWithExpectedInterval(TestValueLevel, TestValueLevel / 4);
valueHistogram.RecordValue(TestValueLevel);
var intervalHistogram = recorder.GetIntervalHistogram();
// The data will include corrected samples:
Assert.AreEqual(1L, intervalHistogram.GetCountAtValue((TestValueLevel * 1) / 4));
Assert.AreEqual(1L, intervalHistogram.GetCountAtValue((TestValueLevel * 2) / 4));
Assert.AreEqual(1L, intervalHistogram.GetCountAtValue((TestValueLevel * 3) / 4));
Assert.AreEqual(1L, intervalHistogram.GetCountAtValue((TestValueLevel * 4) / 4));
Assert.AreEqual(4L, intervalHistogram.TotalCount);
// But the raw data will not:
Assert.AreEqual(0L, valueHistogram.GetCountAtValue((TestValueLevel * 1) / 4));
Assert.AreEqual(0L, valueHistogram.GetCountAtValue((TestValueLevel * 2) / 4));
Assert.AreEqual(0L, valueHistogram.GetCountAtValue((TestValueLevel * 3) / 4));
Assert.AreEqual(1L, valueHistogram.GetCountAtValue((TestValueLevel * 4) / 4));
Assert.AreEqual(1L, valueHistogram.TotalCount);
}
public void TestGetValueAtPercentileForLargeHistogram()
{
const long largestValue = 1000000000000L;
var h = new LongHistogram(largestValue, 5);
h.RecordValue(largestValue);
Assert.That(h.GetValueAtPercentile(100.0) > 0);
}
public void RecordValueWithExpectedInterval()
{
var TestValueLevel = 4L;
var recorder = new Recorder(1, HighestTrackableValue, NumberOfSignificantValueDigits, (id, lowest, highest, sf) => new LongHistogram(id, lowest, highest, sf));
var valueHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
recorder.RecordValueWithExpectedInterval(TestValueLevel, TestValueLevel / 4);
valueHistogram.RecordValue(TestValueLevel);
var intervalHistogram = recorder.GetIntervalHistogram();
// The data will include corrected samples:
Assert.AreEqual(1L, intervalHistogram.GetCountAtValue((TestValueLevel * 1) / 4));
Assert.AreEqual(1L, intervalHistogram.GetCountAtValue((TestValueLevel * 2) / 4));
Assert.AreEqual(1L, intervalHistogram.GetCountAtValue((TestValueLevel * 3) / 4));
Assert.AreEqual(1L, intervalHistogram.GetCountAtValue((TestValueLevel * 4) / 4));
Assert.AreEqual(4L, intervalHistogram.TotalCount);
// But the raw data will not:
Assert.AreEqual(0L, valueHistogram.GetCountAtValue((TestValueLevel * 1) / 4));
Assert.AreEqual(0L, valueHistogram.GetCountAtValue((TestValueLevel * 2) / 4));
Assert.AreEqual(0L, valueHistogram.GetCountAtValue((TestValueLevel * 3) / 4));
Assert.AreEqual(1L, valueHistogram.GetCountAtValue((TestValueLevel * 4) / 4));
Assert.AreEqual(1L, valueHistogram.TotalCount);
}
public void testRecordValue_Overflow_ShouldThrowException()
{
var longHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
Assert.Catch<IndexOutOfRangeException>(()=>longHistogram.RecordValue(HighestTrackableValue * 3));
}
public void TestCopyInto()
{
var longHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
var targetLongHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
longHistogram.RecordValue(TestValueLevel);
longHistogram.RecordValue(TestValueLevel * 10);
longHistogram.RecordValueWithExpectedInterval(longHistogram.HighestTrackableValue - 1, 31000);
Console.WriteLine("Testing copyInto for Histogram:");
longHistogram.CopyInto(targetLongHistogram);
AssertEqual(longHistogram, targetLongHistogram);
longHistogram.RecordValue(TestValueLevel * 20);
longHistogram.CopyInto(targetLongHistogram);
AssertEqual(longHistogram, targetLongHistogram);
var intHistogram = new IntHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
var targetIntHistogram = new IntHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
intHistogram.RecordValue(TestValueLevel);
intHistogram.RecordValue(TestValueLevel * 10);
intHistogram.RecordValueWithExpectedInterval(intHistogram.HighestTrackableValue - 1, 31000);
Console.WriteLine("Testing copyInto for IntHistogram:");
intHistogram.CopyInto(targetIntHistogram);
AssertEqual(intHistogram, targetIntHistogram);
intHistogram.RecordValue(TestValueLevel * 20);
intHistogram.CopyInto(targetIntHistogram);
AssertEqual(intHistogram, targetIntHistogram);
var shortHistogram = new ShortHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
var targetShortHistogram = new ShortHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
shortHistogram.RecordValue(TestValueLevel);
shortHistogram.RecordValue(TestValueLevel * 10);
shortHistogram.RecordValueWithExpectedInterval(shortHistogram.HighestTrackableValue - 1, 31000);
Console.WriteLine("Testing copyInto for ShortHistogram:");
shortHistogram.CopyInto(targetShortHistogram);
AssertEqual(shortHistogram, targetShortHistogram);
shortHistogram.RecordValue(TestValueLevel * 20);
shortHistogram.CopyInto(targetShortHistogram);
AssertEqual(shortHistogram, targetShortHistogram);
Console.WriteLine("Testing copyInto for AtomicHistogram:");
var syncHistogram = new SynchronizedHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
var targetSyncHistogram = new SynchronizedHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
syncHistogram.RecordValue(TestValueLevel);
syncHistogram.RecordValue(TestValueLevel * 10);
syncHistogram.RecordValueWithExpectedInterval(syncHistogram.HighestTrackableValue - 1, 31000); // Should this really be 31, if it is the test takes 1min!!!);
Console.WriteLine("Testing copyInto for SynchronizedHistogram:");
syncHistogram.CopyInto(targetSyncHistogram);
AssertEqual(syncHistogram, targetSyncHistogram);
syncHistogram.RecordValue(TestValueLevel * 20);
syncHistogram.CopyInto(targetSyncHistogram);
AssertEqual(syncHistogram, targetSyncHistogram);
}
public void TestScaledCopyInto()
{
var longHistogram = new LongHistogram(1000, HighestTrackableValue, NumberOfSignificantValueDigits);
var targetLongHistogram = new LongHistogram(1000, HighestTrackableValue, NumberOfSignificantValueDigits);
longHistogram.RecordValue(TestValueLevel);
longHistogram.RecordValue(TestValueLevel * 10);
longHistogram.RecordValueWithExpectedInterval(longHistogram.HighestTrackableValue - 1, 31000);
Console.WriteLine("Testing copyInto for scaled Histogram:");
longHistogram.CopyInto(targetLongHistogram);
AssertEqual(longHistogram, targetLongHistogram);
longHistogram.RecordValue(TestValueLevel * 20);
longHistogram.CopyInto(targetLongHistogram);
AssertEqual(longHistogram, targetLongHistogram);
var intHistogram = new IntHistogram(1000, HighestTrackableValue, NumberOfSignificantValueDigits);
var targetIntHistogram = new IntHistogram(1000, HighestTrackableValue, NumberOfSignificantValueDigits);
intHistogram.RecordValue(TestValueLevel);
intHistogram.RecordValue(TestValueLevel * 10);
intHistogram.RecordValueWithExpectedInterval(intHistogram.HighestTrackableValue - 1, 31000);
Console.WriteLine("Testing copyInto for scaled IntHistogram:");
intHistogram.CopyInto(targetIntHistogram);
AssertEqual(intHistogram, targetIntHistogram);
intHistogram.RecordValue(TestValueLevel * 20);
intHistogram.CopyInto(targetIntHistogram);
AssertEqual(intHistogram, targetIntHistogram);
var shortHistogram = new ShortHistogram(1000, HighestTrackableValue, NumberOfSignificantValueDigits);
var targetShortHistogram = new ShortHistogram(1000, HighestTrackableValue, NumberOfSignificantValueDigits);
shortHistogram.RecordValue(TestValueLevel);
shortHistogram.RecordValue(TestValueLevel * 10);
shortHistogram.RecordValueWithExpectedInterval(shortHistogram.HighestTrackableValue - 1, 31000);
Console.WriteLine("Testing copyInto for scaled ShortHistogram:");
shortHistogram.CopyInto(targetShortHistogram);
AssertEqual(shortHistogram, targetShortHistogram);
shortHistogram.RecordValue(TestValueLevel * 20);
shortHistogram.CopyInto(targetShortHistogram);
AssertEqual(shortHistogram, targetShortHistogram);
var syncHistogram = new SynchronizedHistogram(1000, HighestTrackableValue, NumberOfSignificantValueDigits);
var targetSyncHistogram = new SynchronizedHistogram(1000, HighestTrackableValue, NumberOfSignificantValueDigits);
syncHistogram.RecordValue(TestValueLevel);
syncHistogram.RecordValue(TestValueLevel * 10);
syncHistogram.RecordValueWithExpectedInterval(syncHistogram.HighestTrackableValue - 1, 31000);
Console.WriteLine("Testing copyInto for scaled SynchronizedHistogram:");
syncHistogram.CopyInto(targetSyncHistogram);
AssertEqual(syncHistogram, targetSyncHistogram);
syncHistogram.RecordValue(TestValueLevel * 20);
syncHistogram.CopyInto(targetSyncHistogram);
AssertEqual(syncHistogram, targetSyncHistogram);
}
public void GetIntervalHistogramInto_copies_data_over_provided_Histogram()
{
var originalStart = DateTime.Today.AddDays(-1).MillisecondsSinceUnixEpoch();
var originalEnd = DateTime.Today.MillisecondsSinceUnixEpoch();
var targetHistogram = new LongHistogram(DefautltLowestDiscernibleValue, DefaultHighestTrackableValue, DefaultSignificantFigures);
targetHistogram.StartTimeStamp = originalStart;
targetHistogram.RecordValue(1);
targetHistogram.RecordValue(10);
targetHistogram.RecordValue(100);
targetHistogram.EndTimeStamp = originalEnd;
Assert.AreEqual(3, targetHistogram.TotalCount);
Assert.AreEqual(1, targetHistogram.GetCountAtValue(1));
Assert.AreEqual(1, targetHistogram.GetCountAtValue(10));
Assert.AreEqual(1, targetHistogram.GetCountAtValue(100));
var recorder = Create(DefautltLowestDiscernibleValue, DefaultHighestTrackableValue, DefaultSignificantFigures);
recorder.RecordValue(1000);
recorder.RecordValue(10000);
recorder.RecordValue(100000);
recorder.GetIntervalHistogramInto(targetHistogram);
Assert.AreEqual(3, targetHistogram.TotalCount);
Assert.AreEqual(0, targetHistogram.GetCountAtValue(1));
Assert.AreEqual(0, targetHistogram.GetCountAtValue(10));
Assert.AreEqual(0, targetHistogram.GetCountAtValue(100));
Assert.AreEqual(1, targetHistogram.GetCountAtValue(1000));
Assert.AreEqual(1, targetHistogram.GetCountAtValue(10000));
Assert.AreEqual(1, targetHistogram.GetCountAtValue(100000));
Assert.AreNotEqual(originalStart, targetHistogram.StartTimeStamp);
Assert.AreNotEqual(originalEnd, targetHistogram.EndTimeStamp);
}
private static void LoadFullRange(LongHistogram source)
{
for (long i = 0L; i < HighestTrackableValue; i += 100L)
{
source.RecordValue(i);
}
source.RecordValue(HighestTrackableValue);
}
public void GetIntervalHistogramInto_copies_data_over_provided_Histogram()
{
var originalStart = DateTime.Today.AddDays(-1).MillisecondsSinceUnixEpoch();
var originalEnd = DateTime.Today.MillisecondsSinceUnixEpoch();
var targetHistogram = new LongHistogram(1, HighestTrackableValue, 3);
targetHistogram.StartTimeStamp = originalStart;
targetHistogram.RecordValue(1);
targetHistogram.RecordValue(10);
targetHistogram.RecordValue(100);
targetHistogram.EndTimeStamp = originalEnd;
Assert.AreEqual(3, targetHistogram.TotalCount);
Assert.AreEqual(1, targetHistogram.GetCountAtValue(1));
Assert.AreEqual(1, targetHistogram.GetCountAtValue(10));
Assert.AreEqual(1, targetHistogram.GetCountAtValue(100));
var recorder = new Recorder(1, HighestTrackableValue, NumberOfSignificantValueDigits, (id, lowest, highest, sf) => new LongHistogram(id, lowest, highest, sf));
recorder.RecordValue(1000);
recorder.RecordValue(10000);
recorder.RecordValue(100000);
recorder.GetIntervalHistogramInto(targetHistogram);
Assert.AreEqual(3, targetHistogram.TotalCount);
Assert.AreEqual(0, targetHistogram.GetCountAtValue(1));
Assert.AreEqual(0, targetHistogram.GetCountAtValue(10));
Assert.AreEqual(0, targetHistogram.GetCountAtValue(100));
Assert.AreEqual(1, targetHistogram.GetCountAtValue(1000));
Assert.AreEqual(1, targetHistogram.GetCountAtValue(10000));
Assert.AreEqual(1, targetHistogram.GetCountAtValue(100000));
Assert.AreNotEqual(originalStart, targetHistogram.StartTimeStamp);
Assert.AreNotEqual(originalEnd, targetHistogram.EndTimeStamp);
}
private void StartLoop(int messageSize, int delayMicros, int burst)
{
_cts = new CancellationTokenSource();
var histogram = new LongHistogram(TimeSpan.FromSeconds(10).Ticks, 2);
void FeedClientOnMessageReceived(ReadOnlySpan <byte> bytes)
{
var now = Stopwatch.GetTimestamp();
var count = Interlocked.Increment(ref _messageCounter);
if (count <= _warmupSkipCount)
{
return;
}
var start = Unsafe.ReadUnaligned <long>(ref MemoryMarshal.GetReference(bytes));
var rrt = now - start;
var micros = (long)(rrt * 1_000_000.0 / Stopwatch.Frequency);
if (!_cts.IsCancellationRequested)
{
histogram.RecordValue(micros);
}
}
_feedClientManual.MessageReceived += FeedClientOnMessageReceived;
histogram.Reset();
_messageCounter = 0;
var buffer = new byte[messageSize];
for (var i = 0; i < buffer.Length; i++)
{
buffer[i] = 42;
}
var rateReporter = Task.Run(async() =>
{
var previousCount = 0L;
while (!_cts.IsCancellationRequested)
{
var count = Volatile.Read(ref _messageCounter);
var countPerSec = count - previousCount;
previousCount = count;
Console.WriteLine($"Processed messages: {countPerSec:N0}, total: {count:N0} [GC 0:{GC.CollectionCount(0)} 1:{GC.CollectionCount(1)}]");
await Task.Delay(1000);
}
});
while (!_cts.IsCancellationRequested)
{
_sw.Restart();
for (int i = 0; i < burst; i++)
{
Unsafe.WriteUnaligned(ref buffer[0], Stopwatch.GetTimestamp());
_feedClientManual.Send(buffer);
}
NOP(delayMicros / 1000_000.0);
}
_feedClientManual.MessageReceived -= FeedClientOnMessageReceived;
rateReporter.Wait();
histogram.OutputPercentileDistribution(Console.Out, 1);
using var writer = new StreamWriter(Path.Combine(_outFolder, $"Latency_{messageSize}_{delayMicros}.hgrm"));
histogram.OutputPercentileDistribution(writer);
}
public void CouldCRUDDirectDict()
{
var count = 1000000;
var dd = new PersistentMapFixedLength <long, long>("../CouldCRUDDirectDict", count);
//var dd = new Dictionary<DateTime, long>();
var sw = new Stopwatch();
dd.Clear();
var histogram = new LongHistogram(TimeSpan.TicksPerMillisecond * 100 * 10000, 3);
for (int rounds = 0; rounds < 20; rounds++)
{
//dd.Clear();
sw.Restart();
var dtInit = DateTime.Today;
for (int i = 0; i < count; i++)
{
var startTick = sw.ElapsedTicks;
//dd[dtInit.AddTicks(i)] = i;
dd[i] = i;
var ticks = sw.ElapsedTicks - startTick;
var nanos = (long)(1000000000.0 * (double)ticks / Stopwatch.Frequency);
if (rounds >= 1)
{
histogram.RecordValue(nanos);
}
}
Assert.AreEqual(count, dd.Count);
sw.Stop();
Console.WriteLine($"Add elapsed msec: {sw.ElapsedMilliseconds}");
}
histogram.OutputPercentileDistribution(
printStream: Console.Out,
percentileTicksPerHalfDistance: 3,
outputValueUnitScalingRatio: OutputScalingFactor.None);
var histogram2 = new LongHistogram(TimeSpan.TicksPerMillisecond * 100 * 10000, 3);
for (int rounds = 0; rounds < 20; rounds++)
{
sw.Restart();
var sum = 0L;
for (int i = 0; i < count; i++)
{
var startTick = sw.ElapsedTicks;
sum += dd[i];
var ticks = sw.ElapsedTicks - startTick;
var nanos = (long)(1000000000.0 * (double)ticks / Stopwatch.Frequency);
if (rounds >= 1)
{
histogram2.RecordValue(nanos);
}
}
sw.Stop();
Console.WriteLine($"Read elapsed msec: {sw.ElapsedMilliseconds} for sum {sum}");
}
histogram2.OutputPercentileDistribution(
printStream: Console.Out,
percentileTicksPerHalfDistance: 3,
outputValueUnitScalingRatio: OutputScalingFactor.None);
}
public void CouldReadSortedMapNewValuesWhileTheyAreAddedUsingCursor_NoSemaphore()
{
var cts = new CancellationTokenSource();
var ct = CancellationToken.None; // cts.Token; //
var count = 10000000;
var sw = new Stopwatch();
sw.Start();
var sm = new SortedMap <DateTime, double>();
sm.IsSynchronized = true;
//var sm = new SortedChunkedMap<DateTime, double>();
//sm.Add(DateTime.UtcNow.Date.AddSeconds(-2), 0);
for (int i = 0; i < 5; i++)
{
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
var histogram = new LongHistogram(TimeSpan.TicksPerMillisecond * 100 * 1000, 3);
double sum = 0;
var cnt = 0;
var histogram1 = new LongHistogram(TimeSpan.TicksPerMillisecond * 100 * 1000, 3);
var sumTask = Task.Run(async() =>
{
var c = sm.GetCursor();
var startTick = sw.ElapsedTicks;
while (await c.MoveNext(ct))
{
sum += c.CurrentValue;
if ((int)c.CurrentValue != cnt)
{
//Console.WriteLine("Wrong sequence");
//Assert.Fail($"Wrong sequence: {c.CurrentValue} != {cnt}");
Trace.WriteLine($"Wrong sequence1: {c.CurrentValue} != {cnt}; thread {Thread.CurrentThread.ManagedThreadId}");
}
else
{
//Console.WriteLine("Async move");
}
cnt++;
var ticks = sw.ElapsedTicks - startTick;
var nanos = (long)(1000000000.0 * (double)ticks / Stopwatch.Frequency);
try
{
histogram1.RecordValue(nanos);
}
catch (Exception e)
{
Console.WriteLine($"Nanos: {nanos}; " + e.Message);
}
startTick = sw.ElapsedTicks;
}
});
double sum2 = 0;
var cnt2 = 0;
var histogram2 = new LongHistogram(TimeSpan.TicksPerMillisecond * 100 * 1000, 3);
var sumTask2 = Task.Run(async() =>
{
var c = sm.GetCursor();
var startTick = sw.ElapsedTicks;
while (await c.MoveNext(ct))
{
sum2 += c.CurrentValue;
if ((int)c.CurrentValue != cnt2)
{
//Console.WriteLine("Wrong sequence");
//Assert.Fail($"Wrong sequence: {c.CurrentValue} != {cnt}");
Trace.WriteLine($"Wrong sequence2: {c.CurrentValue} != {cnt2}; thread {Thread.CurrentThread.ManagedThreadId}");
}
else
{
//Console.WriteLine("Async move");
}
cnt2++;
var ticks = sw.ElapsedTicks - startTick;
var nanos = (long)(1000000000.0 * (double)ticks / Stopwatch.Frequency);
try
{
histogram2.RecordValue(nanos);
}
catch (Exception e)
{
Console.WriteLine($"Nanos: {nanos}; " + e.Message);
}
startTick = sw.ElapsedTicks;
}
});
double sum3 = 0;
var cnt3 = 0;
var histogram3 = new LongHistogram(TimeSpan.TicksPerMillisecond * 100 * 1000, 3);
var sumTask3 = Task.Run(async() =>
{
var c = sm.GetCursor();
var startTick = sw.ElapsedTicks;
while (await c.MoveNext(ct))
{
sum3 += c.CurrentValue;
if ((int)c.CurrentValue != cnt3)
{
//Console.WriteLine("Wrong sequence");
//Assert.Fail($"Wrong sequence: {c.CurrentValue} != {cnt}");
Trace.WriteLine($"Wrong sequence3: {c.CurrentValue} != {cnt3}; thread {Thread.CurrentThread.ManagedThreadId}");
}
else
{
//Console.WriteLine("Async move");
}
cnt3++;
var ticks = sw.ElapsedTicks - startTick;
var nanos = (long)(1000000000.0 * (double)ticks / Stopwatch.Frequency);
try
{
histogram3.RecordValue(nanos);
}
catch (Exception e)
{
Console.WriteLine($"Nanos: {nanos}; " + e.Message);
}
startTick = sw.ElapsedTicks;
}
});
Thread.Sleep(1);
var addTask = Task.Run(() =>
{
//Console.WriteLine($"Adding from thread {Thread.CurrentThread.ManagedThreadId}");
try
{
for (int i = 5; i < count; i++)
{
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
sm.Complete();
}
catch (Exception ex)
{
Console.WriteLine(ex);
Environment.FailFast(ex.Message, ex);
}
});
while (!sumTask.Wait(2000))
{
OptimizationSettings.Verbose = true;
Trace.WriteLine($"cnt: {cnt}");
}
while (!sumTask2.Wait(2000))
{
//OptimizationSettings.Verbose = true;
Trace.WriteLine($"cnt2: {cnt2}");
}
while (!sumTask3.Wait(2000))
{
//OptimizationSettings.Verbose = true;
Trace.WriteLine($"cnt3: {cnt3}");
}
addTask.Wait();
histogram.Add(histogram1);
histogram.Add(histogram2);
histogram.Add(histogram3);
histogram.OutputPercentileDistribution(
writer: Console.Out,
percentileTicksPerHalfDistance: 3,
outputValueUnitScalingRatio: OutputScalingFactor.None);
sw.Stop();
Trace.Write($"Elapsed msec: {sw.ElapsedMilliseconds}; ");
Trace.WriteLine($"Ops: {Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2)}");
double expectedSum = 0.0;
for (int i = 0; i < count; i++)
{
expectedSum += i;
}
if (expectedSum != sum)
{
Trace.WriteLine("Sum 1 is wrong");
}
if (expectedSum != sum2)
{
Trace.WriteLine("Sum 2 is wrong");
}
if (expectedSum != sum3)
{
Trace.WriteLine("Sum 3 is wrong");
}
Assert.AreEqual(expectedSum, sum, "Sum 1");
Assert.AreEqual(expectedSum, sum2, "Sum 2");
Assert.AreEqual(expectedSum, sum3, "Sum 3");
//sm.Dispose();
}
private void RecordValue(Stopwatch watch)
{
lock (_hdrHistogram)
_hdrHistogram.RecordValue(watch.ElapsedMilliseconds);
}
static void Main(string[] args)
{
GCSettings.LatencyMode = GCLatencyMode.Interactive;
Console.WriteLine("Starting GC Latency Experiment - {0} - {1} - {2}",
Environment.Is64BitProcess ? "64-bit" : "32-bit",
GCSettings.IsServerGC ? "SERVER" : "WORKSTATION",
GCSettings.LatencyMode);
//A single tick represents one hundred nanoseconds or one ten-millionth of a second.
//There are 10,000 ticks in a millisecond, or 10 million ticks in a second.
// https://msdn.microsoft.com/en-us/library/system.datetime.ticks(v=vs.110).aspx
// A Histogram covering the range from 100 nano-seconds to 1 hour (3,600,000,000,000 ns) with 3 decimal point resolution:
//var histogram = new LongHistogram(TimeSpan.TicksPerHour, 3);
// A Histogram covering the range from 100 nano-seconds to 1 second with 3 decimal point resolution:
var histogram = new LongHistogram(TimeSpan.TicksPerSecond, 3);
var modeName = "NoGCRegion".ToLower();
var noGcRegionMode = args.Any(a => a.ToLower().Contains(modeName));
if (noGcRegionMode)
{
//var totalAllocs = 23500L * 1024 * 1024; // As per PerfView 23,500 MB is more than we allocate
//var totalAllocs = 16000L * 1024 * 1024; // Max we can get (should be able to get)
var totalAllocs = 10000L * 1024 * 1024;
var startNoGcRegionTimer = Stopwatch.StartNew();
var modeEntered = GC.TryStartNoGCRegion(totalAllocs);
startNoGcRegionTimer.Stop();
if (modeEntered == false)
{
Console.WriteLine($"Unable to Start No GC Region mode - requested size = {totalAllocs:N0} bytes ({totalAllocs / 1024.0 / 1024.0:N2} MB)");
return;
}
else
{
Console.WriteLine($"SUCCESSFULLY started GC Region mode, {totalAllocs:N0} bytes ({totalAllocs / 1024.0 / 1024.0:N2} MB), took {startNoGcRegionTimer.ElapsedMilliseconds:N0} ms");
}
}
if (noGcRegionMode)
{
Console.WriteLine($"\nBefore - GC Mode: {GCSettings.LatencyMode}");
}
Console.WriteLine("\nBefore");
PrintProcessMemoryInfo();
var best = new TimeSpan(hours: 0, minutes: 0, seconds: 60);
var worst = new TimeSpan();
var windowSizeRaw = (args.FirstOrDefault(a => a.ToLower().Contains("windowsize")) ?? "")
.Trim('-').Replace("=", "")
.ToLower().Replace("windowsize", "");
//Console.WriteLine($"\nWindow Size Raw = \'{windowSizeRaw}\'");
int altWindowSize;
if (string.IsNullOrWhiteSpace(windowSizeRaw) == false && int.TryParse(windowSizeRaw, out altWindowSize))
{
windowSize = altWindowSize;
Console.WriteLine($"\nUsing alternative windowSize = {windowSize:N0}");
}
var presize = args.Any(a => a.ToLower().Contains("presize"));
var map = presize ? new Dictionary <int, byte[]>(windowSize) : new Dictionary <int, byte[]>();
if (presize)
{
Console.WriteLine($"\nPre-sizing the Dictionary to contain {windowSize:N0} items");
}
var offHeap = args.Any(a => a.ToLower().Contains("offheap"));
if (offHeap)
{
Console.WriteLine($"\nGoing off heap for the byte arrays (also implies 'useArray')");
}
var useArray = args.Any(a => a.ToLower().Contains("usearray"));
var array = (useArray || offHeap) ? new byte[windowSize][] : null;
if (useArray)
{
Console.WriteLine($"\nUsing an Array instead of a Dictionary to store the messages/items");
}
if (offHeap)
{
for (int i = 0; i < array.Length; i++)
{
array[i] = new byte[msgSize];
}
}
var totalTime = Stopwatch.StartNew();
int countForGC = 0;
for (var i = 0; i < msgCount; i++)
{
var sw = Stopwatch.StartNew();
if (useArray || offHeap)
{
pushMessage(array, i, offHeap);
}
else
{
pushMessage(map, i);
}
sw.Stop();
if (sw.Elapsed > worst)
{
worst = sw.Elapsed;
}
else if (sw.Elapsed < best)
{
best = sw.Elapsed;
}
histogram.RecordValue(sw.ElapsedTicks);
++countForGC;
if (countForGC >= GCWaterLine)
{
countForGC = 0;
GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, false);
}
}
totalTime.Stop();
if (noGcRegionMode)
{
Console.WriteLine($"\nAfter - GC Mode: {GCSettings.LatencyMode}");
}
Console.WriteLine("\nAfter");
PrintProcessMemoryInfo();
Console.WriteLine($"\nTotal Time : {totalTime.Elapsed} ({totalTime.Elapsed.TotalMilliseconds:N4} ms)\n");
Console.WriteLine($"Best push time : {best.TotalMilliseconds,9:N4} ms");
Console.WriteLine($"Worst push time : {worst.TotalMilliseconds,9:N4} ms");
//Console.ReadLine();
ProcessHistogrmaResults(histogram);
}
static void Main(string[] args)
{
Console.WriteLine();
var host = args[0];
var port = int.Parse(args[1]);
var count = int.Parse(args[2]);
Rtt = new LongHistogram(1, 10000000, 5);
Encode = new LongHistogram(1, 10000000, 5);
Decode = new LongHistogram(1, 10000000, 5);
var engine = new Engine();
var configuration = new Configuration
{
Role = Role.Initiator,
Version = "FIX.4.2",
Host = host,
Port = port,
Sender = "Client",
Target = "Server",
InboundSeqNum = 1,
OutboundSeqNum = 1,
HeartbeatInterval = 0,
//LogFile = @"messages.log" // Enable to see logging impact
};
using (var initiator = engine.Open(configuration))
{
initiator.Logon();
Console.WriteLine("Logged on");
GC.Collect(0, GCCollectionMode.Forced, true, true);
GC.Collect(1, GCCollectionMode.Forced, true, true);
GC.Collect(2, GCCollectionMode.Forced, true, true);
GC0 = GC.CollectionCount(0);
GC1 = GC.CollectionCount(1);
GC2 = GC.CollectionCount(2);
Process.GetCurrentProcess().PriorityClass = ProcessPriorityClass.High;
Thread.CurrentThread.Priority = ThreadPriority.Highest;
Thread.BeginThreadAffinity();
for (var i = 0; i < count; i++)
{
var time = initiator.Clock.Time;
var order = initiator
.Outbound
.Clear()
.Set(60, time) // TransactTime
.Set(11, time.Ticks) // ClOrdId
.Set(55, "EUR/USD") // Symbol
.Set(54, 1) // Side (buy)
.Set(38, 1000.00) // OrderQty
.Set(44, 1.13200) // Price
.Set(40, 2); // OrdType (limit)
// Send order
initiator.Send("D", order);
// Wait for an execution report (ignore everything else)
while (!initiator.Receive() && !initiator.Inbound[35].Is("8"))
{
}
var rtt = initiator.Clock.Time.Ticks - time.Ticks;
if (i < 1000)
{
continue;
}
// Update statistics
Rtt.RecordValue(rtt);
Encode.RecordValue(initiator.Outbound.Duration);
Decode.RecordValue(initiator.Inbound.Duration);
}
GC0 = GC.CollectionCount(0) - GC0;
GC1 = GC.CollectionCount(1) - GC1;
GC2 = GC.CollectionCount(2) - GC2;
Thread.EndThreadAffinity();
initiator.Logout();
Console.WriteLine("Logged out");
}
PrintStatistics();
SaveStatistics();
}
public async void AmazonMQStreamer([FromQuery] int messageToSend = 10, [FromQuery] int delayBtwMessageInMs = 50)
{
NMSConnectionFactory factory = new NMSConnectionFactory(ActiveMqEndpoint);
var rd = new Random(100);
var histogram = new LongHistogram(TimeStamp.Hours(1), 3);
var writer = new StringWriter();
string[] currencyPairs = new string[] { "EUR/USD", "EUR/JPY", "EUR/GBP", "USD/JPY", "USD/GBP" };
for (int currIndex = 0; currIndex < currencyPairs.Length; currIndex++)
{
var pair = currencyPairs[currIndex];
Task.Run(async() =>
{
using (var connection = factory.CreateConnection(ActiveMqLogin, ActiveMqMdp))
{
connection.Start();
var session = connection.CreateSession(AcknowledgementMode.AutoAcknowledge);
var topic = new ActiveMQTopic("VirtualTopic." + pair);
var producer = session.CreateProducer(topic);
producer.DeliveryMode = MsgDeliveryMode.NonPersistent;
for (var i = 0; i <= messageToSend; i++)
{
var currency = new Currency()
{
Id = i,
CurrencyType = pair,
Price = rd.NextDouble(),
Timestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds().ToString(),
Ladders = new LadderFactory().Build(10)
};
var cur = JsonConvert.SerializeObject(currency);
long startTimestamp = Stopwatch.GetTimestamp();
producer.Send(session.CreateTextMessage(cur));
long elapsed = Stopwatch.GetTimestamp() - startTimestamp;
histogram.RecordValue(elapsed);
await Task.Delay(delayBtwMessageInMs).ConfigureAwait(false);
}
session.Close();
connection.Close();
var scalingRatio = OutputScalingFactor.TimeStampToMilliseconds;
histogram.OutputPercentileDistribution(
writer,
outputValueUnitScalingRatio: scalingRatio);
System.IO.File.WriteAllText(@"d:\cloud\amq.txt", writer.ToString());
}
});
}
}
public void TestAdd()
{
var longHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
var other = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
longHistogram.RecordValue(TestValueLevel);
longHistogram.RecordValue(TestValueLevel * 1000);
other.RecordValue(TestValueLevel);
other.RecordValue(TestValueLevel * 1000);
longHistogram.Add(other);
Assert.AreEqual(2L, longHistogram.GetCountAtValue(TestValueLevel));
Assert.AreEqual(2L, longHistogram.GetCountAtValue(TestValueLevel * 1000));
Assert.AreEqual(4L, longHistogram.TotalCount);
var biggerOther = new LongHistogram(HighestTrackableValue * 2, NumberOfSignificantValueDigits);
biggerOther.RecordValue(TestValueLevel);
biggerOther.RecordValue(TestValueLevel * 1000);
// Adding the smaller histogram to the bigger one should work:
biggerOther.Add(longHistogram);
Assert.AreEqual(3L, biggerOther.GetCountAtValue(TestValueLevel));
Assert.AreEqual(3L, biggerOther.GetCountAtValue(TestValueLevel * 1000));
Assert.AreEqual(6L, biggerOther.TotalCount);
// But trying to add a larger histogram into a smaller one should throw an AIOOB:
Assert.Throws<ArgumentOutOfRangeException>(() => { longHistogram.Add(biggerOther); });
}
private static string RunTest(Func <IObservable <Payload>, IScheduler, IObservable <Payload> > loadShedder, string subscriptionMode)
{
Program.Clean();
var sb = new StringBuilder();
sb.AppendFormat("Events/s {0}", EventsPerSecond);
sb.AppendLine();
sb.AppendFormat("Windows/s {0}", WindowsPerSecond);
sb.AppendLine();
sb.AppendFormat("Items/Window {0}", ItemsPerWindow);
sb.AppendLine();
sb.AppendFormat("Minutes to run {0}", MinutesToRun);
sb.AppendLine();
sb.AppendFormat("Values to produce {0}", ValuesToProduce);
sb.AppendLine();
var itemsProduced = 0;
var producerEls = new EventLoopScheduler(start => new Thread(start)
{
Name = "Producer", IsBackground = true
});
var consumerEls = new EventLoopScheduler(start => new Thread(start)
{
Name = "Consumer", IsBackground = true
});
var source = Observable.Interval(TimeSpan.FromMilliseconds(ProductionWindowMilliseconds), producerEls)
.SelectMany(i => new int[ItemsPerWindow])
.Select((_, idx) => new Payload(idx))
.Take(ValuesToProduce)
.Do(i => itemsProduced++, () => Console.WriteLine(" Producer complete"));
Console.WriteLine("Producer started {0}", subscriptionMode);
var latencyRecorder = new LongHistogram(1000000000L * 60L * 30L, 3); // 1 ns to 30 minutes
var mre = new AutoResetEvent(false);
int receiveCount = 0;
var startTimeStamp = Stopwatch.GetTimestamp();
var subscription = loadShedder(source, consumerEls)
//Only allow the test run to blow out to 5x slower than production
.TakeUntil(Observable.Timer(TimeSpan.FromMinutes(MinutesToRun * 5)))
.Finally(() =>
{
var endTimeStamp = Stopwatch.GetTimestamp();
sb.AppendFormat("Elapsed time {0}", TimeSpan.FromTicks(endTimeStamp - startTimeStamp));
mre.Set();
})
.Subscribe(
payload =>
{
payload.Received();
receiveCount++; //Should be thread-safe here.
latencyRecorder.RecordValue(payload.ReceiveLatency());
//10k -> 0.036
//20k -> 0.09s
//30k -> 0.166s
//40k -> 0.250s
var primeIndex = ((payload.Id % 4) + 1) * 10000;
FindPrimeNumber(primeIndex);
payload.Processed();
if (payload.Id % 1000 == 0)
{
Console.WriteLine(" Processed {0} events in {1}", payload.Id, TimeSpan.FromTicks(Stopwatch.GetTimestamp() - startTimeStamp));
}
},
Console.WriteLine,
() =>
{
if (subscriptionMode == null)
{
return;
}
sb.AppendLine();
sb.AppendFormat("Processing complete - {0}", subscriptionMode);
sb.AppendLine();
sb.AppendFormat("Expected to produced {0} events.", ValuesToProduce);
sb.AppendLine();
sb.AppendFormat("Produced {0} events.", itemsProduced);
sb.AppendLine();
sb.AppendFormat("Received {0} events.", receiveCount);
sb.AppendLine();
var writer = new StringWriter(sb);
latencyRecorder.OutputPercentileDistribution(writer, 10);
sb.AppendLine();
var sw = new StringWriter();
var dateTime = DateTime.Now.ToString("yyyy-MM-ddThh.mm.ss");
var fileName = string.Format(@".\output-{0}-{1}.hgrm", subscriptionMode, dateTime);
latencyRecorder.OutputPercentileDistribution(sw);
File.WriteAllText(fileName, sw.ToString());
sb.AppendFormat("Results saved to {0}", fileName);
});
Console.WriteLine("Waiting...");
mre.WaitOne();
Console.WriteLine("Disposing...");
mre.Dispose();
subscription.Dispose();
//Enqueue the disposal of the event loop as it's last scheduled item.
producerEls.Schedule(() => producerEls.Dispose());
consumerEls.Schedule(() => consumerEls.Dispose());
return(sb.ToString());
}
public void TestCopy()
{
var longHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
longHistogram.RecordValue(TestValueLevel);
longHistogram.RecordValue(TestValueLevel * 10);
longHistogram.RecordValueWithExpectedInterval(longHistogram.HighestTrackableValue - 1, 31000);
Console.WriteLine("Testing copy of Histogram:");
AssertEqual(longHistogram, longHistogram.Copy());
var intHistogram = new IntHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
intHistogram.RecordValue(TestValueLevel);
intHistogram.RecordValue(TestValueLevel * 10);
intHistogram.RecordValueWithExpectedInterval(intHistogram.HighestTrackableValue - 1, 31000);
Console.WriteLine("Testing copy of IntHistogram:");
AssertEqual(intHistogram, intHistogram.Copy());
var shortHistogram = new ShortHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
shortHistogram.RecordValue(TestValueLevel);
shortHistogram.RecordValue(TestValueLevel * 10);
shortHistogram.RecordValueWithExpectedInterval(shortHistogram.HighestTrackableValue - 1, 31000);
Console.WriteLine("Testing copy of ShortHistogram:");
AssertEqual(shortHistogram, shortHistogram.Copy());
var syncHistogram = new SynchronizedHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
syncHistogram.RecordValue(TestValueLevel);
syncHistogram.RecordValue(TestValueLevel * 10);
syncHistogram.RecordValueWithExpectedInterval(syncHistogram.HighestTrackableValue - 1, 31000);
Console.WriteLine("Testing copy of SynchronizedHistogram:");
AssertEqual(syncHistogram, syncHistogram.Copy());
}
public void RecordValueWithExpectedInterval()
{
var TestValueLevel = 4L;
var recorder = Create(DefautltLowestDiscernibleValue, DefaultHighestTrackableValue, DefaultSignificantFigures);
var valueHistogram = new LongHistogram(DefaultHighestTrackableValue, DefaultSignificantFigures);
recorder.RecordValueWithExpectedInterval(TestValueLevel, TestValueLevel / 4);
valueHistogram.RecordValue(TestValueLevel);
var intervalHistogram = recorder.GetIntervalHistogram();
// The data will include corrected samples:
Assert.AreEqual(1L, intervalHistogram.GetCountAtValue((TestValueLevel * 1) / 4));
Assert.AreEqual(1L, intervalHistogram.GetCountAtValue((TestValueLevel * 2) / 4));
Assert.AreEqual(1L, intervalHistogram.GetCountAtValue((TestValueLevel * 3) / 4));
Assert.AreEqual(1L, intervalHistogram.GetCountAtValue((TestValueLevel * 4) / 4));
Assert.AreEqual(4L, intervalHistogram.TotalCount);
// But the raw data will not:
Assert.AreEqual(0L, valueHistogram.GetCountAtValue((TestValueLevel * 1) / 4));
Assert.AreEqual(0L, valueHistogram.GetCountAtValue((TestValueLevel * 2) / 4));
Assert.AreEqual(0L, valueHistogram.GetCountAtValue((TestValueLevel * 3) / 4));
Assert.AreEqual(1L, valueHistogram.GetCountAtValue((TestValueLevel * 4) / 4));
Assert.AreEqual(1L, valueHistogram.TotalCount);
}
public void TestCopyInto()
{
var longHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
var targetLongHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
longHistogram.RecordValue(TestValueLevel);
longHistogram.RecordValue(TestValueLevel * 10);
longHistogram.RecordValueWithExpectedInterval(longHistogram.HighestTrackableValue - 1, 31000);
Console.WriteLine("Testing copyInto for Histogram:");
longHistogram.CopyInto(targetLongHistogram);
AssertEqual(longHistogram, targetLongHistogram);
longHistogram.RecordValue(TestValueLevel * 20);
longHistogram.CopyInto(targetLongHistogram);
AssertEqual(longHistogram, targetLongHistogram);
var intHistogram = new IntHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
var targetIntHistogram = new IntHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
intHistogram.RecordValue(TestValueLevel);
intHistogram.RecordValue(TestValueLevel * 10);
intHistogram.RecordValueWithExpectedInterval(intHistogram.HighestTrackableValue - 1, 31000);
Console.WriteLine("Testing copyInto for IntHistogram:");
intHistogram.CopyInto(targetIntHistogram);
AssertEqual(intHistogram, targetIntHistogram);
intHistogram.RecordValue(TestValueLevel * 20);
intHistogram.CopyInto(targetIntHistogram);
AssertEqual(intHistogram, targetIntHistogram);
var shortHistogram = new ShortHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
var targetShortHistogram = new ShortHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
shortHistogram.RecordValue(TestValueLevel);
shortHistogram.RecordValue(TestValueLevel * 10);
shortHistogram.RecordValueWithExpectedInterval(shortHistogram.HighestTrackableValue - 1, 31000);
Console.WriteLine("Testing copyInto for ShortHistogram:");
shortHistogram.CopyInto(targetShortHistogram);
AssertEqual(shortHistogram, targetShortHistogram);
shortHistogram.RecordValue(TestValueLevel * 20);
shortHistogram.CopyInto(targetShortHistogram);
AssertEqual(shortHistogram, targetShortHistogram);
Console.WriteLine("Testing copyInto for AtomicHistogram:");
var syncHistogram = new SynchronizedHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
var targetSyncHistogram = new SynchronizedHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
syncHistogram.RecordValue(TestValueLevel);
syncHistogram.RecordValue(TestValueLevel * 10);
syncHistogram.RecordValueWithExpectedInterval(syncHistogram.HighestTrackableValue - 1, 31000); // Should this really be 31, if it is the test takes 1min!!!);
Console.WriteLine("Testing copyInto for SynchronizedHistogram:");
syncHistogram.CopyInto(targetSyncHistogram);
AssertEqual(syncHistogram, targetSyncHistogram);
syncHistogram.RecordValue(TestValueLevel * 20);
syncHistogram.CopyInto(targetSyncHistogram);
AssertEqual(syncHistogram, targetSyncHistogram);
}
public void OnEvent(XEvent data, long sequence, bool endOfBatch)
{
_latencyHistogram.RecordValue(Stopwatch.GetTimestamp() - data.Timestamp);
}
public void testRecordValue_Overflow_ShouldThrowException()
{
var longHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
Assert.Catch <IndexOutOfRangeException>(() => longHistogram.RecordValue(HighestTrackableValue * 3));
}
