private void WriteToDisk(Recorder recorder)
{
//Sample every second until flagged as completed.
var accumulatingHistogram = new LongHistogram(TimeStamp.Hours(1), 3);
while (_isCompleted == 0)
{
Thread.Sleep(1000);
var histogram = recorder.GetIntervalHistogram();
accumulatingHistogram.Add(histogram);
_logWriter.Append(histogram);
Console.WriteLine($"{DateTime.Now:o} Interval.TotalCount = {histogram.TotalCount,10:G}. Accumulated.TotalCount = {accumulatingHistogram.TotalCount,10:G}.");
}
_logWriter.Dispose();
_outputStream.Dispose();
Console.WriteLine("Log contents");
Console.WriteLine(File.ReadAllText(LogPath));
Console.WriteLine();
Console.WriteLine("Percentile distribution (values reported in milliseconds)");
accumulatingHistogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToMilliseconds);
Console.WriteLine("Output thread finishing.");
}
private void WriteToDisk(Recorder recorder)
{
//Sample every second until flagged as completed.
var accumulatingHistogram = new LongHistogram(TimeStamp.Hours(1), 3);
while (_isCompleted == 0)
{
Thread.Sleep(1000);
var histogram = recorder.GetIntervalHistogram();
accumulatingHistogram.Add(histogram);
_logWriter.Append(histogram);
Console.WriteLine($"{DateTime.Now:o} Interval.TotalCount = {histogram.TotalCount,10:G}. Accumulated.TotalCount = {accumulatingHistogram.TotalCount,10:G}.");
}
_logWriter.Dispose();
_outputStream.Dispose();
Console.WriteLine("Log contents");
Console.WriteLine(File.ReadAllText(LogPath));
Console.WriteLine();
Console.WriteLine("Percentile distribution (values reported in milliseconds)");
accumulatingHistogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToMilliseconds);
Console.WriteLine("Output thread finishing.");
}
private static void ProcessHistogrmaResults(LongHistogram histogram)
{
Console.WriteLine();
var percentiles = new[] { 50.0, 90.0, 95.0, 99.9, 99.99, 99.999, 99.9999, 99.99999, 99.999999, 100.0 };
foreach (var percentile in percentiles)
{
var value = histogram.GetValueAtPercentile(percentile) / OutputScalingFactor.TimeStampToMilliseconds;
Console.WriteLine($"{percentile,10:##.######}th Percentile : {value,9:N4} ms");
}
Console.WriteLine(
$" Max : {histogram.GetMaxValue() / OutputScalingFactor.TimeStampToMilliseconds,9:N4} ms");
var fileName = "HistogramResults.hgrm";
if (File.Exists(fileName))
{
File.Delete(fileName);
}
using (var writer = new StreamWriter(fileName))
{
histogram.OutputPercentileDistribution(writer, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToMilliseconds);
}
}
[Fact] //BUG https://github.com/HdrHistogram/HdrHistogram.NET/issues/39
public void OnlySingleValueFlaggedAsLastValue()
{
var expected = GetEmbeddedFileText("IsLastValueBug.hgrm");
var histogram = new LongHistogram(highestTrackableValue: 36000000000, numberOfSignificantValueDigits: 3);
histogram.RecordValueWithCount(1L, 7604459);
histogram.RecordValueWithCount(383, 2395524);
histogram.RecordValueWithCount(453, 2);
histogram.RecordValueWithCount(511, 2);
histogram.RecordValueWithCount(537, 3);
histogram.RecordValueWithCount(672, 1);
histogram.RecordValueWithCount(777, 1);
histogram.RecordValueWithCount(18143, 1);
histogram.RecordValueWithCount(208127, 1);
histogram.RecordValueWithCount(224639, 1);
histogram.RecordValueWithCount(229759, 1);
histogram.RecordValueWithCount(230271, 1);
histogram.RecordValueWithCount(258943, 1);
histogram.RecordValueWithCount(275711, 1);
histogram.RecordValueWithCount(282111, 1);
var writer = new StringWriter();
histogram.OutputPercentileDistribution(writer);
var actual = writer.ToString();
Assert.Equal(expected, actual);
}
public async void AmazonIotStreamer([FromQuery] int messageToSend = 10, [FromQuery] int delayBtwMessageInMs = 50)
{
var rd = new Random(100);
var histogram = new LongHistogram(TimeStamp.Hours(1), 3);
var writer = new StringWriter();
var messages = messageToSend;
string[] currencyPairs = new string[] { "EUR/USD", "EUR/JPY", "EUR/GBP", "USD/JPY", "USD/GBP" };
for (int currIndex = 0; currIndex < currencyPairs.Length; currIndex++)
{
var pair = currencyPairs[currIndex];
Task.Run(async() =>
{
var publisher = new AmazonIotDataClient(IotEndpoint, new BasicAWSCredentials(IotAccessKey, IotSecret));
for (var i = 1; i <= messages; i++)
{
long startTimestamp = Stopwatch.GetTimestamp();
long timestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds();
var currency = new Currency()
{
Id = i,
CurrencyType = pair,
Price = rd.NextDouble(),
Timestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds().ToString(),
Ladders = new LadderFactory().Build(10)
};
var cur = JsonConvert.SerializeObject(currency);
publisher.PublishAsync(new Amazon.IotData.Model.PublishRequest()
{
Topic = pair,
Payload = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(cur)),
Qos = 0
});
long elapsed = Stopwatch.GetTimestamp() - startTimestamp;
histogram.RecordValue(elapsed);
await Task.Delay(delayBtwMessageInMs).ConfigureAwait(false);
}
var scalingRatio = OutputScalingFactor.TimeStampToMilliseconds;
histogram.OutputPercentileDistribution(
writer,
outputValueUnitScalingRatio: scalingRatio);
System.IO.File.WriteAllText(@"d:\cloud\appsync.txt", writer.ToString());
});
}
}
/// <summary>
/// Write to the console the memory footprint of the histogram instance and
/// the percentile distribution of all the recorded values.
/// </summary>
private static void OutputMeasurements()
{
var size = Histogram.GetEstimatedFootprintInBytes();
Console.WriteLine("Histogram size = {0} bytes ({1:F2} MB)", size, size / 1024.0 / 1024.0);
Console.WriteLine("Recorded latencies [in system clock ticks] for Create+Close of a DatagramSocket:");
Histogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.None);
Console.WriteLine();
Console.WriteLine("Recorded latencies [in usec] for Create+Close of a DatagramSocket:");
Histogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToMicroseconds);
Console.WriteLine();
Console.WriteLine("Recorded latencies [in msec] for Create+Close of a DatagramSocket:");
Histogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToMilliseconds);
Console.WriteLine();
Console.WriteLine("Recorded latencies [in sec] for Create+Close of a DatagramSocket:");
Histogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToSeconds);
}
private static void RunClient(IFeedClient client)
{
var histogram = new LongHistogram(TimeSpan.FromSeconds(10).Ticks, 2);
var receivedMessageCount = 0;
void OnMessageReceived(ReadOnlySpan <byte> message)
{
var now = Stopwatch.GetTimestamp();
var start = Unsafe.ReadUnaligned <long>(ref MemoryMarshal.GetReference(message));
var rrt = now - start;
var latencyInMicroseconds = unchecked (rrt * 1_000_000 / (double)Stopwatch.Frequency);
receivedMessageCount++;
histogram.RecordValue((long)latencyInMicroseconds);
}
using (client)
{
var connectedSignal = new AutoResetEvent(false);
client.Connected += () => Console.WriteLine($"Connected {connectedSignal.Set()}");
client.MessageReceived += OnMessageReceived;
client.Start("client " + Guid.NewGuid());
connectedSignal.WaitOne();
do
{
Console.WriteLine("Bench? (<message count> <message size> <delay in micro> <burst>, eg.: 100000 128 10 1)");
var benchArgs = Console.ReadLine();
if (!TryParseBenchArgs(benchArgs, out var args))
{
break;
}
histogram.Reset();
RunBench(client, args, ref receivedMessageCount);
histogram.OutputPercentileDistribution(Console.Out, 1);
Console.WriteLine("FailSends : " + Counters.FailedReceivingNextCount);
Console.WriteLine("FailReceives : " + Counters.FailedReceivingNextCount);
Counters.Reset();
} while (true);
client.Stop();
}
}
static void Report(string name, LongHistogram histogram, TimeSpan?maximumMean)
{
Console.Out.WriteLine($"Histogram for {name}:");
histogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToMilliseconds, percentileTicksPerHalfDistance: 1);
Console.Out.WriteLine();
if (maximumMean != null)
{
var max = maximumMean.Value;
var actualMean = TimeSpan.FromMilliseconds(histogram.GetValueAtPercentile(50) / OutputScalingFactor.TimeStampToMilliseconds);
Assert.LessOrEqual(actualMean, max, $"The actual mean for {name} was '{actualMean}' and was bigger than maximum allowed mean '{max}'.");
}
}
public static void Stream(Func <string, string, Random, string> createQuery, int messageToSend, int delayBtwMessageInMs, string apiKey, string endpointUrl)
{
var client = new HttpClient();
System.Net.ServicePointManager.SecurityProtocol =
SecurityProtocolType.Tls12 | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls;
var rd = new Random(100);
var histogram = new LongHistogram(TimeStamp.Hours(1), 3);
var writer = new StringWriter();
var messages = messageToSend;
Task.Run(async() =>
{
for (var i = 1; i <= messages; i++)
{
var req = new HttpRequestMessage();
req.Headers.TryAddWithoutValidation("x-api-key", apiKey);
req.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
req.Headers.TryAddWithoutValidation("Content-Type", "application/json");
req.Method = HttpMethod.Post;
req.RequestUri = new Uri(endpointUrl);
long startTimestamp = Stopwatch.GetTimestamp();
long timestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds();
req.Content = new StringContent(createQuery(i.ToString(), timestamp.ToString(), rd), Encoding.UTF8, "application/json");
await client.SendAsync((req)).ConfigureAwait(false);
long elapsed = Stopwatch.GetTimestamp() - startTimestamp;
histogram.RecordValue(elapsed);
await Task.Delay(delayBtwMessageInMs).ConfigureAwait(false);
}
var scalingRatio = OutputScalingFactor.TimeStampToMilliseconds;
histogram.OutputPercentileDistribution(
writer,
outputValueUnitScalingRatio: scalingRatio);
System.IO.File.WriteAllText(@"d:\cloud\appsync.txt", writer.ToString());
});
}
public void PercentileDistrubution_csv_output_is_in_correct_format(
[Values(36000000000)]long highestTrackableValue,
[Values(1, 2, 3, 4, 5)]int signigicantDigits,
[Values(10000.0)]double scaling,
[Values(5, 10, 20)]int percentileTicksPerHalfDistance)
{
var fileName = $"Sample_10kBy1k_{signigicantDigits}sf_TicksPerHour_asMs_{percentileTicksPerHalfDistance}percPerHalfDistance.csv";
var expected = GetEmbeddedFileText(fileName);
var histogram = new LongHistogram(highestTrackableValue, signigicantDigits);
LoadHistogram(histogram);
var writer = new StringWriter();
histogram.OutputPercentileDistribution(writer, percentileTicksPerHalfDistance, scaling, true);
var actual = writer.ToString();
Assert.AreEqual(expected, actual);
}
public void SignalRStreamer([FromQuery] int messageToSend = 10, [FromQuery] int delayBtwMessageInMs = 50)
{
var histogram = new LongHistogram(TimeStamp.Hours(1), 3);
var writer = new StringWriter();
string[] currencyPairs = new string[] { "EUR/USD", "EUR/JPY", "EUR/GBP", "USD/JPY", "USD/GBP" };
for (int currIndex = 0; currIndex < currencyPairs.Length; currIndex++)
{
var pair = currencyPairs[currIndex];
Task.Run(async() =>
{
var rd = new Random(100);
for (var i = 1; i <= messageToSend; i++)
{
long startTimestamp = Stopwatch.GetTimestamp();
var currency = new Currency()
{
Id = i,
CurrencyType = pair,
Price = rd.NextDouble(),
Timestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds().ToString(),
Ladders = new LadderFactory().Build(10)
};
var cur = JsonConvert.SerializeObject(currency);
await context.Clients.Group(pair).SendAsync("broadcastMessage", "currency", cur).ConfigureAwait(false);
long elapsed = Stopwatch.GetTimestamp() - startTimestamp;
histogram.RecordValue(elapsed);
await Task.Delay(delayBtwMessageInMs).ConfigureAwait(false);
}
var scalingRatio = OutputScalingFactor.TimeStampToMilliseconds;
histogram.OutputPercentileDistribution(
writer,
outputValueUnitScalingRatio: scalingRatio);
System.IO.File.WriteAllText(@"d:\cloud\signalr.txt", writer.ToString());
});
}
}
public static void Main()
{
var ctx = new Aeron.Context()
.AvailableImageHandler(AvailablePongImageHandler);
var fragmentAssembler = new FragmentAssembler(HandlerHelper.ToFragmentHandler(PongHandler));
Console.WriteLine("Publishing Ping at " + PingChannel + " on stream Id " + PingStreamID);
Console.WriteLine("Subscribing Pong at " + PongChannel + " on stream Id " + PongStreamID);
Console.WriteLine("Message length of " + MessageLength + " bytes");
using (var aeron = Aeron.Connect(ctx))
{
Console.WriteLine("Warming up... " + WarmupNumberOfIterations + " iterations of " + WarmupNumberOfMessages + " messages");
using (var publication = aeron.AddPublication(PingChannel, PingStreamID))
using (var subscription = aeron.AddSubscription(PongChannel, PongStreamID))
using (var byteBuffer = BufferUtil.AllocateDirectAligned(MessageLength, BitUtil.CACHE_LINE_LENGTH))
using (var atomicBuffer = new UnsafeBuffer(byteBuffer))
{
Latch.Wait();
for (var i = 0; i < WarmupNumberOfIterations; i++)
{
RoundTripMessages(atomicBuffer, fragmentAssembler, publication, subscription, WarmupNumberOfMessages);
}
Thread.Sleep(100);
do
{
Histogram.Reset();
Console.WriteLine("Pinging " + NumberOfMessages + " messages");
RoundTripMessages(atomicBuffer, fragmentAssembler, publication, subscription, NumberOfMessages);
Console.WriteLine("Histogram of RTT latencies in microseconds.");
Histogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: 1000);
} while (Console.Read() == 'y');
}
}
}
public void PercentileDistrubution_csv_output_is_in_correct_format(
[CombinatorialValues(36000000000)] long highestTrackableValue,
[CombinatorialValues(1, 2, 3, 4, 5)] int signigicantDigits,
[CombinatorialValues(10000.0)] double scaling,
[CombinatorialValues(5, 10, 20)] int percentileTicksPerHalfDistance)
{
var fileName = $"Sample_10kBy1k_{signigicantDigits}sf_TicksPerHour_asMs_{percentileTicksPerHalfDistance}percPerHalfDistance.csv";
var expected = GetEmbeddedFileText(fileName);
var histogram = new LongHistogram(highestTrackableValue, signigicantDigits);
LoadHistogram(histogram);
var writer = new StringWriter();
histogram.OutputPercentileDistribution(writer, percentileTicksPerHalfDistance, scaling, true);
var actual = writer.ToString();
Assert.Equal(expected, actual);
}
public void CouldReadSortedMapNewValuesWhileTheyAreAddedUsingCursor_NoSemaphore()
{
var cts = new CancellationTokenSource();
var ct = CancellationToken.None; // cts.Token; //
var count = 10000000;
var sw = new Stopwatch();
sw.Start();
var sm = new SortedMap <DateTime, double>();
sm.IsSynchronized = true;
//var sm = new SortedChunkedMap<DateTime, double>();
//sm.Add(DateTime.UtcNow.Date.AddSeconds(-2), 0);
for (int i = 0; i < 5; i++)
{
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
var histogram = new LongHistogram(TimeSpan.TicksPerMillisecond * 100 * 1000, 3);
double sum = 0;
var cnt = 0;
var histogram1 = new LongHistogram(TimeSpan.TicksPerMillisecond * 100 * 1000, 3);
var sumTask = Task.Run(async() =>
{
var c = sm.GetCursor();
var startTick = sw.ElapsedTicks;
while (await c.MoveNext(ct))
{
sum += c.CurrentValue;
if ((int)c.CurrentValue != cnt)
{
//Console.WriteLine("Wrong sequence");
//Assert.Fail($"Wrong sequence: {c.CurrentValue} != {cnt}");
Trace.WriteLine($"Wrong sequence1: {c.CurrentValue} != {cnt}; thread {Thread.CurrentThread.ManagedThreadId}");
}
else
{
//Console.WriteLine("Async move");
}
cnt++;
var ticks = sw.ElapsedTicks - startTick;
var nanos = (long)(1000000000.0 * (double)ticks / Stopwatch.Frequency);
try
{
histogram1.RecordValue(nanos);
}
catch (Exception e)
{
Console.WriteLine($"Nanos: {nanos}; " + e.Message);
}
startTick = sw.ElapsedTicks;
}
});
double sum2 = 0;
var cnt2 = 0;
var histogram2 = new LongHistogram(TimeSpan.TicksPerMillisecond * 100 * 1000, 3);
var sumTask2 = Task.Run(async() =>
{
var c = sm.GetCursor();
var startTick = sw.ElapsedTicks;
while (await c.MoveNext(ct))
{
sum2 += c.CurrentValue;
if ((int)c.CurrentValue != cnt2)
{
//Console.WriteLine("Wrong sequence");
//Assert.Fail($"Wrong sequence: {c.CurrentValue} != {cnt}");
Trace.WriteLine($"Wrong sequence2: {c.CurrentValue} != {cnt2}; thread {Thread.CurrentThread.ManagedThreadId}");
}
else
{
//Console.WriteLine("Async move");
}
cnt2++;
var ticks = sw.ElapsedTicks - startTick;
var nanos = (long)(1000000000.0 * (double)ticks / Stopwatch.Frequency);
try
{
histogram2.RecordValue(nanos);
}
catch (Exception e)
{
Console.WriteLine($"Nanos: {nanos}; " + e.Message);
}
startTick = sw.ElapsedTicks;
}
});
double sum3 = 0;
var cnt3 = 0;
var histogram3 = new LongHistogram(TimeSpan.TicksPerMillisecond * 100 * 1000, 3);
var sumTask3 = Task.Run(async() =>
{
var c = sm.GetCursor();
var startTick = sw.ElapsedTicks;
while (await c.MoveNext(ct))
{
sum3 += c.CurrentValue;
if ((int)c.CurrentValue != cnt3)
{
//Console.WriteLine("Wrong sequence");
//Assert.Fail($"Wrong sequence: {c.CurrentValue} != {cnt}");
Trace.WriteLine($"Wrong sequence3: {c.CurrentValue} != {cnt3}; thread {Thread.CurrentThread.ManagedThreadId}");
}
else
{
//Console.WriteLine("Async move");
}
cnt3++;
var ticks = sw.ElapsedTicks - startTick;
var nanos = (long)(1000000000.0 * (double)ticks / Stopwatch.Frequency);
try
{
histogram3.RecordValue(nanos);
}
catch (Exception e)
{
Console.WriteLine($"Nanos: {nanos}; " + e.Message);
}
startTick = sw.ElapsedTicks;
}
});
Thread.Sleep(1);
var addTask = Task.Run(() =>
{
//Console.WriteLine($"Adding from thread {Thread.CurrentThread.ManagedThreadId}");
try
{
for (int i = 5; i < count; i++)
{
sm.Add(DateTime.UtcNow.Date.AddSeconds(i), i);
}
sm.Complete();
}
catch (Exception ex)
{
Console.WriteLine(ex);
Environment.FailFast(ex.Message, ex);
}
});
while (!sumTask.Wait(2000))
{
OptimizationSettings.Verbose = true;
Trace.WriteLine($"cnt: {cnt}");
}
while (!sumTask2.Wait(2000))
{
//OptimizationSettings.Verbose = true;
Trace.WriteLine($"cnt2: {cnt2}");
}
while (!sumTask3.Wait(2000))
{
//OptimizationSettings.Verbose = true;
Trace.WriteLine($"cnt3: {cnt3}");
}
addTask.Wait();
histogram.Add(histogram1);
histogram.Add(histogram2);
histogram.Add(histogram3);
histogram.OutputPercentileDistribution(
writer: Console.Out,
percentileTicksPerHalfDistance: 3,
outputValueUnitScalingRatio: OutputScalingFactor.None);
sw.Stop();
Trace.Write($"Elapsed msec: {sw.ElapsedMilliseconds}; ");
Trace.WriteLine($"Ops: {Math.Round(0.000001 * count * 1000.0 / (sw.ElapsedMilliseconds * 1.0), 2)}");
double expectedSum = 0.0;
for (int i = 0; i < count; i++)
{
expectedSum += i;
}
if (expectedSum != sum)
{
Trace.WriteLine("Sum 1 is wrong");
}
if (expectedSum != sum2)
{
Trace.WriteLine("Sum 2 is wrong");
}
if (expectedSum != sum3)
{
Trace.WriteLine("Sum 3 is wrong");
}
Assert.AreEqual(expectedSum, sum, "Sum 1");
Assert.AreEqual(expectedSum, sum2, "Sum 2");
Assert.AreEqual(expectedSum, sum3, "Sum 3");
//sm.Dispose();
}
private static void SaveStatistics()
{
using (var writer = new StreamWriter(@"..\..\..\.bench\histogram-rtt.hgrm")) Rtt.OutputPercentileDistribution(writer, outputValueUnitScalingRatio: 10);
using (var writer = new StreamWriter(@"..\..\..\.bench\histogram-encode.hgrm")) Encode.OutputPercentileDistribution(writer, outputValueUnitScalingRatio: 10);
using (var writer = new StreamWriter(@"..\..\..\.bench\histogram-decode.hgrm")) Decode.OutputPercentileDistribution(writer, outputValueUnitScalingRatio: 10);
}
private static string RunTest(Func <IObservable <Payload>, IScheduler, IObservable <Payload> > loadShedder, string subscriptionMode)
{
Program.Clean();
var sb = new StringBuilder();
sb.AppendFormat("Events/s {0}", EventsPerSecond);
sb.AppendLine();
sb.AppendFormat("Windows/s {0}", WindowsPerSecond);
sb.AppendLine();
sb.AppendFormat("Items/Window {0}", ItemsPerWindow);
sb.AppendLine();
sb.AppendFormat("Minutes to run {0}", MinutesToRun);
sb.AppendLine();
sb.AppendFormat("Values to produce {0}", ValuesToProduce);
sb.AppendLine();
var itemsProduced = 0;
var producerEls = new EventLoopScheduler(start => new Thread(start)
{
Name = "Producer", IsBackground = true
});
var consumerEls = new EventLoopScheduler(start => new Thread(start)
{
Name = "Consumer", IsBackground = true
});
var source = Observable.Interval(TimeSpan.FromMilliseconds(ProductionWindowMilliseconds), producerEls)
.SelectMany(i => new int[ItemsPerWindow])
.Select((_, idx) => new Payload(idx))
.Take(ValuesToProduce)
.Do(i => itemsProduced++, () => Console.WriteLine(" Producer complete"));
Console.WriteLine("Producer started {0}", subscriptionMode);
var latencyRecorder = new LongHistogram(1000000000L * 60L * 30L, 3); // 1 ns to 30 minutes
var mre = new AutoResetEvent(false);
int receiveCount = 0;
var startTimeStamp = Stopwatch.GetTimestamp();
var subscription = loadShedder(source, consumerEls)
//Only allow the test run to blow out to 5x slower than production
.TakeUntil(Observable.Timer(TimeSpan.FromMinutes(MinutesToRun * 5)))
.Finally(() =>
{
var endTimeStamp = Stopwatch.GetTimestamp();
sb.AppendFormat("Elapsed time {0}", TimeSpan.FromTicks(endTimeStamp - startTimeStamp));
mre.Set();
})
.Subscribe(
payload =>
{
payload.Received();
receiveCount++; //Should be thread-safe here.
latencyRecorder.RecordValue(payload.ReceiveLatency());
//10k -> 0.036
//20k -> 0.09s
//30k -> 0.166s
//40k -> 0.250s
var primeIndex = ((payload.Id % 4) + 1) * 10000;
FindPrimeNumber(primeIndex);
payload.Processed();
if (payload.Id % 1000 == 0)
{
Console.WriteLine(" Processed {0} events in {1}", payload.Id, TimeSpan.FromTicks(Stopwatch.GetTimestamp() - startTimeStamp));
}
},
Console.WriteLine,
() =>
{
if (subscriptionMode == null)
{
return;
}
sb.AppendLine();
sb.AppendFormat("Processing complete - {0}", subscriptionMode);
sb.AppendLine();
sb.AppendFormat("Expected to produced {0} events.", ValuesToProduce);
sb.AppendLine();
sb.AppendFormat("Produced {0} events.", itemsProduced);
sb.AppendLine();
sb.AppendFormat("Received {0} events.", receiveCount);
sb.AppendLine();
var writer = new StringWriter(sb);
latencyRecorder.OutputPercentileDistribution(writer, 10);
sb.AppendLine();
var sw = new StringWriter();
var dateTime = DateTime.Now.ToString("yyyy-MM-ddThh.mm.ss");
var fileName = string.Format(@".\output-{0}-{1}.hgrm", subscriptionMode, dateTime);
latencyRecorder.OutputPercentileDistribution(sw);
File.WriteAllText(fileName, sw.ToString());
sb.AppendFormat("Results saved to {0}", fileName);
});
Console.WriteLine("Waiting...");
mre.WaitOne();
Console.WriteLine("Disposing...");
mre.Dispose();
subscription.Dispose();
//Enqueue the disposal of the event loop as it's last scheduled item.
producerEls.Schedule(() => producerEls.Dispose());
consumerEls.Schedule(() => consumerEls.Dispose());
return(sb.ToString());
}
public async void AmazonMQStreamer([FromQuery] int messageToSend = 10, [FromQuery] int delayBtwMessageInMs = 50)
{
NMSConnectionFactory factory = new NMSConnectionFactory(ActiveMqEndpoint);
var rd = new Random(100);
var histogram = new LongHistogram(TimeStamp.Hours(1), 3);
var writer = new StringWriter();
string[] currencyPairs = new string[] { "EUR/USD", "EUR/JPY", "EUR/GBP", "USD/JPY", "USD/GBP" };
for (int currIndex = 0; currIndex < currencyPairs.Length; currIndex++)
{
var pair = currencyPairs[currIndex];
Task.Run(async() =>
{
using (var connection = factory.CreateConnection(ActiveMqLogin, ActiveMqMdp))
{
connection.Start();
var session = connection.CreateSession(AcknowledgementMode.AutoAcknowledge);
var topic = new ActiveMQTopic("VirtualTopic." + pair);
var producer = session.CreateProducer(topic);
producer.DeliveryMode = MsgDeliveryMode.NonPersistent;
for (var i = 0; i <= messageToSend; i++)
{
var currency = new Currency()
{
Id = i,
CurrencyType = pair,
Price = rd.NextDouble(),
Timestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds().ToString(),
Ladders = new LadderFactory().Build(10)
};
var cur = JsonConvert.SerializeObject(currency);
long startTimestamp = Stopwatch.GetTimestamp();
producer.Send(session.CreateTextMessage(cur));
long elapsed = Stopwatch.GetTimestamp() - startTimestamp;
histogram.RecordValue(elapsed);
await Task.Delay(delayBtwMessageInMs).ConfigureAwait(false);
}
session.Close();
connection.Close();
var scalingRatio = OutputScalingFactor.TimeStampToMilliseconds;
histogram.OutputPercentileDistribution(
writer,
outputValueUnitScalingRatio: scalingRatio);
System.IO.File.WriteAllText(@"d:\cloud\amq.txt", writer.ToString());
}
});
}
}
private void StartLoop(int messageSize, int delayMicros, int burst)
{
_cts = new CancellationTokenSource();
var histogram = new LongHistogram(TimeSpan.FromSeconds(10).Ticks, 2);
void FeedClientOnMessageReceived(ReadOnlySpan <byte> bytes)
{
var now = Stopwatch.GetTimestamp();
var count = Interlocked.Increment(ref _messageCounter);
if (count <= _warmupSkipCount)
{
return;
}
var start = Unsafe.ReadUnaligned <long>(ref MemoryMarshal.GetReference(bytes));
var rrt = now - start;
var micros = (long)(rrt * 1_000_000.0 / Stopwatch.Frequency);
if (!_cts.IsCancellationRequested)
{
histogram.RecordValue(micros);
}
}
_feedClientManual.MessageReceived += FeedClientOnMessageReceived;
histogram.Reset();
_messageCounter = 0;
var buffer = new byte[messageSize];
for (var i = 0; i < buffer.Length; i++)
{
buffer[i] = 42;
}
var rateReporter = Task.Run(async() =>
{
var previousCount = 0L;
while (!_cts.IsCancellationRequested)
{
var count = Volatile.Read(ref _messageCounter);
var countPerSec = count - previousCount;
previousCount = count;
Console.WriteLine($"Processed messages: {countPerSec:N0}, total: {count:N0} [GC 0:{GC.CollectionCount(0)} 1:{GC.CollectionCount(1)}]");
await Task.Delay(1000);
}
});
while (!_cts.IsCancellationRequested)
{
_sw.Restart();
for (int i = 0; i < burst; i++)
{
Unsafe.WriteUnaligned(ref buffer[0], Stopwatch.GetTimestamp());
_feedClientManual.Send(buffer);
}
NOP(delayMicros / 1000_000.0);
}
_feedClientManual.MessageReceived -= FeedClientOnMessageReceived;
rateReporter.Wait();
histogram.OutputPercentileDistribution(Console.Out, 1);
using var writer = new StreamWriter(Path.Combine(_outFolder, $"Latency_{messageSize}_{delayMicros}.hgrm"));
histogram.OutputPercentileDistribution(writer);
}
private async Task Run(string[] args)
{
var host = ConfigurationManager.AppSettings["rabbit.host"];
var user = ConfigurationManager.AppSettings["rabbit.admuser"];
var pwd = ConfigurationManager.AppSettings["rabbit.admpwd"];
var vhost = ConfigurationManager.AppSettings["rabbit.vhost"];
LogAdapter.LogDebugFn = (s, s1, arg3) => { };
LogAdapter.ExtendedLogEnabled = false;
LogAdapter.ProtocolLevelLogEnabled = false;
_hdrHistogram = new LongHistogram(1, 1000 * 10, 5);
int howManyQueues = 10;
bool exclusiveConnections = ConfigurationManager.AppSettings["exclusiveConnections"] == "true";
bool useOfficialClient = ConfigurationManager.AppSettings["useOfficialClient"] == "true";
if (useOfficialClient)
{
RabbitMQ.Client.IConnection conn = null;
RabbitMQ.Client.IModel channel = null;
for (int i = 0; i < howManyQueues; i++)
{
var connFac = new RabbitMQ.Client.ConnectionFactory()
{
HostName = host, UserName = user, Password = pwd, VirtualHost = vhost, AutomaticRecoveryEnabled = false
};
if (exclusiveConnections || conn == null)
{
conn = connFac.CreateConnection();
channel = conn.CreateModel();
channel.BasicQos(0, 300, false);
}
var q = "q." + i;
channel.QueueDeclareNoWait(q, durable: true, exclusive: false, autoDelete: false, arguments: null);
channel.BasicConsume(q, false, "con_" + q, arguments: null, consumer: new Consumer(channel));
}
}
else
{
RabbitMqNext.IConnection conn = null;
RabbitMqNext.IChannel channel = null;
for (int i = 0; i < howManyQueues; i++)
{
if (exclusiveConnections || conn == null)
{
conn = await RabbitMqNext.ConnectionFactory.Connect(host, vhost, user, pwd, recoverySettings : null, connectionName : "mod_perf_server");
channel = await conn.CreateChannel();
await channel.BasicQos(0, 300, false);
}
var q = "q." + i;
await channel.QueueDeclare(q, passive : false, durable : true, exclusive : false, autoDelete : false, arguments : null,
waitConfirmation : false);
// TODO: test with parallel buffer copy + serialized too
await channel.BasicConsume(ConsumeMode.SerializedWithBufferCopy, BuildConsumerFn(channel), q, "consumer_" + q,
false, true, arguments : null, waitConfirmation : false);
}
}
Console.WriteLine("Consuming..");
Console.CancelKeyPress += (sender, eventArgs) =>
{
Console.WriteLine("Done\r\n");
Console.WriteLine("howManyQueues {0} exclusive Connections: {1} official client: {2} count {3}", howManyQueues, exclusiveConnections, useOfficialClient, _hdrHistogram.TotalCount);
Console.WriteLine("\r\n");
_hdrHistogram.OutputPercentileDistribution(Console.Out);
Console.ReadKey();
};
await Task.Delay(1);
Thread.CurrentThread.Join();
}
public void CouldCRUDDirectDict()
{
var count = 1000000;
var dd = new PersistentMapFixedLength <long, long>("../CouldCRUDDirectDict", count);
//var dd = new Dictionary<DateTime, long>();
var sw = new Stopwatch();
dd.Clear();
var histogram = new LongHistogram(TimeSpan.TicksPerMillisecond * 100 * 10000, 3);
for (int rounds = 0; rounds < 20; rounds++)
{
//dd.Clear();
sw.Restart();
var dtInit = DateTime.Today;
for (int i = 0; i < count; i++)
{
var startTick = sw.ElapsedTicks;
//dd[dtInit.AddTicks(i)] = i;
dd[i] = i;
var ticks = sw.ElapsedTicks - startTick;
var nanos = (long)(1000000000.0 * (double)ticks / Stopwatch.Frequency);
if (rounds >= 1)
{
histogram.RecordValue(nanos);
}
}
Assert.AreEqual(count, dd.Count);
sw.Stop();
Console.WriteLine($"Add elapsed msec: {sw.ElapsedMilliseconds}");
}
histogram.OutputPercentileDistribution(
printStream: Console.Out,
percentileTicksPerHalfDistance: 3,
outputValueUnitScalingRatio: OutputScalingFactor.None);
var histogram2 = new LongHistogram(TimeSpan.TicksPerMillisecond * 100 * 10000, 3);
for (int rounds = 0; rounds < 20; rounds++)
{
sw.Restart();
var sum = 0L;
for (int i = 0; i < count; i++)
{
var startTick = sw.ElapsedTicks;
sum += dd[i];
var ticks = sw.ElapsedTicks - startTick;
var nanos = (long)(1000000000.0 * (double)ticks / Stopwatch.Frequency);
if (rounds >= 1)
{
histogram2.RecordValue(nanos);
}
}
sw.Stop();
Console.WriteLine($"Read elapsed msec: {sw.ElapsedMilliseconds} for sum {sum}");
}
histogram2.OutputPercentileDistribution(
printStream: Console.Out,
percentileTicksPerHalfDistance: 3,
outputValueUnitScalingRatio: OutputScalingFactor.None);
}
private async Task Run(string[] args)
{
//
// client side
// sends rpc requests, measures the roundtrip
//
LogAdapter.ExtendedLogEnabled = false;
LogAdapter.ProtocolLevelLogEnabled = false;
LogAdapter.LogDebugFn = (s, s1, arg3) => { };
_hdrHistogram = new LongHistogram(1, 1000 * 10, 5);
var host = ConfigurationManager.AppSettings["rabbit.host"];
var user = ConfigurationManager.AppSettings["rabbit.admuser"];
var pwd = ConfigurationManager.AppSettings["rabbit.admpwd"];
var vhost = ConfigurationManager.AppSettings["rabbit.vhost"];
var postWarmupDelay = TimeSpan.FromSeconds(5);
int howManyQueues = 1;
bool exclusiveConnections = ConfigurationManager.AppSettings["exclusiveConnections"] == "true";
bool useOfficialClient = ConfigurationManager.AppSettings["useOfficialClient"] == "true";
var howManyCalls = 50000;
// var howManyCalls = 100;
_completionSemaphore = new CountdownEvent(howManyQueues);
_startSync = new ManualResetEventSlim(false);
// if (useOfficialClient)
// {
// // Warm up
// {
// var connFac = new RabbitMQ.Client.ConnectionFactory()
// {
// HostName = host,
// UserName = user,
// Password = pwd,
// VirtualHost = vhost,
// AutomaticRecoveryEnabled = false
// };
// var conn2 = connFac.CreateConnection();
// var channel2 = conn2.CreateModel();
// var q = "q." + 0;
// SendLegacyCalls(q, channel2, howManyCalls/4, isWarmUp: true);
// channel2.Dispose();
// conn2.Dispose();
// }
//
// await WarmupComplete(postWarmupDelay);
//
// RabbitMQ.Client.IConnection conn = null;
// RabbitMQ.Client.IModel channel = null;
//
// for (int i = 0; i < howManyQueues; i++)
// {
// var connFac = new RabbitMQ.Client.ConnectionFactory()
// {
// HostName = host,
// UserName = user,
// Password = pwd,
// VirtualHost = vhost,
// AutomaticRecoveryEnabled = false
// };
//
// if (exclusiveConnections || conn == null)
// {
// conn = connFac.CreateConnection();
// channel = conn.CreateModel();
// }
//
// var q = "q." + i;
//
// new Thread(() =>
// {
// SendLegacyCalls(q, channel, howManyCalls, isWarmUp: false);
// }) {IsBackground = true}.Start();
//
// }
// }
// else
{
RabbitMqNext.LogAdapter.LogErrorFn = (scope, message, exc) =>
{
Console.WriteLine("[Error] " + scope + " - " + message + " exception " + exc);
};
RabbitMqNext.LogAdapter.LogWarnFn = (scope, message, exc) =>
{
Console.WriteLine("[Warn] " + scope + " - " + message + " exception " + exc);
};
RabbitMqNext.LogAdapter.LogDebugFn = (scope, message, exc) =>
{
Console.WriteLine("[Dbg] " + scope + " - " + message + " exception " + exc);
};
// Warm up
// {
// var conn2 =
// await
// RabbitMqNext.ConnectionFactory.Connect(host, vhost, user, pwd, recoverySettings: null,
// connectionName: "perf_client");
//
// var channel2 = await conn2.CreateChannel();
// var q = "q." + 0;
// await Task.Delay(1); // Thread switch
// await SendModernCalls(q, channel2, howManyCalls/4, isWarmUp: true);
// await Task.Delay(1); // Thread switch
// channel2.Dispose();
// conn2.Dispose();
// }
//
// await WarmupComplete(postWarmupDelay);
Console.WriteLine("Will initiate...");
RabbitMqNext.IConnection conn = null;
RabbitMqNext.IChannel channel = null;
for (int i = 0; i < howManyQueues; i++)
{
if (exclusiveConnections || conn == null)
{
conn =
await
RabbitMqNext.ConnectionFactory.Connect(host, vhost, user, pwd, recoverySettings : null,
connectionName : "perf_client");
channel = await conn.CreateChannel();
}
var q = "q." + i;
new Thread(async() =>
{
await SendModernCalls(q, channel, howManyCalls, isWarmUp: false);
})
{
IsBackground = true
}.Start();
}
}
_startSync.Set();
Console.WriteLine("Waiting completion");
await Task.Delay(1); // switch
_completionSemaphore.Wait();
Console.WriteLine("Done\r\n");
Console.WriteLine("howManyQueues {0} exclusive Connections: {1} official client: {2} count {3}", howManyQueues,
exclusiveConnections, useOfficialClient, _hdrHistogram.TotalCount);
Console.WriteLine("\r\n");
_hdrHistogram.OutputPercentileDistribution(Console.Out);
Console.ReadKey();
}
public static void Run()
{
_outputStream = File.Create(LogPath);
_logWriter = new HistogramLogWriter(_outputStream);
_logWriter.Write(DateTime.Now);
var recorder = HistogramFactory
.With64BitBucketSize()
?.WithValuesFrom(1)
?.WithValuesUpTo(2345678912345)
?.WithPrecisionOf(3)
?.WithThreadSafeWrites()
?.WithThreadSafeReads()
?.Create();
var accumulatingHistogram = new LongHistogram(2345678912345, 3);
var size = accumulatingHistogram.GetEstimatedFootprintInBytes();
RILogManager.Default?.SendDebug("Histogram size = {0} bytes ({1:F2} MB)", size, size / 1024.0 / 1024.0);
RILogManager.Default?.SendDebug("Recorded latencies [in system clock ticks]");
accumulatingHistogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.None, useCsvFormat: true);
Console.WriteLine();
RILogManager.Default?.SendDebug("Recorded latencies [in usec]");
accumulatingHistogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToMicroseconds, useCsvFormat: true);
Console.WriteLine();
RILogManager.Default?.SendDebug("Recorded latencies [in msec]");
accumulatingHistogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToMilliseconds, useCsvFormat: true);
Console.WriteLine();
RILogManager.Default?.SendDebug("Recorded latencies [in sec]");
accumulatingHistogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToSeconds, useCsvFormat: true);
DocumentResults(accumulatingHistogram, recorder);
RILogManager.Default?.SendDebug("Build Vocabulary.");
DocumentResults(accumulatingHistogram, recorder);
Vocabulary vocabulary = new Vocabulary();
DocumentResults(accumulatingHistogram, recorder);
string trainPath = InternetFileDownloader.Download(DOWNLOAD_URL + TRAIN_FILE, TRAIN_FILE);
DocumentResults(accumulatingHistogram, recorder);
string validPath = InternetFileDownloader.Download(DOWNLOAD_URL + VALID_FILE, VALID_FILE);
DocumentResults(accumulatingHistogram, recorder);
string testPath = InternetFileDownloader.Download(DOWNLOAD_URL + TEST_FILE, TEST_FILE);
DocumentResults(accumulatingHistogram, recorder);
int[] trainData = vocabulary.LoadData(trainPath);
DocumentResults(accumulatingHistogram, recorder);
int[] validData = vocabulary.LoadData(validPath);
DocumentResults(accumulatingHistogram, recorder);
int[] testData = vocabulary.LoadData(testPath);
DocumentResults(accumulatingHistogram, recorder);
int nVocab = vocabulary.Length;
RILogManager.Default?.SendDebug("Network Initializing.");
FunctionStack model = new FunctionStack("Test10",
new EmbedID(nVocab, N_UNITS, name: "l1 EmbedID"),
new Dropout(),
new LSTM(true, N_UNITS, N_UNITS, name: "l2 LSTM"),
new Dropout(),
new LSTM(true, N_UNITS, N_UNITS, name: "l3 LSTM"),
new Dropout(),
new Linear(true, N_UNITS, nVocab, name: "l4 Linear")
);
DocumentResults(accumulatingHistogram, recorder);
// Do not cease at the given threshold, correct the rate by taking the rate from L2Norm of all parameters
GradientClipping gradientClipping = new GradientClipping(threshold: GRAD_CLIP);
SGD sgd = new SGD(learningRate: 1);
model.SetOptimizer(gradientClipping, sgd);
DocumentResults(accumulatingHistogram, recorder);
Real wholeLen = trainData.Length;
int jump = (int)Math.Floor(wholeLen / BATCH_SIZE);
int epoch = 0;
Stack <NdArray[]> backNdArrays = new Stack <NdArray[]>();
RILogManager.Default?.SendDebug("Train Start.");
double dVal;
NdArray x = new NdArray(new[] { 1 }, BATCH_SIZE, (Function)null);
NdArray t = new NdArray(new[] { 1 }, BATCH_SIZE, (Function)null);
for (int i = 0; i < jump * N_EPOCH; i++)
{
for (int j = 0; j < BATCH_SIZE; j++)
{
x.Data[j] = trainData[(int)((jump * j + i) % wholeLen)];
t.Data[j] = trainData[(int)((jump * j + i + 1) % wholeLen)];
}
NdArray[] result = model.Forward(true, x);
Real sumLoss = new SoftmaxCrossEntropy().Evaluate(result, t);
backNdArrays.Push(result);
RILogManager.Default?.SendDebug("[{0}/{1}] Loss: {2}", i + 1, jump, sumLoss);
//Run truncated BPTT
if ((i + 1) % BPROP_LEN == 0)
{
for (int j = 0; backNdArrays.Count > 0; j++)
{
RILogManager.Default?.SendDebug("backward" + backNdArrays.Count);
model.Backward(true, backNdArrays.Pop());
}
model.Update();
model.ResetState();
}
if ((i + 1) % jump == 0)
{
epoch++;
RILogManager.Default?.SendDebug("evaluate");
dVal = Evaluate(model, validData);
RILogManager.Default?.SendDebug($"validation perplexity: {dVal}");
if (epoch >= 6)
{
sgd.LearningRate /= 1.2;
RILogManager.Default?.SendDebug("learning rate =" + sgd.LearningRate);
}
}
DocumentResults(accumulatingHistogram, recorder);
}
RILogManager.Default?.SendDebug("test start");
dVal = Evaluate(model, testData);
RILogManager.Default?.SendDebug("test perplexity:" + dVal);
DocumentResults(accumulatingHistogram, recorder);
_logWriter.Dispose();
_outputStream.Dispose();
RILogManager.Default?.SendDebug("Log contents");
RILogManager.Default?.SendDebug(File.ReadAllText(LogPath));
Console.WriteLine();
RILogManager.Default?.SendDebug("Percentile distribution (values reported in milliseconds)");
accumulatingHistogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToMilliseconds, useCsvFormat: true);
RILogManager.Default?.SendDebug("Mean: " + BytesToString(accumulatingHistogram.GetMean()) + ", StdDev: " +
BytesToString(accumulatingHistogram.GetStdDeviation()));
}
