private static Task ReportMetricsAsync(TextWriter writer,
Metrics metrics,
TimeSpan runDuration)
{
var message = new StringBuilder();
var metric = default(long);
// Run time
var runDurationMilliseconds = Interlocked.CompareExchange(ref metrics.RunDurationMilliseconds, 0.0, 0.0);
var currentDuration = TimeSpan.FromMilliseconds(runDurationMilliseconds > 0.0 ? runDurationMilliseconds : 1);
var averageMemory = metrics.TotalMemoryUsed / metrics.MemorySamples;
message.AppendLine("Run Metrics");
message.AppendLine("=========================");
message.AppendLine($"\tRun Duration:\t\t\t{ runDuration.ToString(@"dd\.hh\:mm\:ss") }");
message.AppendLine($"\tElapsed:\t\t\t{ currentDuration.ToString(@"dd\.hh\:mm\:ss") } ({ (currentDuration / runDuration).ToString("P", CultureInfo.InvariantCulture) })");
message.AppendLine($"\tTotal Processor Time:\t\t{ metrics.TotalProcessorTime.ToString(@"dd\.hh\:mm\:ss") }");
message.AppendLine($"\tAverage Memory Use:\t\t{ FormatBytes(averageMemory) }");
message.AppendLine($"\tCurrent Memory Use:\t\t{ FormatBytes(metrics.MemoryUsed) }");
message.AppendLine($"\tPeak Memory Use:\t\t{ FormatBytes(metrics.PeakPhysicalMemory) }");
message.AppendLine($"\tGC Gen 0 Collections:\t\t{ metrics.GenerationZeroCollections.ToString("n0") }");
message.AppendLine($"\tGC Gen 1 Collections:\t\t{ metrics.GenerationOneCollections.ToString("n0") }");
message.AppendLine($"\tGC Gen 2 Collections:\t\t{ metrics.GenerationTwoCollections.ToString("n0") }");
message.AppendLine();
// Publish and read pairing
message.AppendLine("Processing");
message.AppendLine("=========================");
var serviceOps = (double)Interlocked.Read(ref metrics.TotalServiceOperations);
message.AppendLine($"\tService Operations:\t\t{ serviceOps.ToString("n0") }");
serviceOps = (serviceOps > 0) ? serviceOps : 0.001;
metric = Interlocked.Read(ref metrics.EventHandlerCalls);
message.AppendLine($"\tEvent Handler Calls:\t\t{ metric.ToString("n0") }");
message.AppendLine();
// Validation issues
message.AppendLine("Unexpected Events and Client Health");
message.AppendLine("===================================");
metric = Interlocked.Read(ref metrics.EventsRead);
message.AppendLine($"\tUnknown Events Read:\t\t{ metric.ToString("n0") }");
metric = Interlocked.Read(ref metrics.ProcessorRestarted);
message.AppendLine($"\tProcessor Restarts:\t\t{ metric }");
message.AppendLine();
// Exceptions
message.AppendLine("Exception Breakdown");
message.AppendLine("=========================");
var totalExceptions = (double)Interlocked.Read(ref metrics.TotalExceptions);
message.AppendLine($"\tExceptions for All Operations:\t{ totalExceptions.ToString("n0") } ({ (totalExceptions / serviceOps).ToString("P", CultureInfo.InvariantCulture) })");
totalExceptions = (totalExceptions > 0) ? totalExceptions : 0.001;
metric = Interlocked.Read(ref metrics.ProcessingExceptions);
message.AppendLine($"\tException During Processing:\t{ metric.ToString("n0") } ({ (metric / totalExceptions).ToString("P", CultureInfo.InvariantCulture) })");
metric = Interlocked.Read(ref metrics.GeneralExceptions);
message.AppendLine($"\tGeneral Exceptions:\t\t{ metric.ToString("n0") } ({ (metric / totalExceptions).ToString("P", CultureInfo.InvariantCulture) })");
metric = Interlocked.Read(ref metrics.TimeoutExceptions);
message.AppendLine($"\tTimeout Exceptions:\t\t{ metric.ToString("n0") } ({ (metric / totalExceptions).ToString("P", CultureInfo.InvariantCulture) })");
metric = Interlocked.Read(ref metrics.CommunicationExceptions);
message.AppendLine($"\tCommunication Exceptions:\t{ metric.ToString("n0") } ({ (metric / totalExceptions).ToString("P", CultureInfo.InvariantCulture) })");
metric = Interlocked.Read(ref metrics.ServiceBusyExceptions);
message.AppendLine($"\tService Busy Exceptions:\t{ metric.ToString("n0") } ({ (metric / totalExceptions).ToString("P", CultureInfo.InvariantCulture) })");
// Spacing
message.AppendLine();
message.AppendLine();
message.AppendLine();
message.AppendLine();
return(writer.WriteLineAsync(message.ToString()));
}
public async Task Start(CancellationToken cancellationToken)
{
IsRunning = true;
using var process = Process.GetCurrentProcess();
using var publishCancellationSource = new CancellationTokenSource();
using var processorCancellationSource = new CancellationTokenSource();
var publishingTask = default(Task);
var processorTasks = default(IEnumerable <Task>);
var runDuration = Stopwatch.StartNew();
try
{
// Start processing.
processorTasks = Enumerable
.Range(0, Configuration.ProcessorCount)
.Select(_ => Task.Run(() => new Processor(Configuration, Metrics, ErrorsObserved, ProcessEventHandler, ProcessErrorHandler).Start(processorCancellationSource.Token)))
.ToList();
// Test for missing events and update metrics.
var eventDueInterval = TimeSpan.FromMinutes(Configuration.EventReadLimitMinutes);
while (!cancellationToken.IsCancellationRequested)
{
Metrics.UpdateEnvironmentStatistics(process);
Interlocked.Exchange(ref Metrics.RunDurationMilliseconds, runDuration.Elapsed.TotalMilliseconds);
await Task.Delay(TimeSpan.FromMinutes(5), cancellationToken).ConfigureAwait(false);
}
}
catch (TaskCanceledException)
{
// No action needed.
}
catch (Exception ex) when
(ex is OutOfMemoryException ||
ex is StackOverflowException ||
ex is ThreadAbortException)
{
throw;
}
catch (Exception ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.GeneralExceptions);
ErrorsObserved.Add(ex);
}
// The run is ending. Clean up the outstanding background operations and
// complete the necessary metrics tracking.
try
{
publishCancellationSource.Cancel();
await publishingTask.ConfigureAwait(false);
// Wait a bit after publishing has completed before signaling for
// processing to be canceled, to allow any recently published
// events to be read.
await Task.Delay(TimeSpan.FromMinutes(2)).ConfigureAwait(false);
processorCancellationSource.Cancel();
await Task.WhenAll(processorTasks).ConfigureAwait(false);
}
catch (Exception ex)
{
Interlocked.Increment(ref Metrics.TotalExceptions);
Interlocked.Increment(ref Metrics.GeneralExceptions);
ErrorsObserved.Add(ex);
}
finally
{
runDuration.Stop();
IsRunning = false;
}
}
