public DatabaseHealthCheck(
String dbDescription,
String serverUrl)
: base("MongoDatabaseCheck: " + dbDescription)
{
_dbDescription = dbDescription;
HealthChecks.RegisterHealthCheck(this);
_serverUrl = serverUrl;
}
public void HealthCheck_RegistryDoesNotThrowOnDuplicateRegistration()
{
HealthChecks.UnregisterAllHealthChecks();
HealthChecks.RegisterHealthCheck(new HealthCheck("test", () => { }));
Action action = () => HealthChecks.RegisterHealthCheck(new HealthCheck("test", () => { }));
action.ShouldNotThrow <InvalidOperationException>();
}
public void HealthCheck_RegistryStatusIsHealthyIfAllChecksAreHealthy()
{
HealthChecks.UnregisterAllHealthChecks();
HealthChecks.RegisterHealthCheck(new HealthCheck("ok", () => { }));
HealthChecks.RegisterHealthCheck(new HealthCheck("another", () => HealthCheckResult.Healthy()));
var status = HealthChecks.GetStatus();
status.IsHealthy.Should().BeTrue();
status.Results.Length.Should().Be(2);
}
public void HealthCheck_RegistryStatusIsFailedIfOneCheckFails()
{
HealthChecks.UnregisterAllHealthChecks();
HealthChecks.RegisterHealthCheck(new HealthCheck("ok", () => { }));
HealthChecks.RegisterHealthCheck(new HealthCheck("bad", () => HealthCheckResult.Unhealthy()));
var status = HealthChecks.GetStatus();
status.IsHealthy.Should().BeFalse();
status.Results.Length.Should().Be(2);
}
private void RegisterHealthCheck()
{
HealthChecks.RegisterHealthCheck("Projection Engine Errors", () =>
{
if (_engineFatalErrors.Count == 0)
{
return(HealthCheckResult.Healthy("No error in projection engine"));
}
return(HealthCheckResult.Unhealthy("Error occurred in projection engine: " + String.Join("\n\n", _engineFatalErrors)));
});
}
public void Start()
{
var stats = _loader.GetSlotMetrics();
foreach (var stat in stats)
{
var slotName = stat.Name;
SetCheckpointBehind(stat.Name, () => _loader.GetSlotMetric(slotName).CommitBehind);
HealthChecks.RegisterHealthCheck("Slot-" + slotName, CheckSlotHealth(slotName));
}
SetCheckpointBehind("ALLSLOT", () => _loader.GetSlotMetrics().Max(d => d.CommitBehind));
}
public Startup(IConfiguration configuration, IHostingEnvironment env)
{
Configuration = configuration;
this.env = env;
Metric.Config.WithHttpEndpoint("http://*****:*****@"c:\temp\reports\", TimeSpan.FromSeconds(30))
.WithConsoleReport(TimeSpan.FromSeconds(30))
.WithTextFileReport(@"C:\temp\reports\metrics.txt", TimeSpan.FromSeconds(30))
);
HealthChecks.RegisterHealthCheck(new MetricsHealthCheck("https://www.random.org/integers/?num=1&min=1&max=100&col=1&base=10&format=plain&rnd=new"));
}
private static void UseHealthChecks(IApplicationBuilder app)
{
var healthChecks = app.ApplicationServices.GetServices <HealthCheck>();
if (healthChecks == null || !healthChecks.Any())
{
return;
}
foreach (var check in healthChecks)
{
HealthChecks.RegisterHealthCheck(check);
}
}
public void HealthCheck_RegistryExecutesCheckOnEachGetStatus()
{
HealthChecks.UnregisterAllHealthChecks();
int count = 0;
HealthChecks.RegisterHealthCheck(new HealthCheck("test", () => { count++; }));
count.Should().Be(0);
HealthChecks.GetStatus();
count.Should().Be(1);
HealthChecks.GetStatus();
count.Should().Be(2);
}
public ConsumerActionWrapper(String consumerId, Func <IChunk, Task> consumerAction, CommitPollingClient2 owner)
{
_consumerAction = consumerAction;
_active = true;
_owner = owner;
_consumerId = consumerId;
HealthChecks.RegisterHealthCheck("PollingConsumer-" + consumerId, () =>
{
if (string.IsNullOrEmpty(_error))
{
return(HealthCheckResult.Healthy($"Consumer {_consumerId} healthy"));
}
return(HealthCheckResult.Unhealthy($"Consumer {_consumerId} error: " + _error));
});
}
public void Start()
{
Metric.Gauge("checkpoint-behind-AtomicReadModels", CheckAllAtomicsValue(), Unit.Items);
HealthChecks.RegisterHealthCheck("Slot-All-AtomicReadmodels", CheckSlotHealth());
foreach (var readModelType in _atomicProjectionCheckpointManager.GetAllRegisteredAtomicReadModels())
{
var name = CollectionNames.GetCollectionName(readModelType);
//try to create an instance to grab the version
try
{
var readmodelVersion = _readModelFactory.GetReamdodelVersion(readModelType);
Metric.Gauge("versions-behind-" + name, () => _versionLoader.CountReadModelToUpdateByName(name, readmodelVersion), Unit.Items);
}
catch (Exception ex)
{
Logger.ErrorFormat(ex, "Unable to setup metric for atomic readmodel {0}", readModelType.FullName);
}
}
}
public void IncludeHealthCheck(string name, Func <bool> check)
{
//Wrap it in a delegate
Func <HealthCheckResult> healthCheck = () =>
{
bool success = check();
if (!success)
{
return(HealthCheckResult.Unhealthy());
}
return(HealthCheckResult.Healthy());
};
HealthChecks.RegisterHealthCheck(name, healthCheck);
Checks.Add(name);
}
public void StartAutomaticPolling(
String checkpointTokenFrom,
Int32 intervalInMilliseconds,
Int32 bufferSize = 4000,
String pollerName = "CommitPollingClient")
{
StartManualPolling(checkpointTokenFrom, bufferSize, pollerName);
_pollerTimer = new System.Timers.Timer(intervalInMilliseconds);
_pollerTimer.Elapsed += TimerCallback;
_pollerTimer.Start();
//Add an immediate poll
_commandList.Add(command_poll);
//Set health check for polling
HealthChecks.RegisterHealthCheck("Polling-" + pollerName, () =>
{
if (Status == CommitPollingClientStatus.Stopped)
{
//poller is stopped, system healty
return(HealthCheckResult.Healthy("Automatic polling stopped"));
}
else if (Status == CommitPollingClientStatus.Faulted)
{
//poller is stopped, system healty
var exception = (LastException != null ? LastException.ToString() : "");
exception = exception.Replace("{", "{{").Replace("}", "}}");
return(HealthCheckResult.Unhealthy("Faulted (exception in consumer): " + exception));
}
var elapsed = Math.Abs(Environment.TickCount - _lastActivityTickCount);
if (elapsed > 5000)
{
//more than 5 seconds without a poll, polling probably is stopped
return(HealthCheckResult.Unhealthy(String.Format("poller stuck, last polling {0} ms ago", elapsed)));
}
return(HealthCheckResult.Healthy("Poller alive"));
});
Status = CommitPollingClientStatus.Polling;
}
private void RegisterHealthChecks(string pollerName)
{
MetricsHelper.SetCommitPollingClientBufferSize(pollerName, () => GetClientBufferSize());
//Set health check for polling
HealthChecks.RegisterHealthCheck("Polling-" + pollerName, () =>
{
if (Status == CommitPollingClientStatus.Stopped)
{
//poller is stopped, system healty
return(HealthCheckResult.Healthy("Automatic polling stopped"));
}
else if (Status == CommitPollingClientStatus.Faulted || LastException != null)
{
//poller is stopped, system healty
var exceptionText = (LastException != null ? LastException.ToString() : "");
exceptionText = exceptionText.Replace("{", "{{").Replace("}", "}}");
return(HealthCheckResult.Unhealthy("Faulted (exception in consumer): " + exceptionText));
}
var elapsed = DateTime.UtcNow - _innerClient.LastActivityTimestamp;
if (elapsed.TotalMilliseconds > 5000)
{
//more than 5 seconds without a poll, polling probably is stopped
return(HealthCheckResult.Unhealthy(String.Format("poller stuck, last polling {0} ms ago", elapsed)));
}
return(HealthCheckResult.Healthy("Poller alive"));
});
//Now register health check for the internal NES poller, to diagnose errors in polling.
HealthChecks.RegisterHealthCheck("Polling internal errors: ", () =>
{
if (_innerClient != null && !String.IsNullOrEmpty(_innerClient.LastPollingError))
{
return(HealthCheckResult.Unhealthy($"Inner NES poller has error: {_innerClient.LastPollingError}"));
}
return(HealthCheckResult.Healthy("Inner NES Poller Ok"));
});
}
private void RegisterHealthChecks(string pollerName)
{
KernelMetricsHelper.SetCommitPollingClientBufferSize(pollerName, () => GetClientBufferSize());
//Set health check for polling
HealthChecks.RegisterHealthCheck("Polling-" + pollerName, () =>
{
if (Status == CommitPollingClientStatus.Stopped)
{
//poller is stopped, system healty
return(HealthCheckResult.Healthy("Automatic polling stopped"));
}
else if (Status == CommitPollingClientStatus.Faulted || LastException != null)
{
//poller is stopped, system is not healty anymore.
var exceptionText = (LastException != null ? LastException.ToString() : "");
exceptionText = exceptionText.Replace("{", "{{").Replace("}", "}}");
return(HealthCheckResult.Unhealthy(
"[LastDispatchedPosition: {0}] - Faulted (exception in consumer): {1}",
_lastDispatchedPosition,
exceptionText));
}
return(HealthCheckResult.Healthy("Poller alive"));
});
//Now register health check for the internal NES poller, to diagnose errors in polling.
HealthChecks.RegisterHealthCheck("Polling internal errors: ", () =>
{
if (_innerSubscription?.LastException != null)
{
return(HealthCheckResult.Unhealthy("[LastDispatchedPosition: {0}] - Inner NStore poller has error: {1}",
_lastDispatchedPosition,
_innerSubscription?.LastException));
}
return(HealthCheckResult.Healthy("Inner NES Poller Ok"));
});
}
/// <summary>
/// Registers the health check which will be periodically triggered
/// </summary>
public void RegisterHealthCheck()
{
HealthChecks.RegisterHealthCheck(CacheName, new Func <HealthCheckResult>(StartHealthCheck));
}
/// <summary>
/// Registers the health check which will be periodically triggered
/// </summary>
public void RegisterHealthCheck()
{
HealthChecks.RegisterHealthCheck("SportEventCache", new Func <HealthCheckResult>(StartHealthCheck));
}
public static void RegisterHealthCheck(string name, Func <HealthCheck.HealthCheckResult> getHealthCheck)
{
HealthChecks.RegisterHealthCheck(name, getHealthCheck);
}
/// <summary>
/// Registers the health check which will be periodically triggered
/// </summary>
public void RegisterHealthCheck()
{
HealthChecks.RegisterHealthCheck("MatchStatusCache", new Func <HealthCheckResult>(StartHealthCheck));
}
/// <summary>
/// Registers the health check which will be periodically triggered
/// </summary>
public void RegisterHealthCheck()
{
HealthChecks.RegisterHealthCheck("VariantMarketDescriptorCache", new Func <HealthCheckResult>(StartHealthCheck));
}
public DatabaseHealthCheck(String dbDescription, MongoServer server)
: base("MongoDatabaseCheck:" + dbDescription)
{
this.mongoServer = server;
HealthChecks.RegisterHealthCheck(this);
}
/// <summary>
/// Registers the health check which will be periodically triggered
/// </summary>
public void RegisterHealthCheck()
{
HealthChecks.RegisterHealthCheck("LogHttpDataFetcher", StartHealthCheck);
}
internal async Task DispatchEventAsync(UnwindedDomainEvent unwindedEvent)
{
if (unwindedEvent == UnwindedDomainEvent.LastEvent)
{
Finished = true;
_lastCheckpointRebuilded = LastCheckpointDispatched; //Set to zero metrics, we dispatched everything.
return;
}
var chkpoint = unwindedEvent.CheckpointToken;
if (chkpoint > LastCheckpointDispatched)
{
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("Discharded event {0} commit {1} because last checkpoint dispatched for slot {2} is {3}.", unwindedEvent.CommitId, unwindedEvent.CheckpointToken, SlotName, _maxCheckpointDispatched);
}
return;
}
Interlocked.Increment(ref RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit);
TenantContext.Enter(_config.TenantId);
try
{
string eventName = unwindedEvent.EventType;
foreach (var projection in _projections)
{
var cname = projection.Info.CommonName;
long elapsedticks;
try
{
QueryPerformanceCounter(out long ticks1);
await projection.HandleAsync(unwindedEvent.GetEvent(), true).ConfigureAwait(false);
QueryPerformanceCounter(out long ticks2);
elapsedticks = ticks2 - ticks1;
KernelMetricsHelper.IncrementProjectionCounterRebuild(cname, SlotName, eventName, elapsedticks);
}
catch (Exception ex)
{
_logger.FatalFormat(ex, "[Slot: {3} Projection: {4}] Failed checkpoint: {0} StreamId: {1} Event Name: {2}",
unwindedEvent.CheckpointToken,
unwindedEvent.PartitionId,
eventName,
SlotName,
cname
);
HealthChecks.RegisterHealthCheck($"RebuildDispatcher, slot {SlotName} - FailedCheckpoint {unwindedEvent.CheckpointToken}", () =>
HealthCheckResult.Unhealthy(ex)
);
throw;
}
_metrics.Inc(cname, eventName, elapsedticks);
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("[{3}] [{4}] Handled checkpoint {0}: {1} > {2}",
unwindedEvent.CheckpointToken,
unwindedEvent.PartitionId,
eventName,
SlotName,
cname
);
}
}
}
catch (Exception ex)
{
_logger.ErrorFormat(ex, "Error dispathing commit id: {0}\nMessage: {1}\nError: {2}",
unwindedEvent.CheckpointToken, unwindedEvent.Event, ex.Message);
HealthChecks.RegisterHealthCheck($"RebuildDispatcher, slot {SlotName} - GeneralError", () =>
HealthCheckResult.Unhealthy(ex)
);
throw;
}
_lastCheckpointRebuilded = chkpoint;
KernelMetricsHelper.MarkEventInRebuildDispatchedCount(SlotName, 1);
Interlocked.Decrement(ref RebuildProjectionMetrics.CountOfConcurrentDispatchingCommit);
}
/// <summary>
/// Registers the health check which will be periodically triggered
/// </summary>
public void RegisterHealthCheck()
{
HealthChecks.RegisterHealthCheck("LogHttpDataFetcher", new Func <HealthCheckResult>(StartHealthCheck));
}
static void Main(string[] args)
{
Metric.Config
//.WithAllCounters()
.WithReporting(config => config
.WithConsoleReport(TimeSpan.FromSeconds(30))
.WithApplicationInsights(ConfigurationManager.AppSettings["AI_InstrumentationKey"].ToString(),
Guid.NewGuid().ToString(), TimeSpan.FromSeconds(30))
);
// create some metrics
Counter counter = Metric.Counter("MyItemCounter", Unit.Items);
counter.Increment();
counter.Increment();
counter.Increment();
Counter counterSuppressed = Metric.Counter("MySuppressedCounter", Unit.Items, ApplicationInsightsReport.DoNotReport);
counterSuppressed.Increment();
Metric.Gauge("MyPercentageGauge", () => DateTime.Now.Second, Unit.Percent);
Meter meter = Metric.Meter("MyErrorsPerSecMeter", Unit.Errors, TimeUnit.Seconds);
meter.Mark();
meter.Mark();
meter.Mark();
meter.Mark();
meter.Mark();
meter.Mark();
meter.Mark();
meter.Mark();
meter.Mark();
meter.Mark();
Histogram histogram = Metric.Histogram("MyItemsHistogram", Unit.Items);
histogram.Update(456);
histogram.Update(123);
histogram.Update(789);
HealthChecks.RegisterHealthCheck("Seconds", () =>
{
if (DateTime.Now.Second > 55)
{
return(HealthCheckResult.Unhealthy("Seconds > 55"));
}
else
{
return(HealthCheckResult.Healthy("Seconds <= 55"));
}
});
HealthChecks.RegisterHealthCheck("AlwaysTrue", () => HealthCheckResult.Healthy("Always True check"));
var rnd = new Random();
Metrics.Timer timer = Metric.Timer("MyEventsTimer", Unit.Events);
for (int i = 0; i < 3; i++)
{
using (var context = timer.NewContext("TimerCtx"))
{
Thread.Sleep(132 + rnd.Next(15));
}
}
Console.WriteLine("Press any key to end...");
Console.ReadKey();
}
/// <summary>
/// Registers the health check which will be periodically triggered
/// </summary>
public void RegisterHealthCheck()
{
HealthChecks.RegisterHealthCheck("InvariantMarketDescriptorCache", StartHealthCheck);
}
/// <summary>
/// Registers the health check which will be periodically triggered
/// </summary>
public void RegisterHealthCheck()
{
HealthChecks.RegisterHealthCheck(CacheName, StartHealthCheck);
}
public void RegisterHealthCheck()
{
HealthChecks.RegisterHealthCheck("TestStatsCollector", new Func <HealthCheckResult>(StartHealthCheck));
}
public DatabaseHealthCheck(IDatabase database)
: base("DatabaseCheck")
{
this.database = database;
HealthChecks.RegisterHealthCheck(this);
}
/// <summary>
/// Registers the health check which will be periodically triggered
/// </summary>
public void RegisterHealthCheck()
{
HealthChecks.RegisterHealthCheck("SportEventStatusCache", StartHealthCheck);
}
