public bool TryGetStats(Guid environmentId, out FailureStats stats)
{
if (_errorsPerEnvironment.TryGetValue(environmentId, out stats))
{
return(true);
}
return(false);
}
public void Execute(string databaseName, Exception e, Guid environmentId)
{
var stats = _errorsPerEnvironment.GetOrAdd(environmentId, x => FailureStats.Create(MaxDatabaseUnloads));
if (stats.WillUnloadDatabase == false)
{
if (DateTime.UtcNow - stats.LastUnloadTime > NoFailurePeriod)
{
// let it unload again after it was working fine for a given time with no failure
stats.NumberOfUnloads = 0;
stats.LastUnloadTime = DateTime.MinValue;
}
else
{
return;
}
}
stats.DatabaseUnloadTask = Task.Run(async() =>
{
var title = $"Critical error in '{databaseName}'";
const string message = "Database is about to be unloaded due to an encountered error";
try
{
_serverStore.NotificationCenter.Add(AlertRaised.Create(
databaseName,
title,
message,
AlertType.CatastrophicDatabaseFailure,
NotificationSeverity.Error,
key: databaseName,
details: new ExceptionDetails(e)));
}
catch (Exception)
{
// exception in raising an alert can't prevent us from unloading a database
}
if (_logger.IsOperationsEnabled)
{
_logger.Operations($"{title}. {message}", e);
}
// let it propagate the exception to the client first and do
// the internal failure handling e.g. Index.HandleIndexCorruption
await Task.Delay(TimeToWaitBeforeUnloadingDatabase);
stats.NumberOfUnloads++;
stats.LastUnloadTime = DateTime.UtcNow;
(await _databasesLandlord.UnloadAndLockDatabase(databaseName, "CatastrophicFailure"))?.Dispose();
stats.DatabaseUnloadTask = null;
});
}
