private PerformanceHint GetOrCreatePerformanceLatencies(out RequestLatencyDetail details)
{
//Read() is transactional, so this is thread-safe
using (_notificationsStorage.Read(QueryRequestLatenciesId, out var ntv))
{
if (ntv == null || ntv.Json.TryGet(nameof(PerformanceHint.Details), out BlittableJsonReaderObject detailsJson) == false || detailsJson == null)
{
details = new RequestLatencyDetail();
}
else
{
details = (RequestLatencyDetail)EntityToBlittable.ConvertToEntity(
typeof(RequestLatencyDetail),
QueryRequestLatenciesId,
detailsJson,
DocumentConventions.Default);
}
return(PerformanceHint.Create(
_database,
"Request latency is too high",
"We have detected that some query duration has surpassed the configured threshold",
PerformanceHintType.RequestLatency,
NotificationSeverity.Warning,
"Query",
details
));
}
}
private PerformanceHint GetOrCreateSlowWrites(out SlowWritesDetails details)
{
const string source = "slow-writes";
var id = PerformanceHint.GetKey(PerformanceHintType.SlowIO, source);
using (_notificationsStorage.Read(id, out var ntv))
{
if (ntv == null || ntv.Json.TryGet(nameof(PerformanceHint.Details), out BlittableJsonReaderObject detailsJson) == false || detailsJson == null)
{
details = new SlowWritesDetails();
}
else
{
details = DocumentConventions.DefaultForServer.Serialization.DefaultConverter.FromBlittable <SlowWritesDetails>(detailsJson);
}
return(PerformanceHint.Create(
_database,
"An extremely slow write to disk",
"We have detected very slow writes",
PerformanceHintType.SlowIO,
NotificationSeverity.Info,
source,
details
));
}
}
public void Add(Notification notification)
{
try
{
if (notification.IsPersistent)
{
try
{
if (_notificationsStorage.Store(notification) == false)
{
return;
}
}
catch (Exception e)
{
// if we fail to save the persistent notification in the storage,
// (OOME or any other storage error)
// we still want to send it to any of the connected watchers
if (Logger.IsInfoEnabled)
{
Logger.Info($"Failed to save a persistent notification '{notification.Id}' " +
$"to the notification center. " +
$"Title: {notification.Title}, message: {notification.Message}", e);
}
}
}
if (Watchers.Count == 0)
{
return;
}
using (_notificationsStorage.Read(notification.Id, out NotificationTableValue existing))
{
if (existing?.PostponedUntil > SystemTime.UtcNow)
{
return;
}
}
foreach (var watcher in Watchers)
{
// serialize to avoid race conditions
// please notice we call ToJson inside a loop since DynamicJsonValue is not thread-safe
watcher.NotificationsQueue.Enqueue(notification.ToJson());
}
}
catch (ObjectDisposedException)
{
// we are disposing
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info($"Failed to add notification '{notification.Id}' to the notification center. Title: {notification.Title}, message: {notification.Message}", e);
}
}
}
private AlertRaised GetOrCreateAlert <T>(string processTag, string processName, AlertType etlAlertType, string message, out T details) where T : INotificationDetails, new()
{
Debug.Assert(etlAlertType == AlertType.Etl_LoadError || etlAlertType == AlertType.Etl_TransformationError);
var key = $"{processTag}/{processName}";
var id = AlertRaised.GetKey(etlAlertType, key);
using (_notificationsStorage.Read(id, out NotificationTableValue ntv))
{
details = GetDetails <T>(ntv);
return(AlertRaised.Create(
_databaseName,
$"{processTag}: '{processName}'",
message,
etlAlertType,
NotificationSeverity.Warning,
key: key,
details: details));
}
}