private PerformanceHint GetOrCreatePerformanceLatencies(out RequestLatencyDetail details)
{
//Read() is transactional, so this is thread-safe
using (_notificationsStorage.Read(QueryRequestLatenciesId, out var ntv))
{
if (ntv == null || ntv.Json.TryGet(nameof(PerformanceHint.Details), out BlittableJsonReaderObject detailsJson) == false || detailsJson == null)
{
details = new RequestLatencyDetail();
}
else
{
details = (RequestLatencyDetail)EntityToBlittable.ConvertToEntity(
typeof(RequestLatencyDetail),
QueryRequestLatenciesId,
detailsJson,
DocumentConventions.Default);
}
return(PerformanceHint.Create(
_database,
"Request latency is too high",
"We have detected that some query duration has surpassed the configured threshold",
PerformanceHintType.RequestLatency,
NotificationSeverity.Warning,
"Query",
details
));
}
}
private PerformanceHint GetPagingPerformanceHint(string id, PagingOperationType type)
{
using (_notificationsStorage.Read(id, out NotificationTableValue ntv))
{
PagingPerformanceDetails details;
if (ntv == null || ntv.Json.TryGet(nameof(PerformanceHint.Details), out BlittableJsonReaderObject detailsJson) == false || detailsJson == null)
{
details = new PagingPerformanceDetails();
}
else
{
details = DocumentConventions.DefaultForServer.Serialization.DefaultConverter.FromBlittable <PagingPerformanceDetails>(detailsJson, id);
}
switch (type)
{
case PagingOperationType.Documents:
case PagingOperationType.Queries:
return(PerformanceHint.Create(_database, $"Page size too big ({type.ToString().ToLower()})", "We have detected that some of the requests are returning excessive amount of documents. Consider using smaller page sizes or streaming operations.", PerformanceHintType.Paging, NotificationSeverity.Warning, type.ToString(), details));
case PagingOperationType.Revisions:
return(PerformanceHint.Create(_database, "Page size too big (revisions)", "We have detected that some of the requests are returning excessive amount of revisions. Consider using smaller page sizes.", PerformanceHintType.Paging, NotificationSeverity.Warning, type.ToString(), details));
case PagingOperationType.CompareExchange:
return(PerformanceHint.Create(_database, "Page size too big (compare exchange)", "We have detected that some of the requests are returning excessive amount of compare exchange values. Consider using smaller page sizes.", PerformanceHintType.Paging, NotificationSeverity.Warning, type.ToString(), details));
default:
throw new ArgumentOutOfRangeException(nameof(type), type, null);
}
}
}
private PerformanceHint GetOrCreateSlowWrites(out SlowWritesDetails details)
{
const string source = "slow-writes";
var id = PerformanceHint.GetKey(PerformanceHintType.SlowIO, source);
using (_notificationsStorage.Read(id, out var ntv))
{
if (ntv == null || ntv.Json.TryGet(nameof(PerformanceHint.Details), out BlittableJsonReaderObject detailsJson) == false || detailsJson == null)
{
details = new SlowWritesDetails();
}
else
{
details = DocumentConventions.DefaultForServer.Serialization.DefaultConverter.FromBlittable <SlowWritesDetails>(detailsJson);
}
return(PerformanceHint.Create(
_database,
"An extremely slow write to disk",
"We have detected very slow writes",
PerformanceHintType.SlowIO,
NotificationSeverity.Info,
source,
details
));
}
}
private PerformanceHint GetOrCreatePerformanceHint(out HugeDocumentsDetails details)
{
//Read() is transactional, so this is thread-safe
using (_notificationsStorage.Read(HugeDocumentsId, out var ntv))
{
if (ntv == null || ntv.Json.TryGet(nameof(PerformanceHint.Details), out BlittableJsonReaderObject detailsJson) == false || detailsJson == null)
{
details = new HugeDocumentsDetails();
}
else
{
details = DocumentConventions.DefaultForServer.Serialization.DefaultConverter.FromBlittable <HugeDocumentsDetails>(detailsJson, HugeDocumentsId);
}
string message = $"We have detected that some documents has surpassed the configured size threshold ({new Size(_maxWarnSize, SizeUnit.Bytes)}). It might have performance impact. You can alter warning limits by changing '{RavenConfiguration.GetKey(x => x.PerformanceHints.HugeDocumentSize)}' configuration value.";
return(PerformanceHint.Create(
_database,
"Huge documents",
message,
PerformanceHintType.HugeDocuments,
NotificationSeverity.Warning,
PerformanceHintSource,
details
));
}
}
public Dictionary <string, long> GetLastProcessedDocumentTombstonesPerCollection()
{
var minEtag = MinimalEtagForReplication;
var result = new Dictionary <string, long>(StringComparer.OrdinalIgnoreCase)
{
{ Constants.Documents.Collections.AllDocumentsCollection, minEtag }
};
if (Destinations == null)
{
return(result);
}
ReplicationNode disabledReplicationNode = null;
bool hasDisabled = false;
foreach (var replicationDocumentDestination in Destinations)
{
if (replicationDocumentDestination.Disabled)
{
disabledReplicationNode = replicationDocumentDestination;
hasDisabled = true;
break;
}
}
if (hasDisabled == false)
{
return(result);
}
const int maxTombstones = 16 * 1024;
bool tooManyTombstones;
using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
tooManyTombstones = Database.DocumentsStorage.HasMoreOfTombstonesAfter(context, minEtag, maxTombstones);
}
if (!tooManyTombstones)
{
return(result);
}
Database.NotificationCenter.Add(
PerformanceHint.Create(
database: Database.Name,
title: "Large number of tombstones because of disabled replication destination",
msg:
$"The disabled replication destination {disabledReplicationNode.FromString()} prevents from cleaning large number of tombstones.",
type: PerformanceHintType.Replication,
notificationSeverity: NotificationSeverity.Warning,
source: disabledReplicationNode.FromString()
));
return(result);
}
private static List <Notification> CreateSampleNotificationsForFilterOutTest()
{
return(new List <Notification>
{
AlertRaised.Create(
null,
"DatabaseTopologyWarning",
"DatabaseTopologyWarning_MSG",
AlertType.DatabaseTopologyWarning,
NotificationSeverity.Info),
DatabaseChanged.Create(null, DatabaseChangeType.Put), // filtered out, DatabaseChange
AlertRaised.Create(
null,
"LicenseManager_AGPL3",
"LicenseManager_AGPL3_MSG",
AlertType.ClusterTransactionFailure,
NotificationSeverity.Info),
AlertRaised.Create(
null,
"LicenseManager_AGPL3",
"LicenseManager_AGPL3_MSG",
AlertType.LicenseManager_AGPL3, // filtered out explicitly
NotificationSeverity.Info),
AlertRaised.Create(
null,
"RevisionsConfigurationNotValid",
"RevisionsConfigurationNotValid_MSG",
AlertType.RevisionsConfigurationNotValid, // filtered out explicitly
NotificationSeverity.Info),
AlertRaised.Create(
null,
"Certificates_ReplaceError",
"Certificates_ReplaceError_MSG",
AlertType.Certificates_ReplaceError,
NotificationSeverity.Info),
PerformanceHint.Create(
null,
"SlowIO",
"SlowIO_MSG",
PerformanceHintType.SlowIO, // filtered out, PerformanceHint
NotificationSeverity.Info,
"test"),
PerformanceHint.Create(
null,
"SqlEtl_SlowSql",
"SqlEtl_SlowSql_MSG",
PerformanceHintType.SqlEtl_SlowSql, // filtered out, PerformanceHint
NotificationSeverity.Info,
"test"),
OperationChanged.Create(null, 1, new Operations.OperationDescription(), new OperationState()
{
Result = new PersistableResult()
}, false),
DatabaseChanged.Create(null, DatabaseChangeType.Delete) // filtered out, DatabaseChange
});
}
public bool Update(UpdateStep step)
{
var table = step.WriteTx.OpenTable(step.ConfigurationStorage.NotificationsStorage._actionsSchema, NotificationsStorage.NotificationsSchema.NotificationsTree);
using (Slice.From(step.WriteTx.Allocator, PerformanceHint.GetKey(PerformanceHintType.SlowIO, string.Empty), out Slice slowIoHintPrefix))
{
table.DeleteByPrimaryKeyPrefix(slowIoHintPrefix);
}
return(true);
}
private void AddHint(string id, int size)
{
lock (_addHintSyncObj)
{
if (_performanceHint == null)
{
_performanceHint = GetOrCreatePerformanceHint(out _details);
}
_details.Update(id, size);
_needsSync = true;
if (_timer != null)
{
return;
}
_timer = new Timer(UpdateHugeDocuments, null, TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1));
}
}
public void AddHint(long duration, string action, string query)
{
lock (_addHintSyncObj)
{
if (_performanceHint == null)
{
_performanceHint = GetOrCreatePerformanceLatencies(out _details);
}
_details.Update(duration, action, query);
_needsSync = true;
if (_timer != null)
{
return;
}
_timer = new Timer(UpdateRequestLatency, null, TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1));
}
}
private PerformanceHint GetOrCreatePerformanceHint <T>(string processTag, string processName, PerformanceHintType etlHintType, string message, out T details) where T : INotificationDetails, new()
{
Debug.Assert(etlHintType == PerformanceHintType.SqlEtl_SlowSql);
var key = $"{processTag}/{processName}";
var id = PerformanceHint.GetKey(etlHintType, key);
using (_notificationsStorage.Read(id, out NotificationTableValue ntv))
{
details = GetDetails <T>(ntv);
return(PerformanceHint.Create(
_databaseName,
$"{processTag}: '{processName}'",
message,
etlHintType,
NotificationSeverity.Warning,
source: key,
details: details));
}
}
public static void Notify(CommitStats stats, DocumentDatabase database)
{
if (stats.NumberOf4KbsWrittenToDisk == 0 ||
// we don't want to raise the error too often
stats.WriteToJournalDuration.TotalMilliseconds < 500)
{
return;
}
var writtenDataInMb = stats.NumberOf4KbsWrittenToDisk / (double)256;
var seconds = stats.WriteToJournalDuration.TotalSeconds;
var rateOfWritesInMbPerSec = writtenDataInMb / seconds;
if (rateOfWritesInMbPerSec < 1)
{
database.NotificationCenter.Add(PerformanceHint.Create(database.Name,
$"An extremely slow write to disk.",
$"We wrote {writtenDataInMb:N} MB in {seconds:N} seconds ({rateOfWritesInMbPerSec:N} MB/s) to: '{stats.JournalFilePath}'",
PerformanceHintType.SlowIO,
NotificationSeverity.Info,
$"TxMerger/{Path.GetDirectoryName(stats.JournalFilePath)}"
));
}
}
internal void UpdatePaging(object state)
{
try
{
if (_pagingQueue.IsEmpty)
{
return;
}
PerformanceHint documents = null, queries = null, revisions = null, compareExchange = null;
while (_pagingQueue.TryDequeue(
out PagingInformation pagingInfo))
{
switch (pagingInfo.Type)
{
case PagingOperationType.Documents:
documents ??= GetPagingPerformanceHint(PagingDocumentsId, pagingInfo.Type);
((PagingPerformanceDetails)documents.Details).Update(pagingInfo);
break;
case PagingOperationType.Queries:
queries ??= GetPagingPerformanceHint(PagingQueriesId, pagingInfo.Type);
((PagingPerformanceDetails)queries.Details).Update(pagingInfo);
break;
case PagingOperationType.Revisions:
revisions ??= GetPagingPerformanceHint(PagingRevisionsId, pagingInfo.Type);
((PagingPerformanceDetails)revisions.Details).Update(pagingInfo);
break;
case PagingOperationType.CompareExchange:
compareExchange ??= GetPagingPerformanceHint(PagingCompareExchangeId, pagingInfo.Type);
((PagingPerformanceDetails)compareExchange.Details).Update(pagingInfo);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
if (documents != null)
{
_notificationCenter.Add(documents);
}
if (queries != null)
{
_notificationCenter.Add(queries);
}
if (revisions != null)
{
_notificationCenter.Add(revisions);
}
if (compareExchange != null)
{
_notificationCenter.Add(compareExchange);
}
}
catch (Exception e)
{
if (_logger.IsInfoEnabled)
{
_logger.Info("Error in a notification center paging timer", e);
}
}
}
private static PerformanceHint GetSamplePerformanceHint(string customSource = null)
{
return(PerformanceHint.Create("db", "title", "message", PerformanceHintType.None, NotificationSeverity.Info, source: customSource));
}
internal bool UpdatePagingInternal(object state, out string reasonOfNotUpdating)
{
var outcome = false;
reasonOfNotUpdating = "";
try
{
if (_pagingQueue.IsEmpty)
{
reasonOfNotUpdating += "Queue is empty";
return(false);
}
PerformanceHint documents = null, queries = null, revisions = null, compareExchange = null;
while (_pagingQueue.TryDequeue(
out PagingInformation pagingInfo))
{
switch (pagingInfo.Type)
{
case PagingOperationType.Documents:
documents ??= GetPagingPerformanceHint(PagingDocumentsId, pagingInfo.Type);
((PagingPerformanceDetails)documents.Details).Update(pagingInfo);
break;
case PagingOperationType.Queries:
queries ??= GetPagingPerformanceHint(PagingQueriesId, pagingInfo.Type);
((PagingPerformanceDetails)queries.Details).Update(pagingInfo);
break;
case PagingOperationType.Revisions:
revisions ??= GetPagingPerformanceHint(PagingRevisionsId, pagingInfo.Type);
((PagingPerformanceDetails)revisions.Details).Update(pagingInfo);
break;
case PagingOperationType.CompareExchange:
compareExchange ??= GetPagingPerformanceHint(PagingCompareExchangeId, pagingInfo.Type);
((PagingPerformanceDetails)compareExchange.Details).Update(pagingInfo);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
if (documents != null)
{
_notificationCenter.Add(documents);
outcome = true;
}
if (queries != null)
{
_notificationCenter.Add(queries);
outcome = true;
}
if (revisions != null)
{
_notificationCenter.Add(revisions);
outcome = true;
}
if (compareExchange != null)
{
_notificationCenter.Add(compareExchange);
outcome = true;
}
}
catch (Exception e)
{
if (_logger.IsInfoEnabled)
{
_logger.Info("Error in a notification center paging timer", e);
}
outcome = false;
reasonOfNotUpdating += $"Error in a notification center paging timer. {e}";
}
return(outcome);
}