// NOTE: this method is called on a background thread. Don't touch class members.
private static void WriteLogBatch(IList <Log> logs, IDocumentStore db)
{
// .db is a class member, but it's OK to access because it is a thread-safe class.
using (var dbSession = db.OpenSession())
{
var structuredLogIds = logs
.Select(l => (id: GetStructuredLogId(l), log: l))
.ToList();
var structuredLogs = dbSession.Load <StructuredLog>(structuredLogIds.Select(l => l.id));
foreach (var structuredLogInfo in structuredLogIds)
{
var existingStructuredLog = structuredLogs[structuredLogInfo.id];
if (existingStructuredLog == null)
{
existingStructuredLog = new StructuredLog();
dbSession.Store(existingStructuredLog, structuredLogInfo.id);
}
existingStructuredLog.AddLog(structuredLogInfo.log);
// Update the expiration time for this structured log.
var meta = dbSession.Advanced.GetMetadataFor(existingStructuredLog);
var expireDateIsoString = DateTime.UtcNow.AddDays(RavenStructuredLoggerProvider.ExpirationInDays).ToString("o", System.Globalization.CultureInfo.InvariantCulture);
meta["@expires"] = expireDateIsoString;
}
dbSession.SaveChanges();
}
}
// NOTE: this method is called on a background thread
private static void WriteLogBatch(IList <Log> logs, IDocumentStore db, bool hasRetried = false)
{
using (var dbSession = db.OpenSession())
{
var structuredLogIds = logs
.Select(l => (id: GetStructuredLogId(l), log: l))
.ToList();
var structuredLogs = dbSession.Load <StructuredLog>(structuredLogIds.Select(l => l.id).Distinct());
foreach (var logWithId in structuredLogIds)
{
var existingStructuredLog = structuredLogs[logWithId.id];
if (existingStructuredLog == null)
{
// We don't have a StructuredLog with this exact message.
// Do a fuzzy search using Raven Suggestions to see if there's log with a similar message.
// This is needed because some software will include things like timestamps in the error
// message, making the mesage vary slightly.
// If there's a similar message that varies slightly, use that as the group.
if (!string.IsNullOrWhiteSpace(logWithId.log.Template) || !string.IsNullOrWhiteSpace(logWithId.log.Message))
{
// For simple logs, Template may be null, and we may only have message.
var messageToSearch = string.IsNullOrWhiteSpace(logWithId.log.Template) ? logWithId.log.Message : logWithId.log.Template;
var suggestions = dbSession.Query <StructuredLog>()
.SuggestUsing(builder => builder
.ByField(l => l.MessageTemplate, messageToSearch)
.WithOptions(new Client.Documents.Queries.Suggestions.SuggestionOptions
{
Accuracy = 0.8f // 0.9 is too strict, won't match things that should be grouped. 0.8 seems about right.
}))
.Execute();
var firstSuggestion = suggestions.FirstOrDefault().Value?.Suggestions?.LastOrDefault();
if (firstSuggestion != null)
{
existingStructuredLog = dbSession.Query <StructuredLog>()
.Search(l => l.MessageTemplate, firstSuggestion)
.FirstOrDefault();
if (existingStructuredLog != null)
{
logWithId.log.GroupingDetails = "Couldn't find log with exact message match, so queried for suggestions. Using last suggestion \"" + firstSuggestion + "\". Full suggestions " + string.Join("; ", suggestions.Select(s => $"Key: {s.Key}, Value.Name: {s.Value.Name}, Value.Suggestions: {string.Join(", ", s.Value.Suggestions)}"));
}
}
}
// If we still haven't found a suitable StructuredLog, it's a new message. Create and store a new StructuredLog.
if (existingStructuredLog == null)
{
existingStructuredLog = new StructuredLog();
dbSession.Store(existingStructuredLog, logWithId.id);
}
}
existingStructuredLog.AddLog(logWithId.log);
// Update the expiration time for this structured log.
var meta = dbSession.Advanced.GetMetadataFor(existingStructuredLog);
var expireDateIsoString = DateTime.UtcNow.AddDays(RavenStructuredLoggerProvider.ExpirationInDays).ToString("o", System.Globalization.CultureInfo.InvariantCulture);
meta["@expires"] = expireDateIsoString;
}
try
{
dbSession.SaveChanges();
}
catch (Client.Exceptions.Documents.Session.NonUniqueObjectException) when(hasRetried == false)
{
// This exception can happen in a small race condition:
// If we check for existing ID, it's not there, then another thread saves it with that ID, then we try to save with the same ID.
// When this race condition happens, retry if we haven't already.
WriteLogBatch(logs, db, true);
}
}
}
