// NOTE: this method is called on a background thread. Don't touch class members. private static void WriteLogBatch(IList <Log> logs, IDocumentStore db) { // .db is a class member, but it's OK to access because it is a thread-safe class. using (var dbSession = db.OpenSession()) { var structuredLogIds = logs .Select(l => (id: GetStructuredLogId(l), log: l)) .ToList(); var structuredLogs = dbSession.Load <StructuredLog>(structuredLogIds.Select(l => l.id)); foreach (var structuredLogInfo in structuredLogIds) { var existingStructuredLog = structuredLogs[structuredLogInfo.id]; if (existingStructuredLog == null) { existingStructuredLog = new StructuredLog(); dbSession.Store(existingStructuredLog, structuredLogInfo.id); } existingStructuredLog.AddLog(structuredLogInfo.log); // Update the expiration time for this structured log. var meta = dbSession.Advanced.GetMetadataFor(existingStructuredLog); var expireDateIsoString = DateTime.UtcNow.AddDays(RavenStructuredLoggerProvider.ExpirationInDays).ToString("o", System.Globalization.CultureInfo.InvariantCulture); meta["@expires"] = expireDateIsoString; } dbSession.SaveChanges(); } }
// NOTE: this method is called on a background thread private static void WriteLogBatch(IList <Log> logs, IDocumentStore db, bool hasRetried = false) { using (var dbSession = db.OpenSession()) { var structuredLogIds = logs .Select(l => (id: GetStructuredLogId(l), log: l)) .ToList(); var structuredLogs = dbSession.Load <StructuredLog>(structuredLogIds.Select(l => l.id).Distinct()); foreach (var logWithId in structuredLogIds) { var existingStructuredLog = structuredLogs[logWithId.id]; if (existingStructuredLog == null) { // We don't have a StructuredLog with this exact message. // Do a fuzzy search using Raven Suggestions to see if there's log with a similar message. // This is needed because some software will include things like timestamps in the error // message, making the mesage vary slightly. // If there's a similar message that varies slightly, use that as the group. if (!string.IsNullOrWhiteSpace(logWithId.log.Template) || !string.IsNullOrWhiteSpace(logWithId.log.Message)) { // For simple logs, Template may be null, and we may only have message. var messageToSearch = string.IsNullOrWhiteSpace(logWithId.log.Template) ? logWithId.log.Message : logWithId.log.Template; var suggestions = dbSession.Query <StructuredLog>() .SuggestUsing(builder => builder .ByField(l => l.MessageTemplate, messageToSearch) .WithOptions(new Client.Documents.Queries.Suggestions.SuggestionOptions { Accuracy = 0.8f // 0.9 is too strict, won't match things that should be grouped. 0.8 seems about right. })) .Execute(); var firstSuggestion = suggestions.FirstOrDefault().Value?.Suggestions?.LastOrDefault(); if (firstSuggestion != null) { existingStructuredLog = dbSession.Query <StructuredLog>() .Search(l => l.MessageTemplate, firstSuggestion) .FirstOrDefault(); if (existingStructuredLog != null) { logWithId.log.GroupingDetails = "Couldn't find log with exact message match, so queried for suggestions. Using last suggestion \"" + firstSuggestion + "\". Full suggestions " + string.Join("; ", suggestions.Select(s => $"Key: {s.Key}, Value.Name: {s.Value.Name}, Value.Suggestions: {string.Join(", ", s.Value.Suggestions)}")); } } } // If we still haven't found a suitable StructuredLog, it's a new message. Create and store a new StructuredLog. if (existingStructuredLog == null) { existingStructuredLog = new StructuredLog(); dbSession.Store(existingStructuredLog, logWithId.id); } } existingStructuredLog.AddLog(logWithId.log); // Update the expiration time for this structured log. var meta = dbSession.Advanced.GetMetadataFor(existingStructuredLog); var expireDateIsoString = DateTime.UtcNow.AddDays(RavenStructuredLoggerProvider.ExpirationInDays).ToString("o", System.Globalization.CultureInfo.InvariantCulture); meta["@expires"] = expireDateIsoString; } try { dbSession.SaveChanges(); } catch (Client.Exceptions.Documents.Session.NonUniqueObjectException) when(hasRetried == false) { // This exception can happen in a small race condition: // If we check for existing ID, it's not there, then another thread saves it with that ID, then we try to save with the same ID. // When this race condition happens, retry if we haven't already. WriteLogBatch(logs, db, true); } } }