static LogEvent[] ReadBatch(LogEventReader reader, Func <LogEvent, bool> filter,
int count, InvalidDataHandling invalidDataHandling)
{
var batch = new List <LogEvent>();
do
{
try
{
while (batch.Count < count && reader.TryRead(out var evt))
{
if (filter == null || filter(evt))
{
batch.Add(evt);
}
}
}
catch (Exception ex)
{
if (ex is JsonReaderException || ex is InvalidDataException)
{
if (invalidDataHandling == InvalidDataHandling.Ignore)
{
continue;
}
}
throw;
}
return(batch.ToArray());
} while (true);
}
public IActionResult LogEvent([FromQuery] bool clef)
{
var isValidMediaType = clef || this.Request.ContentType?.StartsWith(ClefMediaType, StringComparison.OrdinalIgnoreCase) == true;
if (!isValidMediaType)
{
return(this.StatusCode((int)HttpStatusCode.UnsupportedMediaType));
}
using (var streamReader = new StreamReader(this.Request.Body))
using (var logEventReader = new LogEventReader(streamReader))
{
try
{
while (logEventReader.TryRead(out var logEvent))
{
Log.Write(logEvent);
}
}
catch (Exception ex)
{
return(this.BadRequest(ex.Message));
}
}
// LINK (Cameron): https://github.com/serilog/serilog-sinks-seq/blob/9fe533663448c5ca799c4bb3d3e57c9ca6ed302b/src/Serilog.Sinks.Seq/Sinks/Seq/SeqApi.cs#L37
return(new ContentResult
{
Content = @"{""MinimumLevelAccepted"":null}",
ContentType = "application/json",
StatusCode = (int)HttpStatusCode.Created,
});
}
private RockLogEvent GetLogEventFromLogLine(string logLine)
{
try
{
var evt = LogEventReader.ReadFromString(logLine, _jsonSerializer);
var domain = evt.Properties["domain"].ToString();
evt.RemovePropertyIfPresent("domain");
var message = evt.RenderMessage().Replace("{domain}", "").Trim();
domain = domain.Replace("\"", "");
return(new RockLogEvent
{
DateTime = evt.Timestamp.DateTime.ToLocalTime(),
Exception = evt.Exception,
Level = GetRockLogLevelFromSerilogLevel(evt.Level),
Domain = domain,
Message = message
});
}
catch (Exception ex)
{
// In rare instances it is possible that the event didn't get completely flushed to the log
// and when that happens it won't be able to be correctly parsed so we need to handle that gracefully.
ExceptionLogService.LogException(ex);
return(null);
}
}
public void MissingMessagesAreAcceptedAsEmpty()
{
const string document = "{\"@t\":\"2016-10-12T04:20:58.0554314Z\"}";
var evt = LogEventReader.ReadFromString(document);
Assert.Empty(evt.MessageTemplate.Tokens);
}
public static LogEvent ReadFromJObject(JObject jObject)
{
if (!jObject.TryGetValue("@t", out _))
{
jObject.Add("@t", new JValue(DateTime.UtcNow.ToString("O")));
}
if (jObject.TryGetValue("@l", out var levelToken))
{
jObject.Remove("@l");
var serilogLevel = LevelMapping.ToSerilogLevel(levelToken.Value <string>());
if (serilogLevel != LogEventLevel.Information)
{
jObject.Add("@l", new JValue(serilogLevel.ToString()));
}
jObject.Add(SurrogateLevelProperty.PropertyName, levelToken);
}
else
{
jObject.Add(SurrogateLevelProperty.PropertyName, new JValue("Information"));
}
return(LogEventReader.ReadFromJObject(jObject));
}
public static void PipeEvents(LogEventReader source, Logger destination, InvalidDataHandling invalidDataHandling)
{
do
{
try
{
while (source.TryRead(out var evt))
{
destination.Write(evt);
}
return;
}
catch (Exception ex)
{
if (ex is JsonReaderException || ex is InvalidDataException)
{
if (invalidDataHandling == InvalidDataHandling.Ignore)
{
continue;
}
if (invalidDataHandling == InvalidDataHandling.Report)
{
destination.Error(ex, "An event was not in CLEF format.");
continue;
}
}
throw;
}
} while (true);
}
public void MessagesAreEscapedIntoTemplates()
{
const string document = "{\"@t\":\"2016-10-12T04:20:58.0554314Z\",\"@m\":\"Hello, {text}\"}";
var evt = LogEventReader.ReadFromString(document);
Assert.Equal("Hello, {{text}}", evt.MessageTemplate.Text);
}
/// <summary>Creates an new instance by loading the specified reader.</summary>
/// <param name="reader">The reader to load the instance from.</param>
/// <returns>A new instance deserialize from the specifed reader.</returns>
public static LogEvent Load(XmlReader reader)
{
var logReader = new LogEventReader();
var instance = logReader.Read(reader);
return(instance);
}
public LogRecord(MainWindowViewModel outer, string rowText, int lineNumber)
{
outer.WeakPropertyChanged += (sender, args) =>
{
switch (args.PropertyName)
{
case nameof(MainWindowViewModel.Render):
RaisePropertiesChanged(nameof(DisplayText));
break;
case nameof(MainWindowViewModel.ShowUTC):
RaisePropertiesChanged(nameof(Timestamp));
break;
}
};
_outer = outer;
LineNumber = lineNumber;
RowText = rowText.Trim();
_logEvent = new Lazy <LogEvent>(() =>
{
try
{
return(LogEventReader.ReadFromJObject(JObject.Parse(RowText)));
}
catch
{
return(new LogEvent(DateTimeOffset.MinValue, LogEventLevel.Verbose, new Exception(), MessageTemplate.Empty, Enumerable.Empty <LogEventProperty>()));
}
});
}
public List <LogEvent> ReadLogs(string filePath, Logger logger = null)
{
var logItems = new List <LogEvent>();
using (var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
using (var stream = new StreamReader(fs))
{
var reader = new LogEventReader(stream);
while (TryRead(reader, out var evt))
{
if (evt == null)
{
continue;
}
if (logger != null)
{
//We can persist the log item (using the passed in Serilog config)
//In this case a Logger with File Sink setup
logger.Write(evt);
}
logItems.Add(evt);
}
}
}
_logItems = logItems;
LogFilePath = filePath;
LogIsOpen = true;
return(_logItems);
}
public void ReadPropertiesWhiteSpace()
{
string xml = @"<LogEvent>
<Properties>
<Property>
<Name>Test</Name>
<Value>ErrorWrite</Value>
</Property>
</Properties>
</LogEvent>";
var sr = new StringReader(xml);
var reader = XmlReader.Create(sr);
var logReader = new LogEventReader();
var log = logReader.Read(reader);
log.Should().NotBeNull();
log.Properties.Should().NotBeNull();
log.Properties.Should().NotBeEmpty();
log.Properties.Count.Should().Be(1);
log.Properties[0].Name.Should().Be("Test");
log.Properties[0].Value.Should().Be("ErrorWrite");
}
protected override IReadOnlyList <LogEvent> GetLogs(LogTimePeriod logTimePeriod, ILogFilter filter, int skip,
int take)
{
var logs = new List <LogEvent>();
var count = 0;
// foreach full day in the range - see if we can find one or more filenames that end with
// yyyyMMdd.json - Ends with due to MachineName in filenames - could be 1 or more due to load balancing
for (DateTime day = logTimePeriod.StartTime.Date; day.Date <= logTimePeriod.EndTime.Date; day = day.AddDays(1))
{
// Filename ending to search for (As could be multiple)
var filesToFind = GetSearchPattern(day);
var filesForCurrentDay = Directory.GetFiles(_logsPath, filesToFind);
// Foreach file we find - open it
foreach (var filePath in filesForCurrentDay)
{
// Open log file & add contents to the log collection
// Which we then use LINQ to page over
using (var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
using (var stream = new StreamReader(fs))
{
var reader = new LogEventReader(stream);
while (TryRead(reader, out LogEvent? evt))
{
// We may get a null if log line is malformed
if (evt == null)
{
continue;
}
if (count > skip + take)
{
break;
}
if (count < skip)
{
count++;
continue;
}
if (filter.TakeLogEvent(evt))
{
logs.Add(evt);
}
count++;
}
}
}
}
}
return(logs);
}
public void HandlesDefaultJsonNetSerialization()
{
const string document = "{\"@t\":\"2016-10-12T04:20:58.0554314Z\",\"@m\":\"Hello\"}";
var jObject = JsonConvert.DeserializeObject <JObject>(document);
var evt = LogEventReader.ReadFromJObject(jObject);
Assert.Equal(DateTimeOffset.Parse("2016-10-12T04:20:58.0554314Z"), evt.Timestamp);
}
private AnalogyLogMessage ParserJObject(JObject jo, ILogger analogy)
{
var evt = LogEventReader.ReadFromJObject(jo);
{
analogy.Write(evt);
return(CommonParser.ParseLogEventProperties(evt));
}
}
private void HandleLogEvent(string serializedEvent)
{
var eventReader = new LogEventReader(new StringReader(serializedEvent));
while (eventReader.TryRead(out var logEvent))
{
LogEventArrived?.Invoke(this, new LogEventArrivedEventArgs(logEvent));
}
}
public void RoundTripsTypeTags()
{
const string document = "{\"@t\":\"2016-10-12T04:20:58.0554314Z\",\"@m\":\"Hello\",\"User\":{\"$type\":\"TestUser\",\"Name\":\"nblumhardt\"}}";
var evt = LogEventReader.ReadFromString(document);
var user = (StructureValue)evt.Properties["User"];
Assert.Equal("TestUser", user.TypeTag);
}
public async Task Invoke(HttpContext context, Func <Task> next)
{
if (context.Request.Path != _endpointPath)
{
await next();
return;
}
var reader = new StreamReader(context.Request.Body);
var line = await reader.ReadLineAsync();
while (line != null)
{
if (!string.IsNullOrWhiteSpace(line))
{
if (line.Length > _eventBodyLimitBytes)
{
var startToLog = (int)Math.Min(_eventBodyLimitBytes ?? 1024, 1024);
var prefix = line.Substring(0, startToLog);
SelfLog.WriteLine("Dropping oversize event from {0} of {1} chars: {2}", context.Connection.RemoteIpAddress, line.Length, prefix);
line = reader.ReadLine();
continue;
}
try
{
var jObject = JsonConvert.DeserializeObject <JObject>(line);
var evt = LogEventReader.ReadFromJObject(jObject);
if (_clientLevelSwitch == null || evt.Level >= _clientLevelSwitch.MinimumLevel)
{
if (_originPropertyName != null)
{
evt.RemovePropertyIfPresent(_originPropertyName); // Ensure the client can't override this
}
_log.Write(evt);
}
}
catch (Exception ex)
{
SelfLog.WriteLine("Failed to deserialize event from {0}: {1}", context.Connection.RemoteIpAddress, ex);
}
}
line = await reader.ReadLineAsync();
}
context.Response.StatusCode = 201;
if (_clientLevelSwitch != null)
{
await context.Response.WriteAsync("{\"MinimumLevelAccepted\":\"" + _clientLevelSwitch.MinimumLevel + "\"}");
}
}
public void PassesThroughUnrecognizedReifiedProperties()
{
const string document = "{\"@t\":\"2016-10-12T04:20:58.0554314Z\",\"@m\":\"Hello\",\"@foo\":42}";
var evt = LogEventReader.ReadFromString(document);
var foo = evt.Properties["@foo"];
Assert.Equal(42L, ((ScalarValue)foo).Value);
// Ensure we don't just forward everything
Assert.False(evt.Properties.ContainsKey("@m"));
}
public void ReadErrorNull()
{
string xml = "<LogEvent><Error /></LogEvent>";
var sr = new StringReader(xml);
var reader = XmlReader.Create(sr);
var logReader = new LogEventReader();
var log = logReader.Read(reader);
log.Should().NotBeNull();
}
static LogEvent[] PipeEvents(string input, InvalidDataHandling invalidDataHandling)
{
var output = new CollectingSink();
using (var source = new LogEventReader(new StringReader(input)))
using (var destination = new LoggerConfiguration()
.MinimumLevel.Is(LevelAlias.Minimum)
.WriteTo.Sink(output)
.CreateLogger())
{
EventPipe.PipeEvents(source, destination, invalidDataHandling);
}
return(output.Events);
}
private bool TryRead(LogEventReader reader, out LogEvent evt)
{
try
{
return(reader.TryRead(out evt));
}
catch (JsonReaderException ex)
{
// As we are reading/streaming one line at a time in the JSON file
// Thus we can not report the line number, as it will always be 1
_logger.Error <JsonLogViewer>(ex, "Unable to parse a line in the JSON log file");
evt = null;
return(true);
}
}
public void BasicFieldsAreRead()
{
const string document = "{\"@t\":\"2016-10-12T04:20:58.0554314Z\",\"@mt\":\"Hello, {@User}\",\"User\":{\"Name\":\"nblumhardt\",\"Id\":101}}";
var evt = LogEventReader.ReadFromString(document);
Assert.Equal(DateTimeOffset.Parse("2016-10-12T04:20:58.0554314Z"), evt.Timestamp);
Assert.Equal(LogEventLevel.Information, evt.Level);
Assert.Equal("Hello, {@User}", evt.MessageTemplate.Text);
var user = (StructureValue)evt.Properties["User"];
Assert.Equal("Name", user.Properties[0].Name);
Assert.Equal(new ScalarValue("nblumhardt"), (ScalarValue)user.Properties[0].Value);
Assert.Equal("Id", user.Properties[1].Name);
Assert.Equal(101L, ((ScalarValue)user.Properties[1].Value).Value);
}
public void AllEventsAreRead()
{
var all = new List <LogEvent>();
using (var clef = File.OpenText("LogEventReaderTests.clef"))
{
var reader = new LogEventReader(clef);
LogEvent evt;
while (reader.TryRead(out evt))
{
all.Add(evt);
}
}
Assert.Equal(5, all.Count);
}
public IActionResult GetSystemLog()
{
List <LogEvent> logs = new List <LogEvent>();
// configure strongly typed settings objects
var logPath = Configuration.GetSection("AppSettings").Get <AppSettings>().LogPath;
DirectoryInfo di = new DirectoryInfo(logPath);
di = di.Parent;
DateTime now = DateTime.Now;
var patternYesterday = $"log-{now.AddDays(-1).ToString("yyyyMMdd")}*.log";
var patternToday = $"log-{now.ToString("yyyyMMdd")}*.log";
//get log files and sort
var yesterdayLogs = di.GetFiles(patternYesterday).ToList();
var todayLogs = di.GetFiles(patternToday).ToList();
yesterdayLogs = yesterdayLogs.OrderBy(o => o.CreationTime).ToList();
todayLogs = todayLogs.OrderBy(o => o.CreationTime).ToList();
List <FileInfo> logfiles = new List <FileInfo>();
logfiles.AddRange(yesterdayLogs);
logfiles.AddRange(todayLogs);
string result = string.Empty;
foreach (var fi in logfiles)
{
using (var clef = System.IO.File.Open(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
TextReader tr = new StreamReader(clef);
var reader = new LogEventReader(tr);
LogEvent evt;
while (reader.TryRead(out evt))
{
logs.Add(evt);
}
}
}
logs.Reverse();
return(Ok(new { Logs = logs }));
}
public void ReaderNoWhiteSpace()
{
string xml = @"<LogEvent><SequenceID>1</SequenceID><TimeStamp>2012-09-24T09:09:59.5433403-05:00</TimeStamp><Level>Error</Level><LoggerName>LogEventWriterTest</LoggerName><Message>Error reading file 'blah.txt'.</Message></LogEvent>";
var sr = new StringReader(xml);
var reader = XmlReader.Create(sr);
var logReader = new LogEventReader();
var log = logReader.Read(reader);
log.Should().NotBeNull();
log.SequenceID.Should().Be(1);
log.TimeStamp.Should().Be(DateTime.Parse("2012-09-24T09:09:59.5433403-05:00"));
log.Level.Should().Be("Error");
log.LoggerName.Should().Be("LogEventWriterTest");
log.Message.Should().Be("Error reading file 'blah.txt'.");
}
public async Task <IEnumerable <AnalogyLogMessage> > Process(string fileName, CancellationToken token, ILogMessageCreatedHandler messagesHandler)
{
var messages = await Task <IEnumerable <AnalogyLogMessage> > .Factory.StartNew(() =>
{
List <AnalogyLogMessage> parsedMessages = new List <AnalogyLogMessage>();
try
{
using (var analogy = new LoggerConfiguration()
.WriteTo.Analogy()
.CreateLogger())
{
using (var fileStream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (var clef = new StreamReader(fileStream, encoding: Encoding.UTF8))
{
var reader = new LogEventReader(clef);
while (reader.TryRead(out var evt))
{
analogy.Write(evt);
AnalogyLogMessage m = CommonParser.ParseLogEventProperties(evt);
if (TimeStampTimeZone != null)
{
m.Date = TimeZoneInfo.ConvertTimeFromUtc(m.Date, TimeStampTimeZone);
}
parsedMessages.Add(m);
}
messagesHandler.AppendMessages(parsedMessages, fileName);
return(parsedMessages);
}
}
}
catch (Exception e)
{
AnalogyLogMessage empty = new AnalogyLogMessage($"Error reading file {fileName}: Error: {e.Message}",
AnalogyLogLevel.Error, AnalogyLogClass.General, "Analogy", "None");
empty.Source = nameof(ClefParser);
empty.Module = "Analogy.LogViewer.Serilog";
parsedMessages.Add(empty);
messagesHandler.AppendMessages(parsedMessages, fileName);
return(parsedMessages);
}
});
return(messages);
}
private async void ReadEventsLoop()
{
var waitStream = new AsyncToSyncProgressStream(logConnection, cancellationTokenSource.Token);
var reader = new LogEventReader(new StreamReader(waitStream));
while (await waitStream.WaitForData())
{
try
{
if (reader.TryRead(out var logEvent))
{
HandleEvent(logEvent);
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
}
public UIElement GetView(HttpResponseMessage responseMessage, CommandResultDto dto, IServiceProvider serviceProvider)
{
if (responseMessage.Content?.Headers.ContentType.MediaType != "maze/jsonlog")
{
return(null);
}
var log = responseMessage.Content.ReadAsStringAsync().Result;
var logEvents = new List <LogEvent>();
using (var stringReader = new StringReader(log))
{
var logReader = new LogEventReader(stringReader);
while (logReader.TryRead(out var logEvent))
{
logEvents.Add(logEvent);
}
}
return(new LogView(logEvents));
}
private RockLogEvent GetLogEventFromLogLine(string logLine)
{
var evt = LogEventReader.ReadFromString(logLine, jsonSerializer);
var domain = evt.Properties["domain"].ToString();
evt.RemovePropertyIfPresent("domain");
var message = evt.RenderMessage().Replace("{domain}", "").Trim();
domain = domain.Replace("\"", "");
return(new RockLogEvent
{
DateTime = evt.Timestamp.DateTime.ToLocalTime(),
Exception = evt.Exception,
Level = GetRockLogLevelFromSerilogLevel(evt.Level),
Domain = domain,
Message = message
});
}
public async Task <ReadResult> TryReadAsync()
{
var frame = await _reader.TryReadAsync();
if (!frame.HasValue)
{
return(new ReadResult(null, frame.IsAtEnd));
}
if (frame.IsOrphan)
{
throw new InvalidDataException($"A line arrived late or could not be parsed: `{frame.Value.Trim()}`.");
}
var frameValue = new JsonTextReader(new StringReader(frame.Value));
if (!(_serializer.Deserialize <JToken>(frameValue) is JObject jobject))
{
throw new InvalidDataException($"The line is not a JSON object: `{frame.Value.Trim()}`.");
}
if (!jobject.TryGetValue("@t", out _))
{
jobject.Add("@t", new JValue(DateTime.UtcNow.ToString("O")));
}
if (jobject.TryGetValue("@l", out var levelToken))
{
jobject.Remove("@l");
jobject.Add(SurrogateLevelProperty.PropertyName, levelToken);
}
else
{
jobject.Add(SurrogateLevelProperty.PropertyName, new JValue("Information"));
}
var evt = LogEventReader.ReadFromJObject(jobject);
return(new ReadResult(evt, frame.IsAtEnd));
}
