private void Parse(string entry)
{
// giving CPU a break every so often
if (_syslogEntryCount % 10 == 0)
{
_syslogEntryCount = 0;
Thread.Yield();
}
// Transform to RFC
var text = $"<{_defaultPriority}> {entry}";
// Begin parsing using Microsoft.Syslog
var ctx = new ParserContext(text);
if (_syslogParser.TryParse(ctx))
{
var serverEntry = new ServerSyslogEntry()
{
Payload = entry,
Entry = ctx.Entry,
ParseErrorMessages = ctx.ErrorMessages
};
var dict = SyslogEntryToDictionaryConverter.Convert(serverEntry);
_eventStream.Broadcast(dict);
}
}
private static void FileListener_EntryReceived(object sender, ServerSyslogEntry e)
{
var parseErrors = e.ParseErrorMessages;
if (parseErrors != null && parseErrors.Count > 0)
{
var strErrors = "Parser errors encountered " + string.Join(Environment.NewLine, parseErrors);
Console.WriteLine(strErrors);
}
}
public bool Allow(ServerSyslogEntry serverEntry)
{
if (Keywords == null || Keywords.Length == 0)
{
return(true);
}
var text = serverEntry.Payload;
for (int i = 0; i < Keywords.Length; i++)
{
if (text.Contains(Keywords[i]))
{
return(false);
}
}
return(true);
}
private void ConvertToDictionary(ServerSyslogEntry serverEntry)
{
_eventStream.Broadcast(SyslogEntryToDictionaryConverter.Convert(serverEntry));
}
public static KustoSyslogRecord Convert(ServerSyslogEntry serverEntry)
{
// entry.StructuredData is RFC-5424 structured data;
// ExtractedData is key-values extracted from syslog body for other formats - we add SturcturedData values there as well
var allParams = new List <NameValuePair>();
allParams.AddRange(serverEntry.Entry.ExtractedData);
// This is RFC-5424 specific data, structured in its own way - we take all parameters (kv pairs) from there
var structData = serverEntry.Entry.StructuredData;
if (structData != null)
{
var structDataParams = structData.SelectMany(de => de.Value).ToList();
allParams.AddRange(structDataParams);
}
// now convert to dictionary; first group param values by param name
var grouped = allParams.GroupBy(p => p.Name, p => p.Value).ToDictionary(g => g.Key, g => g.ToArray());
// copy to final dictionary - keep multiple values as an array;
// for single value put it as a single object (not object[1]); - except for IP addresses - these are always arrays, even if there's only one
var allDataDict = new Dictionary <string, object>();
foreach (var kv in grouped)
{
if (kv.Value.Length > 1 || kv.Key == "IPv4" || kv.Key == "IPv6")
{
allDataDict[kv.Key] = kv.Value; //array
}
else
{
allDataDict[kv.Key] = kv.Value[0]; //single value
}
}
var logFileLineage = new Dictionary <string, object>
{
["PayloadType"] = serverEntry.Entry.PayloadType.ToString(),
["RelayServer"] = Environment.MachineName,
["ReceivedTimestamp"] = serverEntry.UdpPacket?.ReceivedOn,
};
// add parser errors if any
var parserErrors = serverEntry.ParseErrorMessages;
if (parserErrors != null && parserErrors.Count > 0)
{
logFileLineage["SyslogParserErrors"] = string.Join("; ", parserErrors);
}
var hdr = serverEntry.Entry.Header;
var rec = new KustoSyslogRecord()
{
DeviceTimestamp = hdr.Timestamp,
Facility = serverEntry.Entry.Facility.ToString(),
Severity = serverEntry.Entry.Severity.ToString(),
HostName = hdr.HostName,
AppName = hdr.AppName,
ProcId = hdr.ProcId,
MsgId = hdr.MsgId,
ExtractedData = allDataDict,
Payload = serverEntry.Payload,
LogFileLineage = logFileLineage,
SourceIpAddress = serverEntry.UdpPacket?.SourceIpAddress?.ToString(),
};
return(rec);
}
public void OnNext(ServerSyslogEntry serverEntry)
{
var rec = Convert(serverEntry);
Broadcast(rec.AsDictionary());
}
internal SyslogEntryEventArgs(ServerSyslogEntry entry)
{
ServerEntry = entry;
}
