/// <summary>Creates syslog parser for SIEMfx. Adds specific keyword and pattern-based extractors to default parser. </summary>
/// <returns></returns>
public static SyslogParser CreateSIEMfxSyslogParser()
{
var parser = SyslogParser.CreateDefault();
parser.AddValueExtractors(new KeywordValuesExtractor(), new PatternBasedValuesExtractor());
return(parser);
}
public SyslogListener(SyslogParser parser, UdpClient udpClient, int parseProcessCount = 4)
{
_parser = parser;
_parseProcessCount = parseProcessCount;
_udpListener = new UdpListener(udpClient);
_udpListener.Error += UdpListener_Error;
_udpBuffer = new BatchingQueue <UdpPacket>();
// hook buffer to UdpListener output
_udpListener.Subscribe(_udpBuffer);
}
// constructor for reading syslog from local file
public SyslogFileListener(SyslogParser parser, FileStream fileStream, int parseProcessCount = 4)
{
_parser = parser;
_parseProcessCount = parseProcessCount;
//_priority = priority;
_logFileListener = new LogFileListener(fileStream);
_logFileListener.Error += LogfileListener_Error;
_logFileBuffer = new BatchingQueue <string>();
// hook buffer to LogfileListener output
_logFileListener.Subscribe(_logFileBuffer);
}
public SyslogListener(SyslogParser parser, IPAddress address, int port = 514,
int udpBufferSize = 500 * 1024 * 1024, int parseProcessCount = 4)
{
_parser = parser;
_parseProcessCount = parseProcessCount;
_udpListener = new UdpListener(address, port, udpBufferSize);
_udpListener.Error += UdpListener_Error;
_udpBuffer = new BatchingQueue <UdpPacket>();
// hook buffer to UdpListener output
_udpListener.Subscribe(_udpBuffer);
}
public SyslogFileReader(string fileName, IOutput output, params string[] queries)
: base(output, queries)
{
_fileName = fileName;
_eventStream = new Observable <IDictionary <string, object> >();
// Setting up syslog parser
_syslogParser = SyslogParser.CreateDefault();
_syslogParser.AddValueExtractors(new SyslogKeywordValuesExtractor(), new SyslogPatternBasedValuesExtractor());
_syslogEntryCount = 0;
}
public void TestBuildSyslogParser()
{
var sysparser = new SyslogParser();
Assert.IsNotNull(sysparser);
var anyRegex = new Regex(@"\<(?<PRIVAL>\d+?)\>\s*(?<MESSAGE>.+)");
var sysparserWithRegex = new SyslogParser(anyRegex);
Assert.IsNotNull(sysparserWithRegex);
}
public void TestSyslogCustomParserParsing()
{
var testprival = "<140>";
var severity = (Severity)Enum.ToObject(typeof(Severity), 140 & 0x7);
var facility = (Facility)Enum.ToObject(typeof(Facility), 140 >> 3);
var testmessage = "A message with any pri-val";
var testString = testprival + testmessage;
var udData = new ArraySegment <byte>(Encoding.ASCII.GetBytes(testString));
var pHeader = new IpPacketHeader(
IPAddress.Parse("127.0.0.1"),
IPAddress.Parse("127.0.0.1"),
false,
4,
0,
0,
(ushort)(udData.Array.Length + 20 + 8),
0,
0,
0,
255,
0
);
var udHeader = new UdpDatagramHeader(16, 16, (ushort)udData.Array.Length, 0);
UdpDatagram ud = new UdpDatagram()
{
UdpDatagramHeader = udHeader,
UdpData = udData,
PacketHeader = pHeader,
ReceivedTime = DateTimeOffset.UtcNow
};
var customParser = new Regex(@"(?<WithAny>with\sany)", RegexOptions.ExplicitCapture);
var sysparser = new SyslogParser(customParser);
var sys = sysparser.Parse(ud);
Assert.IsNotNull(sys);
Assert.AreEqual(testmessage, sys.Message);
Assert.AreEqual(severity, sys.LogSeverity);
Assert.AreEqual(facility, sys.LogFacility);
Assert.AreEqual("with any", sys.NamedCollectedMatches["WithAny"]);
}
public override bool Start()
{
// Setting up pipeline
if (!Start(_eventStream, "syslogserver", true))
{
return(false);
}
// Set up for listening on port
IPAddress localIp = null;
if (!string.IsNullOrEmpty(_adapterName))
{
localIp = GetLocalIp(_adapterName);
}
else
{
localIp = IPAddress.IPv6Any;
}
var endPoint = new IPEndPoint(localIp, _udpport);
var PortListener = new UdpClient(AddressFamily.InterNetworkV6);
PortListener.Client.DualMode = true;
PortListener.Client.Bind(endPoint);
PortListener.Client.ReceiveBufferSize = 10 * 1024 * 1024;
// Setting up syslog parser
var parser = SyslogParser.CreateDefault();
parser.AddValueExtractors(new SyslogKeywordValuesExtractor(), new SyslogPatternBasedValuesExtractor());
// Setting up syslog listener
var listener = new SyslogListener(parser, PortListener);
listener.Error += Listener_Error;
listener.EntryReceived += Listener_EntryReceived;
listener.Subscribe(ConvertToDictionary);
listener.Start();
return(true);
}
public void TestSyslogParser(string logLine, DateTime expectedDateTime, string hostname, string program, string message, string syslogTimestamp)
{
using (Stream logStream = Utility.StringToStream(logLine))
using (StreamReader logStreamReader = new StreamReader(logStream))
{
SyslogParser parser = new SyslogParser(NullLogger.Instance, false);
var records = parser.ParseRecords(logStreamReader, new LogContext()).ToArray();
Assert.NotNull(records);
Assert.Single(records);
var record = records[0];
// make sure the Timestamp of the envelope is in universal time
Assert.Equal(expectedDateTime.ToUniversalTime(), record.Timestamp);
// assert extracted syslog data
Assert.Equal(hostname, record.Data.Hostname);
Assert.Equal(program, record.Data.Program);
Assert.Equal(message, record.Data.Message);
Assert.Equal(syslogTimestamp, record.Data.SyslogTimestamp);
}
}
//List<Stream> oldStream;
//bool firstrun = true;
#endregion fields & pins
public void Evaluate(int spreadMax)
{
FMessage.SliceCount = spreadMax;
FFacility.SliceCount = spreadMax;
FSeverity.SliceCount = spreadMax;
FTimestamp.SliceCount = spreadMax;
FHostname.SliceCount = spreadMax;
FAppName.SliceCount = spreadMax;
FProcId.SliceCount = spreadMax;
FMsgId.SliceCount = spreadMax;
FStructuredDataName.SliceCount = spreadMax;
FStructuredDataValue.SliceCount = spreadMax;
#region test
//if (firstrun)
// oldStream = new List<Stream>();
//if (firstrun || oldStream.Count != FStreamIn.SliceCount)
//{
// oldStream.Clear();
// foreach (Stream s in FStreamIn)
// oldStream.Add(s);
//}
#endregion test
for (int i = 0; i < spreadMax; i++)
{
#region test
//if (firstrun || oldStream[i] != FStreamIn[i] ) //IsChanged doesn't work with Streams
//{
// if (firstrun == true)
// firstrun = false;
// oldStream[i] = FStreamIn[i];
#endregion test
if (FStreamIn[i].Length > 0)
{
// Stream > string
byte[] buffer = new byte[FStreamIn[i].Length];
int test = FStreamIn[i].Read(buffer, 0, (int)FStreamIn[i].Length);
string stringMessage = System.Text.Encoding.ASCII.GetString(buffer, 0, buffer.Length);
SyslogParser prs = new SyslogParser();
SyslogMessage msg = prs.Parse(stringMessage);
if (msg != null)
{
FMessage[i] = msg.MessageText;
FFacility[i] = msg.Facility;
FSeverity[i] = msg.Severity;
FTimestamp[i] = msg.TimeStamp.ToString();
FHostname[i] = msg.HostName;
FAppName[i] = msg.AppName;
FProcId[i] = msg.ProcessID;
FMsgId[i] = msg.MessageID;
int sdCount = msg.StructuredData.Count; // is normally 1 (when created with Syslog (Raw join), because it can't pack several sd elements into one sd package)
FStructuredDataName[i].SliceCount = 0;
FStructuredDataValue[i].SliceCount = 0;
for (int s = 0; s < sdCount; s++)
{
// each sd contains n sd elements
int keycount = msg.StructuredData[s].Properties.Count;
for (int k = 0; k < keycount; k++)
{
FStructuredDataName[i].Add <string>(msg.StructuredData[s].Properties.AllKeys[k]);
FStructuredDataValue[i].Add <string>(msg.StructuredData[s].Properties.GetValues(k)[0]);
}
}
}
else
{
FMessage[i] = "corrupt message";
FFacility[i] = SyslogFacility.Unknown;
FSeverity[i] = SyslogSeverity.Unknown;
FTimestamp[i] = "";
FHostname[i] = "";
FAppName[i] = "";
FProcId[i] = "";
FMsgId[i] = "";
FStructuredDataName[i].SliceCount = 0;
FStructuredDataValue[i].SliceCount = 0;
}
}
}
}
