public void Should_Parse_Logs() { try { // Instantiate the LogQuery object LogQuery oLogQuery = new LogQuery(); // Instantiate the Event Log Input Format object EventLogInputFormat oEVTInputFormat = new EventLogInputFormat(); // Set its "direction" parameter to "BW" oEVTInputFormat.direction = "BW"; // Create the query string query = @"SELECT TOP 50 SourceName, EventID, Message FROM System"; // Execute the query LogRecordSet oRecordSet = oLogQuery.Execute(query, oEVTInputFormat); // Browse the recordset for (; !oRecordSet.atEnd(); oRecordSet.moveNext()) { //Console.WriteLine(oRecordSet.getRecord().toNativeString(",")); } // Close the recordset oRecordSet.close(); } catch (System.Runtime.InteropServices.COMException exc) { Console.WriteLine("Unexpected error: " + exc.Message); } }
private void EventWatcher(object ploc) { string location = ploc.ToString(); LogQuery oLogQuery = new LogQuery(); LogManager.GetCurrentClassLogger().Info("WindowsEvent Input Listener Ready"); // Instantiate the Event Log Input Format object var iFmt = new EventLogInputFormat() { binaryFormat = _arguments.BinaryFormat.ToString(), direction = _arguments.Direction.ToString(), formatMsg = _arguments.FormatMsg, fullEventCode = _arguments.FullEventCode, fullText = _arguments.FullText, msgErrorMode = _arguments.MsgErrorMode.ToString(), stringsSep = _arguments.StringsSep, resolveSIDs = _arguments.ResolveSIDS }; oLogQuery = null; Dictionary <string, Int64> logFileMaxRecords = new Dictionary <string, Int64>(); // Execute the query while (!CancelToken.IsCancellationRequested) { try { Thread.CurrentThread.Priority = ThreadPriority.BelowNormal; oLogQuery = new LogQuery(); var qfiles = string.Format("SELECT Distinct [EventLog] FROM {0}", location); var rsfiles = oLogQuery.Execute(qfiles, iFmt); for (; !rsfiles.atEnd(); rsfiles.moveNext()) { var record = rsfiles.getRecord(); string logName = record.getValue("EventLog") as string; if (!logFileMaxRecords.ContainsKey(logName)) { var qcount = string.Format("SELECT max(RecordNumber) as MaxRecordNumber FROM {0}", logName); var rcount = oLogQuery.Execute(qcount, iFmt); var qr = rcount.getRecord(); var lrn = (Int64)qr.getValueEx("MaxRecordNumber"); logFileMaxRecords[logName] = lrn; } } foreach (string fileName in logFileMaxRecords.Keys.ToList()) { var lastRecordNumber = logFileMaxRecords[fileName]; var query = string.Format("SELECT * FROM {0} where RecordNumber > {1}", location, lastRecordNumber); var rs = oLogQuery.Execute(query, iFmt); // Browse the recordset for (; !rs.atEnd(); rs.moveNext()) { var record = rs.getRecord(); var json = new JObject(); foreach (var field in _arguments.Fields) { object v = record.getValue(field.Name); if (field.Name == "Data") { v = ToPrintable(v.ToString()); } json.Add(new JProperty(field.Name, v)); } var lrn = (Int64)record.getValueEx("RecordNumber"); logFileMaxRecords[fileName] = lrn; record = null; ProcessJson(json); _receivedMessages++; json = null; } // Close the recordset rs.close(); rs = null; GC.Collect(); } } catch (System.Threading.ThreadAbortException tex) { Thread.ResetAbort(); break; } catch (Exception ex) { LogManager.GetCurrentClassLogger().Error(ex); } try { Thread.CurrentThread.Priority = ThreadPriority.Normal; System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000); } catch (System.Threading.ThreadAbortException tex) { Thread.ResetAbort(); break; } catch (Exception ex) { LogManager.GetCurrentClassLogger().Error(ex); } } Finished(); }
private void IISW3CWatcher(string location) { LogManager.GetCurrentClassLogger().Info("IISW3Listener Ready For {0}", location); var oLogQuery = new LogQuery(); var iFmt = new W3CLogInputFormat() { codepage = _arguments.CodePage, iCodepage = _arguments.CodePage, doubleQuotedStrings = _arguments.DoubleQuotes, detectTypesLines = _arguments.DtLines, dQuotes = _arguments.DoubleQuotes, separator = _arguments.Separator }; Dictionary<string, Int64> logFileMaxRecords = new Dictionary<string, Int64>(); using (var syncHandle = new ManualResetEventSlim()) { // Execute the query while (!Stop) { // Execute the query if (!CancelToken.IsCancellationRequested) { try { oLogQuery = new LogQuery(); var qfiles = string.Format("SELECT Distinct [LogFilename] FROM {0}", location); var rsfiles = oLogQuery.Execute(qfiles, iFmt); for (; !rsfiles.atEnd(); rsfiles.moveNext()) { var record = rsfiles.getRecord(); string fileName = record.getValue("LogFilename") as string; if (!logFileMaxRecords.ContainsKey(fileName)) { var qcount = string.Format("SELECT max(RowNumber) as MaxRecordNumber FROM {0}", fileName); var rcount = oLogQuery.Execute(qcount, iFmt); var qr = rcount.getRecord(); var lrn = (Int64)qr.getValueEx("MaxRecordNumber"); logFileMaxRecords[fileName] = lrn; } } foreach (string fileName in logFileMaxRecords.Keys.ToList()) { var lastRecordNumber = logFileMaxRecords[fileName]; var query = string.Format( "SELECT * FROM '{0}' Where RowNumber > {1} order by RowNumber", fileName, lastRecordNumber); var rs = oLogQuery.Execute(query, iFmt); var colMap = new Dictionary<string, int>(); for (int col = 0; col < rs.getColumnCount(); col++) { string colName = rs.getColumnName(col); colMap[colName] = col; } // Browse the recordset for (; !rs.atEnd(); rs.moveNext()) { var record = rs.getRecord(); var json = new JObject(); foreach (var field in colMap.Keys) { object v = record.getValue(field); if (field == "date" || field == "time") { DateTime dt = DateTime.Parse(v.ToString()); json.Add(new JProperty(field, dt)); } else json.Add(new JProperty(field, v)); } ProcessJson(json); _receivedMessages++; var lrn = (Int64)record.getValueEx("RowNumber"); logFileMaxRecords[fileName] = lrn; record = null; json = null; } // Close the recordset rs.close(); } if (!Stop) syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken); } catch (OperationCanceledException) { break; } catch (Exception ex) { LogManager.GetCurrentClassLogger().Error(ex); } } } } Finished(); }
private void IISW3CWatcher(string location) { LogManager.GetCurrentClassLogger().Info("IISW3Listener Ready For {0}", location); var oLogQuery = new LogQuery(); var iFmt = new W3CLogInputFormat() { codepage = _arguments.CodePage, iCodepage = _arguments.CodePage, doubleQuotedStrings = _arguments.DoubleQuotes, detectTypesLines = _arguments.DtLines, dQuotes = _arguments.DoubleQuotes, separator = _arguments.Separator }; Dictionary <string, Int64> logFileMaxRecords = new Dictionary <string, Int64>(); using (var syncHandle = new ManualResetEventSlim()) { // Execute the query while (!Stop) { // Execute the query if (!CancelToken.IsCancellationRequested) { try { oLogQuery = new LogQuery(); var qfiles = string.Format("SELECT Distinct [LogFilename] FROM {0}", location); var rsfiles = oLogQuery.Execute(qfiles, iFmt); for (; !rsfiles.atEnd(); rsfiles.moveNext()) { var record = rsfiles.getRecord(); string fileName = record.getValue("LogFilename") as string; if (!logFileMaxRecords.ContainsKey(fileName)) { var qcount = string.Format("SELECT max(RowNumber) as MaxRecordNumber FROM {0}", fileName); var rcount = oLogQuery.Execute(qcount, iFmt); var qr = rcount.getRecord(); var lrn = (Int64)qr.getValueEx("MaxRecordNumber"); logFileMaxRecords[fileName] = lrn; } } foreach (string fileName in logFileMaxRecords.Keys.ToList()) { var lastRecordNumber = logFileMaxRecords[fileName]; var query = string.Format( "SELECT * FROM '{0}' Where RowNumber > {1} order by RowNumber", fileName, lastRecordNumber); var rs = oLogQuery.Execute(query, iFmt); var colMap = new Dictionary <string, int>(); for (int col = 0; col < rs.getColumnCount(); col++) { string colName = rs.getColumnName(col); colMap[colName] = col; } // Browse the recordset for (; !rs.atEnd(); rs.moveNext()) { var record = rs.getRecord(); var json = new JObject(); foreach (var field in colMap.Keys) { object v = record.getValue(field); if (field == "date" || field == "time") { DateTime dt = DateTime.Parse(v.ToString()); json.Add(new JProperty(field, dt)); } else { json.Add(new JProperty(field, v)); } } ProcessJson(json); _receivedMessages++; var lrn = (Int64)record.getValueEx("RowNumber"); logFileMaxRecords[fileName] = lrn; record = null; json = null; } // Close the recordset rs.close(); } if (!Stop) { syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken); } } catch (OperationCanceledException) { break; } catch (Exception ex) { LogManager.GetCurrentClassLogger().Error(ex); } } } } Finished(); }
private void FileWatcher(string fileToWatch) { var iFmt = new TextLineInputFormat() { iCodepage = _arguments.CodePage, splitLongLines = _arguments.SplitLongLines, recurse = _arguments.Recurse }; // Execute the query while (!CancelToken.IsCancellationRequested) { var oLogQuery = new LogQuery(); try { Thread.CurrentThread.Priority = ThreadPriority.BelowNormal; var qfiles = string.Format("SELECT Distinct [LogFilename] FROM {0}", fileToWatch); var rsfiles = oLogQuery.Execute(qfiles, iFmt); for (; !rsfiles.atEnd(); rsfiles.moveNext()) { var record = rsfiles.getRecord(); string logName = record.getValue("LogFilename") as string; FileInfo fi = new FileInfo(logName); if (!fi.Exists) { _logFileCreationTimes.Remove(logName); _logFileMaxRecords.Remove(logName); _logFileSizes.Remove(logName); } _logFileSampleTimes[logName] = DateTime.UtcNow; DateTime creationTime = fi.CreationTimeUtc; bool logHasRolled = (_logFileCreationTimes.ContainsKey(logName) && creationTime > _logFileCreationTimes[logName]) || (_logFileSizes.ContainsKey(logName) && fi.Length < _logFileSizes[logName]); if (!_logFileMaxRecords.ContainsKey(logName) || logHasRolled) { _logFileCreationTimes[logName] = creationTime; _logFileSizes[logName] = fi.Length; var qcount = string.Format("SELECT max(Index) as MaxRecordNumber FROM {0}", logName); var rcount = oLogQuery.Execute(qcount, iFmt); var qr = rcount.getRecord(); var lrn = (Int64)qr.getValueEx("MaxRecordNumber"); if (logHasRolled) { LogManager.GetCurrentClassLogger().Info("Log {0} has rolled", logName); lrn = 0; } _logFileMaxRecords[logName] = lrn; } _logFileSizes[logName] = fi.Length; } rsfiles.close(); foreach (string fileName in _logFileMaxRecords.Keys.ToList()) { var lastRecordNumber = _logFileMaxRecords[fileName]; var query = string.Format("SELECT * FROM {0} where Index > {1}", fileName, lastRecordNumber); var rs = oLogQuery.Execute(query, iFmt); Dictionary <string, int> colMap = new Dictionary <string, int>(); for (int col = 0; col < rs.getColumnCount(); col++) { string colName = rs.getColumnName(col); colMap[colName] = col; } // Browse the recordset for (; !rs.atEnd(); rs.moveNext()) { var record = rs.getRecord(); var json = new JObject(); foreach (var field in _arguments.Fields) { if (!colMap.ContainsKey(field.Name)) { continue; } object v = record.getValue(field.Name); if (field.DataType == typeof(DateTime)) { DateTime dt = DateTime.Parse(v.ToString()); json.Add(new JProperty(field.Name, dt)); } else { json.Add(new JProperty(field.Name, v)); } } string msg = json["Text"].ToString(); if (!string.IsNullOrEmpty(msg)) { ProcessJson(json); _receivedMessages++; } var lrn = (Int64)record.getValueEx("Index"); _logFileMaxRecords[fileName] = lrn; GC.Collect(); } colMap.Clear(); // Close the recordset rs.close(); rs = null; GC.Collect(); } } catch (Exception ex) { LogManager.GetCurrentClassLogger().Error(ex); } finally { oLogQuery = null; } Thread.CurrentThread.Priority = ThreadPriority.Normal; System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000); } Finished(); }
private void FileWatcher(string fileToWatch) { var iFmt = new TextLineInputFormat() { iCodepage = _arguments.CodePage, splitLongLines = _arguments.SplitLongLines, recurse = _arguments.Recurse }; Dictionary<string, string> _fnfmap = new Dictionary<string, string>(); using (var syncHandle = new ManualResetEventSlim()) { // Execute the query while (!Stop) { var oLogQuery = new LogQuery(); if (!CancelToken.IsCancellationRequested) { try { var qfiles = string.Format("SELECT Distinct [LogFilename] FROM {0}", fileToWatch); var rsfiles = oLogQuery.Execute(qfiles, iFmt); for (; !rsfiles.atEnd(); rsfiles.moveNext()) { var record = rsfiles.getRecord(); string logName = record.getValue("LogFilename") as string; FileInfo fi = new FileInfo(logName); var dbe = LogsFileDatabase.LookupLogFile(logName); SaveVisitedFileName(dbe.FileName); DateTime creationTime = fi.CreationTimeUtc; bool logHasRolled = dbe.NewFile || (creationTime != dbe.CreationTimeUtc || fi.Length < dbe.LastPosition); if (logHasRolled) { LogManager.GetCurrentClassLogger().Info("Log {0} has rolled", logName); LogsFileDatabase.Roll(dbe); } // Log has rolled or this is a new file, or we haven't processed yet. bool processWholeFile = logHasRolled || !dbe.ProcessedFile; if (processWholeFile) LogsFileDatabase.Update(dbe, true, 0); } rsfiles.close(); foreach (string fileName in Files.ToList()) { var dbe = LogsFileDatabase.LookupLogFile(fileName); var lastRecordNumber = dbe.LastPosition; var query = string.Format("SELECT * FROM {0} where Index > {1}", fileName, lastRecordNumber); var rs = oLogQuery.Execute(query, iFmt); Dictionary<string, int> colMap = new Dictionary<string, int>(); for (int col = 0; col < rs.getColumnCount(); col++) { string colName = rs.getColumnName(col); colMap[colName] = col; } // Browse the recordset for (; !rs.atEnd(); rs.moveNext()) { var record = rs.getRecord(); var json = new JObject(); foreach (var field in _arguments.Fields) { if (!colMap.ContainsKey(field.Name)) continue; if (json["logSource"] == null) { if (string.IsNullOrEmpty(_arguments.LogSource)) json.Add(new JProperty("logSource", fileName)); else json.Add(new JProperty("logSource", _arguments.LogSource)); } object v = record.getValue(field.Name); if (field.DataType == typeof(DateTime)) { DateTime dt = DateTime.Parse(v.ToString()); json.Add(new JProperty(field.Name, dt)); } else json.Add(new JProperty(field.Name, v)); } string msg = json["Text"].ToString(); if (!string.IsNullOrEmpty(msg)) { if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline) { _codec.Apply(msg, this); _receivedMessages++; dbe.IncrementLineCount(); } else { ProcessJson(json); dbe.IncrementLineCount(); _receivedMessages++; } } var lrn = (Int64)record.getValueEx("Index"); LogsFileDatabase.Update(dbe, true, lrn); GC.Collect(); } colMap.Clear(); // Close the recordset rs.close(); rs = null; GC.Collect(); } } catch (FileNotFoundException fnfex) { string fn = fnfex.FileName; if (!string.IsNullOrEmpty(fn) && !_fnfmap.ContainsKey(fn)) { LogManager.GetCurrentClassLogger().Warn(fnfex.Message); _fnfmap[fn] = fn; } } catch (OperationCanceledException) { break; } catch (Exception ex) { LogManager.GetCurrentClassLogger().Error(ex); } finally { try { oLogQuery = null; // Sleep if (!Stop) syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken); } catch (OperationCanceledException) { } catch (Exception ex1) { LogManager.GetCurrentClassLogger().Warn(ex1); } } } } Finished(); } }
private void IISW3CWatcher(string location) { LogManager.GetCurrentClassLogger().Info("IISW3Listener Ready For {0}", location); var oLogQuery = new LogQuery(); var iFmt = new IISW3CLogInputFormat() { codepage = _arguments.CodePage, consolidateLogs = true, dirTime = _arguments.DirTime, dQuotes = _arguments.DoubleQuotes, recurse = _arguments.Recurse, useDoubleQuotes = _arguments.DoubleQuotes }; if (_arguments.MinDateMod.HasValue) { iFmt.minDateMod = _arguments.MinDateMod.Value.ToString("yyyy-MM-dd hh:mm:ss"); } Dictionary <string, Int64> logFileMaxRecords = new Dictionary <string, Int64>(); // Execute the query while (!CancelToken.IsCancellationRequested) { try { oLogQuery = new LogQuery(); var qfiles = string.Format("SELECT Distinct [LogFilename] FROM {0}", location); var rsfiles = oLogQuery.Execute(qfiles, iFmt); for (; !rsfiles.atEnd(); rsfiles.moveNext()) { var record = rsfiles.getRecord(); string fileName = record.getValue("LogFilename") as string; if (!logFileMaxRecords.ContainsKey(fileName)) { var qcount = string.Format("SELECT max(LogRow) as MaxRecordNumber FROM {0}", fileName); var rcount = oLogQuery.Execute(qcount, iFmt); var qr = rcount.getRecord(); var lrn = (Int64)qr.getValueEx("MaxRecordNumber"); logFileMaxRecords[fileName] = lrn; } } foreach (string fileName in logFileMaxRecords.Keys.ToList()) { var lastRecordNumber = logFileMaxRecords[fileName]; var query = string.Format("SELECT * FROM '{0}' Where LogRow > {1}", fileName, lastRecordNumber); var rs = oLogQuery.Execute(query, iFmt); var colMap = new Dictionary <string, int>(); for (int col = 0; col < rs.getColumnCount(); col++) { string colName = rs.getColumnName(col); colMap[colName] = col; } // Browse the recordset for (; !rs.atEnd(); rs.moveNext()) { var record = rs.getRecord(); var json = new JObject(); foreach (var field in _arguments.Fields) { if (!colMap.ContainsKey(field.Name)) { continue; } object v = record.getValue(field.Name); if (field.DataType == typeof(DateTime)) { DateTime dt = DateTime.Parse(v.ToString()); json.Add(new JProperty(field.Name, dt)); } else { json.Add(new JProperty(field.Name, v)); } } ProcessJson(json); _receivedMessages++; var lrn = (Int64)record.getValueEx("LogRow"); logFileMaxRecords[fileName] = lrn; record = null; json = null; } // Close the recordset rs.close(); GC.Collect(); } } catch (Exception ex) { LogManager.GetCurrentClassLogger().Error(ex); } System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000); } Finished(); }
private void FileWatcher(string fileToWatch) { var iFmt = new TextLineInputFormat() { iCodepage = _arguments.CodePage, splitLongLines = _arguments.SplitLongLines, recurse = _arguments.Recurse }; Dictionary <string, string> _fnfmap = new Dictionary <string, string>(); using (var syncHandle = new ManualResetEventSlim()) { // Execute the query while (!Stop) { var oLogQuery = new LogQuery(); if (!CancelToken.IsCancellationRequested) { try { var qfiles = string.Format("SELECT Distinct [LogFilename] FROM {0}", fileToWatch); var rsfiles = oLogQuery.Execute(qfiles, iFmt); for (; !rsfiles.atEnd(); rsfiles.moveNext()) { var record = rsfiles.getRecord(); string logName = record.getValue("LogFilename") as string; FileInfo fi = new FileInfo(logName); var dbe = LogsFileDatabase.LookupLogFile(logName); SaveVisitedFileName(dbe.FileName); DateTime creationTime = fi.CreationTimeUtc; bool logHasRolled = dbe.NewFile || (creationTime != dbe.CreationTimeUtc || fi.Length < dbe.LastPosition); if (logHasRolled) { LogManager.GetCurrentClassLogger().Info("Log {0} has rolled", logName); LogsFileDatabase.Roll(dbe); } // Log has rolled or this is a new file, or we haven't processed yet. bool processWholeFile = logHasRolled || !dbe.ProcessedFile; if (processWholeFile) { LogsFileDatabase.Update(dbe, true, 0); } } rsfiles.close(); foreach (string fileName in Files.ToList()) { var dbe = LogsFileDatabase.LookupLogFile(fileName); var lastRecordNumber = dbe.LastPosition; var query = string.Format("SELECT * FROM {0} where Index > {1}", fileName, lastRecordNumber); var rs = oLogQuery.Execute(query, iFmt); Dictionary <string, int> colMap = new Dictionary <string, int>(); for (int col = 0; col < rs.getColumnCount(); col++) { string colName = rs.getColumnName(col); colMap[colName] = col; } // Browse the recordset for (; !rs.atEnd(); rs.moveNext()) { var record = rs.getRecord(); var json = new JObject(); foreach (var field in _arguments.Fields) { if (!colMap.ContainsKey(field.Name)) { continue; } if (json["logSource"] == null) { if (string.IsNullOrEmpty(_arguments.LogSource)) { json.Add(new JProperty("logSource", fileName)); } else { json.Add(new JProperty("logSource", _arguments.LogSource)); } } object v = record.getValue(field.Name); if (field.DataType == typeof(DateTime)) { DateTime dt = DateTime.Parse(v.ToString()); json.Add(new JProperty(field.Name, dt)); } else { json.Add(new JProperty(field.Name, v)); } } string msg = json["Text"].ToString(); if (!string.IsNullOrEmpty(msg)) { if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline) { _codec.Apply(msg, this); _receivedMessages++; dbe.IncrementLineCount(); } else { ProcessJson(json); dbe.IncrementLineCount(); _receivedMessages++; } } var lrn = (Int64)record.getValueEx("Index"); LogsFileDatabase.Update(dbe, true, lrn); GC.Collect(); } colMap.Clear(); // Close the recordset rs.close(); rs = null; GC.Collect(); } } catch (FileNotFoundException fnfex) { string fn = fnfex.FileName; if (!string.IsNullOrEmpty(fn) && !_fnfmap.ContainsKey(fn)) { LogManager.GetCurrentClassLogger().Warn(fnfex.Message); _fnfmap[fn] = fn; } } catch (OperationCanceledException) { break; } catch (Exception ex) { LogManager.GetCurrentClassLogger().Error(ex); } finally { try { oLogQuery = null; // Sleep if (!Stop) { syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken); } } catch (OperationCanceledException) { } catch (Exception ex1) { LogManager.GetCurrentClassLogger().Warn(ex1); } } } } Finished(); } }