public void Should_Parse_Logs()
        {
            try
            {
                // Instantiate the LogQuery object
                LogQuery oLogQuery = new LogQuery();

                // Instantiate the Event Log Input Format object
                EventLogInputFormat oEVTInputFormat = new EventLogInputFormat();

                // Set its "direction" parameter to "BW"
                oEVTInputFormat.direction = "BW";

                // Create the query
                string query = @"SELECT TOP 50 SourceName, EventID, Message FROM System";

                // Execute the query
                LogRecordSet oRecordSet = oLogQuery.Execute(query, oEVTInputFormat);

                // Browse the recordset
                for (; !oRecordSet.atEnd(); oRecordSet.moveNext())
                {
                    //Console.WriteLine(oRecordSet.getRecord().toNativeString(","));
                }

                // Close the recordset
                oRecordSet.close();
            }
            catch (System.Runtime.InteropServices.COMException exc)
            {
                Console.WriteLine("Unexpected error: " + exc.Message);
            }
        }
Beispiel #2
0
        private void EventWatcher(object ploc)
        {
            string location = ploc.ToString();

            LogQuery oLogQuery = new LogQuery();

            LogManager.GetCurrentClassLogger().Info("WindowsEvent Input Listener Ready");

            // Instantiate the Event Log Input Format object
            var iFmt = new EventLogInputFormat()
            {
                binaryFormat  = _arguments.BinaryFormat.ToString(),
                direction     = _arguments.Direction.ToString(),
                formatMsg     = _arguments.FormatMsg,
                fullEventCode = _arguments.FullEventCode,
                fullText      = _arguments.FullText,
                msgErrorMode  = _arguments.MsgErrorMode.ToString(),
                stringsSep    = _arguments.StringsSep,
                resolveSIDs   = _arguments.ResolveSIDS
            };

            oLogQuery = null;

            Dictionary <string, Int64> logFileMaxRecords = new Dictionary <string, Int64>();

            // Execute the query
            while (!CancelToken.IsCancellationRequested)
            {
                try
                {
                    Thread.CurrentThread.Priority = ThreadPriority.BelowNormal;

                    oLogQuery = new LogQuery();

                    var qfiles  = string.Format("SELECT Distinct [EventLog] FROM {0}", location);
                    var rsfiles = oLogQuery.Execute(qfiles, iFmt);
                    for (; !rsfiles.atEnd(); rsfiles.moveNext())
                    {
                        var    record  = rsfiles.getRecord();
                        string logName = record.getValue("EventLog") as string;
                        if (!logFileMaxRecords.ContainsKey(logName))
                        {
                            var qcount = string.Format("SELECT max(RecordNumber) as MaxRecordNumber FROM {0}", logName);
                            var rcount = oLogQuery.Execute(qcount, iFmt);
                            var qr     = rcount.getRecord();
                            var lrn    = (Int64)qr.getValueEx("MaxRecordNumber");
                            logFileMaxRecords[logName] = lrn;
                        }
                    }


                    foreach (string fileName in logFileMaxRecords.Keys.ToList())
                    {
                        var lastRecordNumber = logFileMaxRecords[fileName];
                        var query            = string.Format("SELECT * FROM {0} where RecordNumber > {1}", location, lastRecordNumber);

                        var rs = oLogQuery.Execute(query, iFmt);
                        // Browse the recordset
                        for (; !rs.atEnd(); rs.moveNext())
                        {
                            var record = rs.getRecord();
                            var json   = new JObject();
                            foreach (var field in _arguments.Fields)
                            {
                                object v = record.getValue(field.Name);
                                if (field.Name == "Data")
                                {
                                    v = ToPrintable(v.ToString());
                                }
                                json.Add(new JProperty(field.Name, v));
                            }

                            var lrn = (Int64)record.getValueEx("RecordNumber");
                            logFileMaxRecords[fileName] = lrn;

                            record = null;
                            ProcessJson(json);
                            _receivedMessages++;
                            json = null;
                        }
                        // Close the recordset
                        rs.close();
                        rs = null;
                        GC.Collect();
                    }
                }
                catch (System.Threading.ThreadAbortException tex)
                {
                    Thread.ResetAbort();
                    break;
                }
                catch (Exception ex)
                {
                    LogManager.GetCurrentClassLogger().Error(ex);
                }

                try
                {
                    Thread.CurrentThread.Priority = ThreadPriority.Normal;
                    System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000);
                }
                catch (System.Threading.ThreadAbortException tex)
                {
                    Thread.ResetAbort();
                    break;
                }
                catch (Exception ex)
                {
                    LogManager.GetCurrentClassLogger().Error(ex);
                }
            }

            Finished();
        }
        private void IISW3CWatcher(string location)
        {
            LogManager.GetCurrentClassLogger().Info("IISW3Listener Ready For {0}", location);

            var oLogQuery = new LogQuery();

            var iFmt = new W3CLogInputFormat()
            {
                codepage = _arguments.CodePage,
                iCodepage = _arguments.CodePage,
                doubleQuotedStrings = _arguments.DoubleQuotes,
                detectTypesLines = _arguments.DtLines,
                dQuotes = _arguments.DoubleQuotes,
                separator = _arguments.Separator
            };

            Dictionary<string, Int64> logFileMaxRecords = new Dictionary<string, Int64>();
            using (var syncHandle = new ManualResetEventSlim())
            {
                // Execute the query
                while (!Stop)
                {
                    // Execute the query
                    if (!CancelToken.IsCancellationRequested)
                    {
                        try
                        {
                            oLogQuery = new LogQuery();

                            var qfiles = string.Format("SELECT Distinct [LogFilename] FROM {0}", location);
                            var rsfiles = oLogQuery.Execute(qfiles, iFmt);
                            for (; !rsfiles.atEnd(); rsfiles.moveNext())
                            {
                                var record = rsfiles.getRecord();
                                string fileName = record.getValue("LogFilename") as string;
                                if (!logFileMaxRecords.ContainsKey(fileName))
                                {
                                    var qcount = string.Format("SELECT max(RowNumber) as MaxRecordNumber FROM {0}",
                                        fileName);
                                    var rcount = oLogQuery.Execute(qcount, iFmt);
                                    var qr = rcount.getRecord();
                                    var lrn = (Int64)qr.getValueEx("MaxRecordNumber");
                                    logFileMaxRecords[fileName] = lrn;
                                }
                            }

                            foreach (string fileName in logFileMaxRecords.Keys.ToList())
                            {
                                var lastRecordNumber = logFileMaxRecords[fileName];
                                var query = string.Format(
                                    "SELECT * FROM '{0}' Where RowNumber > {1} order by RowNumber", fileName,
                                    lastRecordNumber);
                                var rs = oLogQuery.Execute(query, iFmt);
                                var colMap = new Dictionary<string, int>();
                                for (int col = 0; col < rs.getColumnCount(); col++)
                                {
                                    string colName = rs.getColumnName(col);
                                    colMap[colName] = col;
                                }

                                // Browse the recordset
                                for (; !rs.atEnd(); rs.moveNext())
                                {
                                    var record = rs.getRecord();
                                    var json = new JObject();
                                    foreach (var field in colMap.Keys)
                                    {
                                        object v = record.getValue(field);
                                        if (field == "date" || field == "time")
                                        {
                                            DateTime dt = DateTime.Parse(v.ToString());
                                            json.Add(new JProperty(field, dt));
                                        }
                                        else
                                            json.Add(new JProperty(field, v));
                                    }
                                    ProcessJson(json);
                                    _receivedMessages++;
                                    var lrn = (Int64)record.getValueEx("RowNumber");
                                    logFileMaxRecords[fileName] = lrn;
                                    record = null;
                                    json = null;
                                }
                                // Close the recordset
                                rs.close();
                            }
                            if (!Stop)
                                syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
                        }
                        catch (OperationCanceledException)
                        {
                            break;
                        }
                        catch (Exception ex)
                        {
                            LogManager.GetCurrentClassLogger().Error(ex);
                        }
                    }
                }
            }

            Finished();
        }
        private void IISW3CWatcher(string location)
        {
            LogManager.GetCurrentClassLogger().Info("IISW3Listener Ready For {0}", location);

            var oLogQuery = new LogQuery();

            var iFmt = new W3CLogInputFormat()
            {
                codepage            = _arguments.CodePage,
                iCodepage           = _arguments.CodePage,
                doubleQuotedStrings = _arguments.DoubleQuotes,
                detectTypesLines    = _arguments.DtLines,
                dQuotes             = _arguments.DoubleQuotes,
                separator           = _arguments.Separator
            };

            Dictionary <string, Int64> logFileMaxRecords = new Dictionary <string, Int64>();

            using (var syncHandle = new ManualResetEventSlim())
            {
                // Execute the query
                while (!Stop)
                {
                    // Execute the query
                    if (!CancelToken.IsCancellationRequested)
                    {
                        try
                        {
                            oLogQuery = new LogQuery();

                            var qfiles  = string.Format("SELECT Distinct [LogFilename] FROM {0}", location);
                            var rsfiles = oLogQuery.Execute(qfiles, iFmt);
                            for (; !rsfiles.atEnd(); rsfiles.moveNext())
                            {
                                var    record   = rsfiles.getRecord();
                                string fileName = record.getValue("LogFilename") as string;
                                if (!logFileMaxRecords.ContainsKey(fileName))
                                {
                                    var qcount = string.Format("SELECT max(RowNumber) as MaxRecordNumber FROM {0}",
                                                               fileName);
                                    var rcount = oLogQuery.Execute(qcount, iFmt);
                                    var qr     = rcount.getRecord();
                                    var lrn    = (Int64)qr.getValueEx("MaxRecordNumber");
                                    logFileMaxRecords[fileName] = lrn;
                                }
                            }


                            foreach (string fileName in logFileMaxRecords.Keys.ToList())
                            {
                                var lastRecordNumber = logFileMaxRecords[fileName];
                                var query            = string.Format(
                                    "SELECT * FROM '{0}' Where RowNumber > {1} order by RowNumber", fileName,
                                    lastRecordNumber);
                                var rs     = oLogQuery.Execute(query, iFmt);
                                var colMap = new Dictionary <string, int>();
                                for (int col = 0; col < rs.getColumnCount(); col++)
                                {
                                    string colName = rs.getColumnName(col);
                                    colMap[colName] = col;
                                }

                                // Browse the recordset
                                for (; !rs.atEnd(); rs.moveNext())
                                {
                                    var record = rs.getRecord();
                                    var json   = new JObject();
                                    foreach (var field in colMap.Keys)
                                    {
                                        object v = record.getValue(field);
                                        if (field == "date" || field == "time")
                                        {
                                            DateTime dt = DateTime.Parse(v.ToString());
                                            json.Add(new JProperty(field, dt));
                                        }
                                        else
                                        {
                                            json.Add(new JProperty(field, v));
                                        }
                                    }
                                    ProcessJson(json);
                                    _receivedMessages++;
                                    var lrn = (Int64)record.getValueEx("RowNumber");
                                    logFileMaxRecords[fileName] = lrn;
                                    record = null;
                                    json   = null;
                                }
                                // Close the recordset
                                rs.close();
                            }
                            if (!Stop)
                            {
                                syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
                            }
                        }
                        catch (OperationCanceledException)
                        {
                            break;
                        }
                        catch (Exception ex)
                        {
                            LogManager.GetCurrentClassLogger().Error(ex);
                        }
                    }
                }
            }

            Finished();
        }
Beispiel #5
0
        private void FileWatcher(string fileToWatch)
        {
            var iFmt = new TextLineInputFormat()
            {
                iCodepage      = _arguments.CodePage,
                splitLongLines = _arguments.SplitLongLines,
                recurse        = _arguments.Recurse
            };


            // Execute the query
            while (!CancelToken.IsCancellationRequested)
            {
                var oLogQuery = new LogQuery();
                try
                {
                    Thread.CurrentThread.Priority = ThreadPriority.BelowNormal;

                    var qfiles  = string.Format("SELECT Distinct [LogFilename] FROM {0}", fileToWatch);
                    var rsfiles = oLogQuery.Execute(qfiles, iFmt);
                    for (; !rsfiles.atEnd(); rsfiles.moveNext())
                    {
                        var      record  = rsfiles.getRecord();
                        string   logName = record.getValue("LogFilename") as string;
                        FileInfo fi      = new FileInfo(logName);

                        if (!fi.Exists)
                        {
                            _logFileCreationTimes.Remove(logName);
                            _logFileMaxRecords.Remove(logName);
                            _logFileSizes.Remove(logName);
                        }

                        _logFileSampleTimes[logName] = DateTime.UtcNow;

                        DateTime creationTime = fi.CreationTimeUtc;
                        bool     logHasRolled = (_logFileCreationTimes.ContainsKey(logName) && creationTime > _logFileCreationTimes[logName]) ||
                                                (_logFileSizes.ContainsKey(logName) && fi.Length < _logFileSizes[logName]);


                        if (!_logFileMaxRecords.ContainsKey(logName) || logHasRolled)
                        {
                            _logFileCreationTimes[logName] = creationTime;
                            _logFileSizes[logName]         = fi.Length;
                            var qcount = string.Format("SELECT max(Index) as MaxRecordNumber FROM {0}", logName);
                            var rcount = oLogQuery.Execute(qcount, iFmt);
                            var qr     = rcount.getRecord();
                            var lrn    = (Int64)qr.getValueEx("MaxRecordNumber");
                            if (logHasRolled)
                            {
                                LogManager.GetCurrentClassLogger().Info("Log {0} has rolled", logName);
                                lrn = 0;
                            }
                            _logFileMaxRecords[logName] = lrn;
                        }

                        _logFileSizes[logName] = fi.Length;
                    }
                    rsfiles.close();
                    foreach (string fileName in _logFileMaxRecords.Keys.ToList())
                    {
                        var lastRecordNumber = _logFileMaxRecords[fileName];
                        var query            = string.Format("SELECT * FROM {0} where Index > {1}", fileName, lastRecordNumber);

                        var rs = oLogQuery.Execute(query, iFmt);
                        Dictionary <string, int> colMap = new Dictionary <string, int>();
                        for (int col = 0; col < rs.getColumnCount(); col++)
                        {
                            string colName = rs.getColumnName(col);
                            colMap[colName] = col;
                        }

                        // Browse the recordset
                        for (; !rs.atEnd(); rs.moveNext())
                        {
                            var record = rs.getRecord();

                            var json = new JObject();
                            foreach (var field in _arguments.Fields)
                            {
                                if (!colMap.ContainsKey(field.Name))
                                {
                                    continue;
                                }

                                object v = record.getValue(field.Name);
                                if (field.DataType == typeof(DateTime))
                                {
                                    DateTime dt = DateTime.Parse(v.ToString());
                                    json.Add(new JProperty(field.Name, dt));
                                }
                                else
                                {
                                    json.Add(new JProperty(field.Name, v));
                                }
                            }
                            string msg = json["Text"].ToString();
                            if (!string.IsNullOrEmpty(msg))
                            {
                                ProcessJson(json);
                                _receivedMessages++;
                            }

                            var lrn = (Int64)record.getValueEx("Index");
                            _logFileMaxRecords[fileName] = lrn;
                            GC.Collect();
                        }

                        colMap.Clear();
                        // Close the recordset
                        rs.close();
                        rs = null;
                        GC.Collect();
                    }
                }
                catch (Exception ex)
                {
                    LogManager.GetCurrentClassLogger().Error(ex);
                }
                finally
                {
                    oLogQuery = null;
                }

                Thread.CurrentThread.Priority = ThreadPriority.Normal;
                System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000);
            }

            Finished();
        }
        private void FileWatcher(string fileToWatch)
        {
            var iFmt = new TextLineInputFormat()
            {
                iCodepage = _arguments.CodePage,
                splitLongLines = _arguments.SplitLongLines,
                recurse = _arguments.Recurse
            };

            Dictionary<string, string> _fnfmap = new Dictionary<string, string>();

            using (var syncHandle = new ManualResetEventSlim())
            {
                // Execute the query
                while (!Stop)
                {
                    var oLogQuery = new LogQuery();
                    if (!CancelToken.IsCancellationRequested)
                    {
                        try
                        {
                            var qfiles = string.Format("SELECT Distinct [LogFilename] FROM {0}", fileToWatch);
                            var rsfiles = oLogQuery.Execute(qfiles, iFmt);
                            for (; !rsfiles.atEnd(); rsfiles.moveNext())
                            {
                                var record = rsfiles.getRecord();
                                string logName = record.getValue("LogFilename") as string;
                                FileInfo fi = new FileInfo(logName);

                                var dbe = LogsFileDatabase.LookupLogFile(logName);

                                SaveVisitedFileName(dbe.FileName);

                                DateTime creationTime = fi.CreationTimeUtc;
                                bool logHasRolled = dbe.NewFile || (creationTime != dbe.CreationTimeUtc || fi.Length < dbe.LastPosition);

                                if (logHasRolled)
                                {
                                    LogManager.GetCurrentClassLogger().Info("Log {0} has rolled", logName);
                                    LogsFileDatabase.Roll(dbe);
                                }

                                // Log has rolled or this is a new file, or we haven't processed yet.
                                bool processWholeFile = logHasRolled || !dbe.ProcessedFile;

                                if (processWholeFile)
                                    LogsFileDatabase.Update(dbe, true, 0);

                            }
                            rsfiles.close();
                            foreach (string fileName in Files.ToList())
                            {
                                var dbe = LogsFileDatabase.LookupLogFile(fileName);

                                var lastRecordNumber = dbe.LastPosition;
                                var query = string.Format("SELECT * FROM {0} where Index > {1}", fileName,
                                    lastRecordNumber);

                                var rs = oLogQuery.Execute(query, iFmt);
                                Dictionary<string, int> colMap = new Dictionary<string, int>();
                                for (int col = 0; col < rs.getColumnCount(); col++)
                                {
                                    string colName = rs.getColumnName(col);
                                    colMap[colName] = col;
                                }

                                // Browse the recordset
                                for (; !rs.atEnd(); rs.moveNext())
                                {
                                    var record = rs.getRecord();

                                    var json = new JObject();
                                    foreach (var field in _arguments.Fields)
                                    {
                                        if (!colMap.ContainsKey(field.Name))
                                            continue;

                                        if (json["logSource"] == null)
                                        {
                                            if (string.IsNullOrEmpty(_arguments.LogSource))
                                                json.Add(new JProperty("logSource", fileName));
                                            else
                                                json.Add(new JProperty("logSource", _arguments.LogSource));
                                        }

                                        object v = record.getValue(field.Name);
                                        if (field.DataType == typeof(DateTime))
                                        {
                                            DateTime dt = DateTime.Parse(v.ToString());
                                            json.Add(new JProperty(field.Name, dt));
                                        }
                                        else
                                            json.Add(new JProperty(field.Name, v));
                                    }
                                    string msg = json["Text"].ToString();
                                    if (!string.IsNullOrEmpty(msg))
                                    {
                                        if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
                                        {
                                            _codec.Apply(msg, this);
                                            _receivedMessages++;
                                            dbe.IncrementLineCount();
                                        }
                                        else
                                        {
                                            ProcessJson(json);
                                            dbe.IncrementLineCount();
                                             _receivedMessages++;
                                        }
                                    }

                                    var lrn = (Int64)record.getValueEx("Index");
                                    LogsFileDatabase.Update(dbe, true, lrn);
                                    GC.Collect();
                                }

                                colMap.Clear();
                                // Close the recordset
                                rs.close();
                                rs = null;
                                GC.Collect();

                            }
                        }
                        catch (FileNotFoundException fnfex)
                        {
                            string fn = fnfex.FileName;

                            if (!string.IsNullOrEmpty(fn) && !_fnfmap.ContainsKey(fn))
                            {
                                LogManager.GetCurrentClassLogger().Warn(fnfex.Message);
                                _fnfmap[fn] = fn;
                            }
                        }
                        catch (OperationCanceledException)
                        {
                            break;
                        }
                        catch (Exception ex)
                        {
                            LogManager.GetCurrentClassLogger().Error(ex);
                        }
                        finally
                        {
                            try
                            {
                                oLogQuery = null;
                                // Sleep
                                if (!Stop)
                                    syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
                            }
                            catch (OperationCanceledException)
                            {
                            }
                            catch (Exception ex1)
                            {
                                LogManager.GetCurrentClassLogger().Warn(ex1);
                            }
                        }
                    }
                }
                Finished();
            }
        }
        private void IISW3CWatcher(string location)
        {
            LogManager.GetCurrentClassLogger().Info("IISW3Listener Ready For {0}", location);

            var oLogQuery = new LogQuery();

            var iFmt = new IISW3CLogInputFormat()
            {
                codepage        = _arguments.CodePage,
                consolidateLogs = true,
                dirTime         = _arguments.DirTime,
                dQuotes         = _arguments.DoubleQuotes,
                recurse         = _arguments.Recurse,
                useDoubleQuotes = _arguments.DoubleQuotes
            };

            if (_arguments.MinDateMod.HasValue)
            {
                iFmt.minDateMod = _arguments.MinDateMod.Value.ToString("yyyy-MM-dd hh:mm:ss");
            }

            Dictionary <string, Int64> logFileMaxRecords = new Dictionary <string, Int64>();


            // Execute the query
            while (!CancelToken.IsCancellationRequested)
            {
                try
                {
                    oLogQuery = new LogQuery();

                    var qfiles  = string.Format("SELECT Distinct [LogFilename] FROM {0}", location);
                    var rsfiles = oLogQuery.Execute(qfiles, iFmt);
                    for (; !rsfiles.atEnd(); rsfiles.moveNext())
                    {
                        var    record   = rsfiles.getRecord();
                        string fileName = record.getValue("LogFilename") as string;
                        if (!logFileMaxRecords.ContainsKey(fileName))
                        {
                            var qcount = string.Format("SELECT max(LogRow) as MaxRecordNumber FROM {0}", fileName);
                            var rcount = oLogQuery.Execute(qcount, iFmt);
                            var qr     = rcount.getRecord();
                            var lrn    = (Int64)qr.getValueEx("MaxRecordNumber");
                            logFileMaxRecords[fileName] = lrn;
                        }
                    }


                    foreach (string fileName in logFileMaxRecords.Keys.ToList())
                    {
                        var lastRecordNumber = logFileMaxRecords[fileName];
                        var query            = string.Format("SELECT * FROM '{0}' Where LogRow > {1}", fileName, lastRecordNumber);

                        var rs     = oLogQuery.Execute(query, iFmt);
                        var colMap = new Dictionary <string, int>();
                        for (int col = 0; col < rs.getColumnCount(); col++)
                        {
                            string colName = rs.getColumnName(col);
                            colMap[colName] = col;
                        }

                        // Browse the recordset
                        for (; !rs.atEnd(); rs.moveNext())
                        {
                            var record = rs.getRecord();
                            var json   = new JObject();
                            foreach (var field in _arguments.Fields)
                            {
                                if (!colMap.ContainsKey(field.Name))
                                {
                                    continue;
                                }

                                object v = record.getValue(field.Name);
                                if (field.DataType == typeof(DateTime))
                                {
                                    DateTime dt = DateTime.Parse(v.ToString());
                                    json.Add(new JProperty(field.Name, dt));
                                }
                                else
                                {
                                    json.Add(new JProperty(field.Name, v));
                                }
                            }
                            ProcessJson(json);
                            _receivedMessages++;
                            var lrn = (Int64)record.getValueEx("LogRow");
                            logFileMaxRecords[fileName] = lrn;
                            record = null;
                            json   = null;
                        }
                        // Close the recordset
                        rs.close();
                        GC.Collect();
                    }
                }
                catch (Exception ex)
                {
                    LogManager.GetCurrentClassLogger().Error(ex);
                }

                System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000);
            }

            Finished();
        }
Beispiel #8
0
        private void FileWatcher(string fileToWatch)
        {
            var iFmt = new TextLineInputFormat()
            {
                iCodepage      = _arguments.CodePage,
                splitLongLines = _arguments.SplitLongLines,
                recurse        = _arguments.Recurse
            };

            Dictionary <string, string> _fnfmap = new Dictionary <string, string>();

            using (var syncHandle = new ManualResetEventSlim())
            {
                // Execute the query
                while (!Stop)
                {
                    var oLogQuery = new LogQuery();
                    if (!CancelToken.IsCancellationRequested)
                    {
                        try
                        {
                            var qfiles  = string.Format("SELECT Distinct [LogFilename] FROM {0}", fileToWatch);
                            var rsfiles = oLogQuery.Execute(qfiles, iFmt);
                            for (; !rsfiles.atEnd(); rsfiles.moveNext())
                            {
                                var      record  = rsfiles.getRecord();
                                string   logName = record.getValue("LogFilename") as string;
                                FileInfo fi      = new FileInfo(logName);

                                var dbe = LogsFileDatabase.LookupLogFile(logName);

                                SaveVisitedFileName(dbe.FileName);

                                DateTime creationTime = fi.CreationTimeUtc;
                                bool     logHasRolled = dbe.NewFile || (creationTime != dbe.CreationTimeUtc || fi.Length < dbe.LastPosition);

                                if (logHasRolled)
                                {
                                    LogManager.GetCurrentClassLogger().Info("Log {0} has rolled", logName);
                                    LogsFileDatabase.Roll(dbe);
                                }

                                // Log has rolled or this is a new file, or we haven't processed yet.
                                bool processWholeFile = logHasRolled || !dbe.ProcessedFile;

                                if (processWholeFile)
                                {
                                    LogsFileDatabase.Update(dbe, true, 0);
                                }
                            }
                            rsfiles.close();
                            foreach (string fileName in Files.ToList())
                            {
                                var dbe = LogsFileDatabase.LookupLogFile(fileName);

                                var lastRecordNumber = dbe.LastPosition;
                                var query            = string.Format("SELECT * FROM {0} where Index > {1}", fileName,
                                                                     lastRecordNumber);

                                var rs = oLogQuery.Execute(query, iFmt);
                                Dictionary <string, int> colMap = new Dictionary <string, int>();
                                for (int col = 0; col < rs.getColumnCount(); col++)
                                {
                                    string colName = rs.getColumnName(col);
                                    colMap[colName] = col;
                                }

                                // Browse the recordset
                                for (; !rs.atEnd(); rs.moveNext())
                                {
                                    var record = rs.getRecord();

                                    var json = new JObject();
                                    foreach (var field in _arguments.Fields)
                                    {
                                        if (!colMap.ContainsKey(field.Name))
                                        {
                                            continue;
                                        }

                                        if (json["logSource"] == null)
                                        {
                                            if (string.IsNullOrEmpty(_arguments.LogSource))
                                            {
                                                json.Add(new JProperty("logSource", fileName));
                                            }
                                            else
                                            {
                                                json.Add(new JProperty("logSource", _arguments.LogSource));
                                            }
                                        }

                                        object v = record.getValue(field.Name);
                                        if (field.DataType == typeof(DateTime))
                                        {
                                            DateTime dt = DateTime.Parse(v.ToString());
                                            json.Add(new JProperty(field.Name, dt));
                                        }
                                        else
                                        {
                                            json.Add(new JProperty(field.Name, v));
                                        }
                                    }
                                    string msg = json["Text"].ToString();
                                    if (!string.IsNullOrEmpty(msg))
                                    {
                                        if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
                                        {
                                            _codec.Apply(msg, this);
                                            _receivedMessages++;
                                            dbe.IncrementLineCount();
                                        }
                                        else
                                        {
                                            ProcessJson(json);
                                            dbe.IncrementLineCount();
                                            _receivedMessages++;
                                        }
                                    }

                                    var lrn = (Int64)record.getValueEx("Index");
                                    LogsFileDatabase.Update(dbe, true, lrn);
                                    GC.Collect();
                                }

                                colMap.Clear();
                                // Close the recordset
                                rs.close();
                                rs = null;
                                GC.Collect();
                            }
                        }
                        catch (FileNotFoundException fnfex)
                        {
                            string fn = fnfex.FileName;

                            if (!string.IsNullOrEmpty(fn) && !_fnfmap.ContainsKey(fn))
                            {
                                LogManager.GetCurrentClassLogger().Warn(fnfex.Message);
                                _fnfmap[fn] = fn;
                            }
                        }
                        catch (OperationCanceledException)
                        {
                            break;
                        }
                        catch (Exception ex)
                        {
                            LogManager.GetCurrentClassLogger().Error(ex);
                        }
                        finally
                        {
                            try
                            {
                                oLogQuery = null;
                                // Sleep
                                if (!Stop)
                                {
                                    syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
                                }
                            }
                            catch (OperationCanceledException)
                            {
                            }
                            catch (Exception ex1)
                            {
                                LogManager.GetCurrentClassLogger().Warn(ex1);
                            }
                        }
                    }
                }
                Finished();
            }
        }