private void FileWatcher(string fileToWatch) { var iFmt = new TextLineInputFormat() { iCodepage = _arguments.CodePage, splitLongLines = _arguments.SplitLongLines, recurse = _arguments.Recurse }; Dictionary <string, string> _fnfmap = new Dictionary <string, string>(); using (var syncHandle = new ManualResetEventSlim()) { // Execute the query while (!Stop) { var oLogQuery = new LogQuery(); if (!CancelToken.IsCancellationRequested) { try { var qfiles = string.Format("SELECT Distinct [LogFilename] FROM {0}", fileToWatch); var rsfiles = oLogQuery.Execute(qfiles, iFmt); for (; !rsfiles.atEnd(); rsfiles.moveNext()) { var record = rsfiles.getRecord(); string logName = record.getValue("LogFilename") as string; FileInfo fi = new FileInfo(logName); var dbe = LogsFileDatabase.LookupLogFile(logName); SaveVisitedFileName(dbe.FileName); DateTime creationTime = fi.CreationTimeUtc; bool logHasRolled = dbe.NewFile || (creationTime != dbe.CreationTimeUtc || fi.Length < dbe.LastPosition); if (logHasRolled) { LogManager.GetCurrentClassLogger().Info("Log {0} has rolled", logName); LogsFileDatabase.Roll(dbe); } // Log has rolled or this is a new file, or we haven't processed yet. bool processWholeFile = logHasRolled || !dbe.ProcessedFile; if (processWholeFile) { LogsFileDatabase.Update(dbe, true, 0); } } rsfiles.close(); foreach (string fileName in Files.ToList()) { var dbe = LogsFileDatabase.LookupLogFile(fileName); var lastRecordNumber = dbe.LastPosition; var query = string.Format("SELECT * FROM {0} where Index > {1}", fileName, lastRecordNumber); var rs = oLogQuery.Execute(query, iFmt); Dictionary <string, int> colMap = new Dictionary <string, int>(); for (int col = 0; col < rs.getColumnCount(); col++) { string colName = rs.getColumnName(col); colMap[colName] = col; } // Browse the recordset for (; !rs.atEnd(); rs.moveNext()) { var record = rs.getRecord(); var json = new JObject(); foreach (var field in _arguments.Fields) { if (!colMap.ContainsKey(field.Name)) { continue; } if (json["logSource"] == null) { if (string.IsNullOrEmpty(_arguments.LogSource)) { json.Add(new JProperty("logSource", fileName)); } else { json.Add(new JProperty("logSource", _arguments.LogSource)); } } object v = record.getValue(field.Name); if (field.DataType == typeof(DateTime)) { DateTime dt = DateTime.Parse(v.ToString()); json.Add(new JProperty(field.Name, dt)); } else { json.Add(new JProperty(field.Name, v)); } } string msg = json["Text"].ToString(); if (!string.IsNullOrEmpty(msg)) { if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline) { _codec.Apply(msg, this); _receivedMessages++; dbe.IncrementLineCount(); } else { ProcessJson(json); dbe.IncrementLineCount(); _receivedMessages++; } } var lrn = (Int64)record.getValueEx("Index"); LogsFileDatabase.Update(dbe, true, lrn); GC.Collect(); } colMap.Clear(); // Close the recordset rs.close(); rs = null; GC.Collect(); } } catch (FileNotFoundException fnfex) { string fn = fnfex.FileName; if (!string.IsNullOrEmpty(fn) && !_fnfmap.ContainsKey(fn)) { LogManager.GetCurrentClassLogger().Warn(fnfex.Message); _fnfmap[fn] = fn; } } catch (OperationCanceledException) { break; } catch (Exception ex) { LogManager.GetCurrentClassLogger().Error(ex); } finally { try { oLogQuery = null; // Sleep if (!Stop) { syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken); } } catch (OperationCanceledException) { } catch (Exception ex1) { LogManager.GetCurrentClassLogger().Warn(ex1); } } } } Finished(); } }
// One thread for each kind of file to watch, i.e. "*.log,*.txt" would be two separate // threads. private void TailFileWatcher(string fileToWatch) { Dictionary <string, string> _fnfmap = new Dictionary <string, string>(); using (var syncHandle = new ManualResetEventSlim()) { // Execute the query while (!Stop && !CancelToken.IsCancellationRequested) { try { if (!CancelToken.IsCancellationRequested) { var isWildcardPattern = fileToWatch.Contains('*'); string path = Path.GetDirectoryName(fileToWatch); string name = Path.GetFileName(fileToWatch); if (string.IsNullOrEmpty(path)) { path = "."; } LogManager.GetCurrentClassLogger().Trace(":{0} Tailing File: {1}", Thread.CurrentThread.ManagedThreadId, Path.Combine(path, name)); // Ok, we have a potential file filter here as 'fileToWatch' could be foo.log or *.log SearchOption so = SearchOption.TopDirectoryOnly; if (_arguments.Recurse == -1) { so = SearchOption.AllDirectories; } foreach (string fileName in Directory.GetFiles(path, name, so)) { var dbe = LogsFileDatabase.LookupLogFile(fileName); // We only spin up 1 thread for a file we haven't yet seen. if (isWildcardPattern && !HaveSeenFile(fileName) && dbe.NewFile) { LogManager.GetCurrentClassLogger().Debug(":{0} Starting Thread Tailing File: {1}", Thread.CurrentThread.ManagedThreadId, dbe.FileName); LogsFileDatabase.Update(dbe, false, dbe.LastPosition); Task.Factory.StartNew(() => TailFileWatcher(fileName)); } else if (!isWildcardPattern) { FileInfo fi = new FileInfo(dbe.FileName); SaveVisitedFileName(fileName); //LogManager.GetCurrentClassLogger().Info("Located File: {0}, New: {1}", dbe.FileName, dbe.NewFile); long length = fi.Length; bool logHasRolled = false; if (fi.Length < dbe.LastPosition || fi.CreationTimeUtc != dbe.CreationTimeUtc) { LogManager.GetCurrentClassLogger().Info("{0}: Log has Rolled: {1}", Thread.CurrentThread.ManagedThreadId, dbe.FileName); logHasRolled = true; LogsFileDatabase.Roll(dbe); } // Log has rolled or this is a file we are seeing for the first time. bool processWholeFile = logHasRolled || !dbe.ProcessedFile || dbe.NewFile; if (processWholeFile) { LogsFileDatabase.Update(dbe, true, 0); LogManager.GetCurrentClassLogger().Debug("{0}: Process Whole File: {1}", Thread.CurrentThread.ManagedThreadId, dbe.FileName); TailFileContents(dbe.FileName, 0, dbe); } else { TailFileContents(dbe.FileName, dbe.LastPosition, dbe); } } } } } catch (FileNotFoundException fnfex) { string fn = fnfex.FileName; if (!_fnfmap.ContainsKey(fn)) { LogManager.GetCurrentClassLogger().Warn(fnfex.Message); } _fnfmap[fn] = fn; } catch (IOException ioex) { LogManager.GetCurrentClassLogger().Debug("Log has rolled: {0}", ioex.Message); } catch (OperationCanceledException) { break; } catch (Exception ex) { LogManager.GetCurrentClassLogger().Error(ex); } finally { try { if (!Stop) { syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken); } } catch (OperationCanceledException) { Stop = true; } catch (Exception ex1) { LogManager.GetCurrentClassLogger().Warn(ex1); } } } } Finished(); }