public PluginContext(IConfiguration config, ILogger logger, IMetrics metrics, BookmarkManager bookmarkManager, IDictionary <string, ICredentialProvider> credentialProviders, IParameterStore parameterStore) { _config = config; _logger = logger; _metrics = metrics; _bookmarkManager = bookmarkManager; _credentialProviders = credentialProviders; _parameterStore = parameterStore; }
public PluginContext(IConfiguration config, ILogger logger, IMetrics metrics, BookmarkManager bookmarkManager) : this(config, logger, metrics, bookmarkManager, null, null) { }
private void ProcessNewOrExpandedFiles(string filePath, string relativeFilePath, long fileSize) { //Only process new or expanded files if (_logFiles.TryGetValue(relativeFilePath, out TContext context)) { // If there is no registered bookmark and we're bookmarking on buffer flush, // that means the file was read and events buffered, but never uploaded by the source. var position = this.bookmarkOnBufferFlush ? (BookmarkManager.GetBookmark(this.GetBookmarkName(filePath)) ?? BookmarkManager.RegisterBookmark(this.GetBookmarkName(filePath), 0, (pos) => this.SaveBookmark())).Position : context.Position; if (fileSize > position) { // The file is expanded compared with position saved in bookmark AddToBuffer(relativeFilePath); } } else { //New file _logFiles.Add(relativeFilePath, CreateLogSourceInfo(filePath, 0)); if (this.bookmarkOnBufferFlush) { BookmarkManager.RegisterBookmark(this.GetBookmarkName(filePath), 0, (pos) => this.SaveBookmark()); } AddToBuffer(relativeFilePath); } }
private void ReadBookmarkFromLogFiles() { var candidateFiles = _fileFilters.SelectMany(filter => this.GetFiles(_directory, filter)) .Where(file => !ShouldExclude(file)); if (_fileFilters.Length > 1) { //If there are multiple filters, they may overlap so we need to dedupe candidateFiles = candidateFiles.Distinct(); } string[] files = candidateFiles.ToArray(); foreach (string filePath in files) { FileInfo fi = new FileInfo(filePath); var relativeFilePath = GetRelativeFilePath(filePath, _directory); long fileSize = fi.Length; if (_hasBookmark && this.InitialPosition != InitialPositionEnum.EOS) { ProcessNewOrExpandedFiles(filePath, relativeFilePath, fileSize); continue; } switch (this.InitialPosition) { case InitialPositionEnum.EOS: _logFiles[relativeFilePath] = CreateLogSourceInfo(filePath, fi.Length); break; case InitialPositionEnum.BOS: //Process all files _logFiles[relativeFilePath] = CreateLogSourceInfo(filePath, 0); if (this.bookmarkOnBufferFlush) { BookmarkManager.RegisterBookmark(this.GetBookmarkName(filePath), 0, (pos) => this.SaveBookmark()); } AddToBuffer(relativeFilePath); break; case InitialPositionEnum.Bookmark: _logFiles[relativeFilePath] = CreateLogSourceInfo(filePath, fi.Length); if (this.bookmarkOnBufferFlush) { BookmarkManager.RegisterBookmark(this.GetBookmarkName(filePath), fi.Length, (pos) => this.SaveBookmark()); } break; case InitialPositionEnum.Timestamp: if (fi.LastWriteTimeUtc > this.InitialPositionTimestamp) { _logFiles[relativeFilePath] = CreateLogSourceInfo(filePath, 0); if (this.bookmarkOnBufferFlush) { BookmarkManager.RegisterBookmark(this.GetBookmarkName(filePath), 0, (pos) => this.SaveBookmark()); } AddToBuffer(relativeFilePath); } else { _logFiles[relativeFilePath] = CreateLogSourceInfo(filePath, fi.Length); if (this.bookmarkOnBufferFlush) { BookmarkManager.RegisterBookmark(this.GetBookmarkName(filePath), fi.Length, (pos) => this.SaveBookmark()); } } break; default: throw new NotImplementedException($"Initial Position {this.InitialPosition} is not supported"); } } }
protected virtual (long recordsRead, long bytesRead) ParseLogFile(string relativeFilePath, string fullPath) { long recordsRead = 0; long bytesRead = 0; int bookmarkId; if (!_logFiles.TryGetValue(relativeFilePath, out TContext sourceInfo)) { sourceInfo = CreateLogSourceInfo(fullPath, 0); _logFiles.Add(relativeFilePath, sourceInfo); bookmarkId = this.bookmarkOnBufferFlush ? BookmarkManager.RegisterBookmark(this.GetBookmarkName(fullPath), 0, (pos) => this.SaveBookmark()).Id : 0; } else { bookmarkId = BookmarkManager.GetBookmarkId(this.GetBookmarkName(fullPath)); } try { using (var fs = new FileStream(fullPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) using (var sr = CreateStreamReader(fs, _encoding)) { var records = _recordParser.ParseRecords(sr, sourceInfo); foreach (var record in records) { ILogEnvelope envelope = (ILogEnvelope)record; if (envelope != null && record.Timestamp > (this.InitialPositionTimestamp ?? DateTime.MinValue) && envelope.LineNumber > _skipLines) { record.BookmarkId = bookmarkId; _recordSubject.OnNext(record); recordsRead++; } if (!_started) { break; } } //Need to grab the position before disposing the reader because disposing the reader will dispose the stream bytesRead = fs.Position - sourceInfo.Position; sourceInfo.Position = fs.Position; sourceInfo.ConsecutiveIOExceptionCount = 0; } } catch (IOException ex) { //Add it back to buffer for processing AddToBuffer(relativeFilePath); sourceInfo.ConsecutiveIOExceptionCount++; if (sourceInfo.ConsecutiveIOExceptionCount >= this.NumberOfConsecutiveIOExceptionsToLogError) { _logger?.LogError(ex.ToMinimized()); } } catch (Exception ex) { _logger?.LogError(ex.ToMinimized()); } return(recordsRead, bytesRead); }
protected void OnTimer(object stateInfo) { _timer.Change(Timeout.Infinite, Timeout.Infinite); try { if (!Directory.Exists(_directory)) { Reset(); return; } //First to check whether filewatch events were missed long bytesToRead = 0; long filesToProcess = 0; foreach (string relativeFilePath in _logFiles.Keys) { if (!_started) { break; } try { TContext fileContext = _logFiles[relativeFilePath]; FileInfo fi = new FileInfo(fileContext.FilePath); long fileLength = fi.Length; if (fileLength == fileContext.Position) //No change { continue; } else if (fileLength < fileContext.Position) //shrink or truncate { _logger?.LogWarning($"File: {fi.Name} shrunk or truncated from {fileContext.Position} to {fi.Length}"); //Other than malicious attack, the most likely scenario is file truncate so we will read from the beginning fileContext.Position = 0; BookmarkManager.ResetBookmarkPosition(this.GetBookmarkName(fileContext.FilePath), -1); } bytesToRead += fi.Length - fileContext.Position; filesToProcess++; AddToBuffer(relativeFilePath); } catch { } } _metrics?.PublishCounters(this.Id, MetricsConstants.CATEGORY_SOURCE, Metrics.CounterTypeEnum.CurrentValue, new Dictionary <string, MetricValue>() { { MetricsConstants.DIRECTORY_SOURCE_BYTES_TO_READ, new MetricValue(bytesToRead, MetricUnit.Bytes) }, { MetricsConstants.DIRECTORY_SOURCE_FILES_TO_PROCESS, new MetricValue(filesToProcess) }, }); string[] files = null; lock (_buffer) { files = new string[_buffer.Count]; _buffer.CopyTo(files, 0); _buffer.Clear(); } (long recordsRead, long bytesRead) = ParseLogFiles(files); if (!this.bookmarkOnBufferFlush) { SaveBookmark(); } _metrics?.PublishCounters(this.Id, MetricsConstants.CATEGORY_SOURCE, Metrics.CounterTypeEnum.Increment, new Dictionary <string, MetricValue>() { { MetricsConstants.DIRECTORY_SOURCE_RECORDS_READ, new MetricValue(recordsRead) }, { MetricsConstants.DIRECTORY_SOURCE_BYTES_READ, new MetricValue(bytesRead, MetricUnit.Bytes) }, }); } catch (Exception ex) { _logger?.LogError(ex.ToMinimized()); } finally { if (_started) { _timer.Change(_interval, Timeout.Infinite); } } }