public PluginContext(IConfiguration config, ILogger logger, IMetrics metrics, BookmarkManager bookmarkManager,
IDictionary <string, ICredentialProvider> credentialProviders, IParameterStore parameterStore)
{
_config = config;
_logger = logger;
_metrics = metrics;
_bookmarkManager = bookmarkManager;
_credentialProviders = credentialProviders;
_parameterStore = parameterStore;
}
public PluginContext(IConfiguration config, ILogger logger, IMetrics metrics, BookmarkManager bookmarkManager)
: this(config, logger, metrics, bookmarkManager, null, null)
{
}
private void ProcessNewOrExpandedFiles(string filePath, string relativeFilePath, long fileSize)
{
//Only process new or expanded files
if (_logFiles.TryGetValue(relativeFilePath, out TContext context))
{
// If there is no registered bookmark and we're bookmarking on buffer flush,
// that means the file was read and events buffered, but never uploaded by the source.
var position = this.bookmarkOnBufferFlush
? (BookmarkManager.GetBookmark(this.GetBookmarkName(filePath)) ?? BookmarkManager.RegisterBookmark(this.GetBookmarkName(filePath), 0, (pos) => this.SaveBookmark())).Position
: context.Position;
if (fileSize > position)
{
// The file is expanded compared with position saved in bookmark
AddToBuffer(relativeFilePath);
}
}
else
{
//New file
_logFiles.Add(relativeFilePath, CreateLogSourceInfo(filePath, 0));
if (this.bookmarkOnBufferFlush)
{
BookmarkManager.RegisterBookmark(this.GetBookmarkName(filePath), 0, (pos) => this.SaveBookmark());
}
AddToBuffer(relativeFilePath);
}
}
private void ReadBookmarkFromLogFiles()
{
var candidateFiles = _fileFilters.SelectMany(filter => this.GetFiles(_directory, filter))
.Where(file => !ShouldExclude(file));
if (_fileFilters.Length > 1)
{
//If there are multiple filters, they may overlap so we need to dedupe
candidateFiles = candidateFiles.Distinct();
}
string[] files = candidateFiles.ToArray();
foreach (string filePath in files)
{
FileInfo fi = new FileInfo(filePath);
var relativeFilePath = GetRelativeFilePath(filePath, _directory);
long fileSize = fi.Length;
if (_hasBookmark && this.InitialPosition != InitialPositionEnum.EOS)
{
ProcessNewOrExpandedFiles(filePath, relativeFilePath, fileSize);
continue;
}
switch (this.InitialPosition)
{
case InitialPositionEnum.EOS:
_logFiles[relativeFilePath] = CreateLogSourceInfo(filePath, fi.Length);
break;
case InitialPositionEnum.BOS:
//Process all files
_logFiles[relativeFilePath] = CreateLogSourceInfo(filePath, 0);
if (this.bookmarkOnBufferFlush)
{
BookmarkManager.RegisterBookmark(this.GetBookmarkName(filePath), 0, (pos) => this.SaveBookmark());
}
AddToBuffer(relativeFilePath);
break;
case InitialPositionEnum.Bookmark:
_logFiles[relativeFilePath] = CreateLogSourceInfo(filePath, fi.Length);
if (this.bookmarkOnBufferFlush)
{
BookmarkManager.RegisterBookmark(this.GetBookmarkName(filePath), fi.Length, (pos) => this.SaveBookmark());
}
break;
case InitialPositionEnum.Timestamp:
if (fi.LastWriteTimeUtc > this.InitialPositionTimestamp)
{
_logFiles[relativeFilePath] = CreateLogSourceInfo(filePath, 0);
if (this.bookmarkOnBufferFlush)
{
BookmarkManager.RegisterBookmark(this.GetBookmarkName(filePath), 0, (pos) => this.SaveBookmark());
}
AddToBuffer(relativeFilePath);
}
else
{
_logFiles[relativeFilePath] = CreateLogSourceInfo(filePath, fi.Length);
if (this.bookmarkOnBufferFlush)
{
BookmarkManager.RegisterBookmark(this.GetBookmarkName(filePath), fi.Length, (pos) => this.SaveBookmark());
}
}
break;
default:
throw new NotImplementedException($"Initial Position {this.InitialPosition} is not supported");
}
}
}
protected virtual (long recordsRead, long bytesRead) ParseLogFile(string relativeFilePath, string fullPath)
{
long recordsRead = 0;
long bytesRead = 0;
int bookmarkId;
if (!_logFiles.TryGetValue(relativeFilePath, out TContext sourceInfo))
{
sourceInfo = CreateLogSourceInfo(fullPath, 0);
_logFiles.Add(relativeFilePath, sourceInfo);
bookmarkId = this.bookmarkOnBufferFlush ? BookmarkManager.RegisterBookmark(this.GetBookmarkName(fullPath), 0, (pos) => this.SaveBookmark()).Id : 0;
}
else
{
bookmarkId = BookmarkManager.GetBookmarkId(this.GetBookmarkName(fullPath));
}
try
{
using (var fs = new FileStream(fullPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (var sr = CreateStreamReader(fs, _encoding))
{
var records = _recordParser.ParseRecords(sr, sourceInfo);
foreach (var record in records)
{
ILogEnvelope envelope = (ILogEnvelope)record;
if (envelope != null &&
record.Timestamp > (this.InitialPositionTimestamp ?? DateTime.MinValue) &&
envelope.LineNumber > _skipLines)
{
record.BookmarkId = bookmarkId;
_recordSubject.OnNext(record);
recordsRead++;
}
if (!_started)
{
break;
}
}
//Need to grab the position before disposing the reader because disposing the reader will dispose the stream
bytesRead = fs.Position - sourceInfo.Position;
sourceInfo.Position = fs.Position;
sourceInfo.ConsecutiveIOExceptionCount = 0;
}
}
catch (IOException ex)
{
//Add it back to buffer for processing
AddToBuffer(relativeFilePath);
sourceInfo.ConsecutiveIOExceptionCount++;
if (sourceInfo.ConsecutiveIOExceptionCount >= this.NumberOfConsecutiveIOExceptionsToLogError)
{
_logger?.LogError(ex.ToMinimized());
}
}
catch (Exception ex)
{
_logger?.LogError(ex.ToMinimized());
}
return(recordsRead, bytesRead);
}
protected void OnTimer(object stateInfo)
{
_timer.Change(Timeout.Infinite, Timeout.Infinite);
try
{
if (!Directory.Exists(_directory))
{
Reset();
return;
}
//First to check whether filewatch events were missed
long bytesToRead = 0;
long filesToProcess = 0;
foreach (string relativeFilePath in _logFiles.Keys)
{
if (!_started)
{
break;
}
try
{
TContext fileContext = _logFiles[relativeFilePath];
FileInfo fi = new FileInfo(fileContext.FilePath);
long fileLength = fi.Length;
if (fileLength == fileContext.Position) //No change
{
continue;
}
else if (fileLength < fileContext.Position) //shrink or truncate
{
_logger?.LogWarning($"File: {fi.Name} shrunk or truncated from {fileContext.Position} to {fi.Length}");
//Other than malicious attack, the most likely scenario is file truncate so we will read from the beginning
fileContext.Position = 0;
BookmarkManager.ResetBookmarkPosition(this.GetBookmarkName(fileContext.FilePath), -1);
}
bytesToRead += fi.Length - fileContext.Position;
filesToProcess++;
AddToBuffer(relativeFilePath);
}
catch { }
}
_metrics?.PublishCounters(this.Id, MetricsConstants.CATEGORY_SOURCE, Metrics.CounterTypeEnum.CurrentValue, new Dictionary <string, MetricValue>()
{
{ MetricsConstants.DIRECTORY_SOURCE_BYTES_TO_READ, new MetricValue(bytesToRead, MetricUnit.Bytes) },
{ MetricsConstants.DIRECTORY_SOURCE_FILES_TO_PROCESS, new MetricValue(filesToProcess) },
});
string[] files = null;
lock (_buffer)
{
files = new string[_buffer.Count];
_buffer.CopyTo(files, 0);
_buffer.Clear();
}
(long recordsRead, long bytesRead) = ParseLogFiles(files);
if (!this.bookmarkOnBufferFlush)
{
SaveBookmark();
}
_metrics?.PublishCounters(this.Id, MetricsConstants.CATEGORY_SOURCE, Metrics.CounterTypeEnum.Increment, new Dictionary <string, MetricValue>()
{
{ MetricsConstants.DIRECTORY_SOURCE_RECORDS_READ, new MetricValue(recordsRead) },
{ MetricsConstants.DIRECTORY_SOURCE_BYTES_READ, new MetricValue(bytesRead, MetricUnit.Bytes) },
});
}
catch (Exception ex)
{
_logger?.LogError(ex.ToMinimized());
}
finally
{
if (_started)
{
_timer.Change(_interval, Timeout.Infinite);
}
}
}