private void ProcessNewOrExpandedFiles(string filePath, string relativeFilePath, long fileSize)
{
//Only process new or expanded files
if (_logFiles.TryGetValue(relativeFilePath, out TContext context))
{
// If there is no registered bookmark and we're bookmarking on buffer flush,
// that means the file was read and events buffered, but never uploaded by the source.
var position = this.bookmarkOnBufferFlush
? (BookmarkManager.GetBookmark(this.GetBookmarkName(filePath)) ?? BookmarkManager.RegisterBookmark(this.GetBookmarkName(filePath), 0, (pos) => this.SaveBookmark())).Position
: context.Position;
if (fileSize > position)
{
// The file is expanded compared with position saved in bookmark
AddToBuffer(relativeFilePath);
}
}
else
{
//New file
_logFiles.Add(relativeFilePath, CreateLogSourceInfo(filePath, 0));
if (this.bookmarkOnBufferFlush)
{
BookmarkManager.RegisterBookmark(this.GetBookmarkName(filePath), 0, (pos) => this.SaveBookmark());
}
AddToBuffer(relativeFilePath);
}
}
protected virtual void OnChanged(object source, FileSystemEventArgs e)
{
try
{
if (this.includeSubdirectories && this.ShouldSkip(e.FullPath))
{
return;
}
string relativeFilePath = e.Name;
//Sometimes we receive event where e.name is null so we should just skip it
if (string.IsNullOrEmpty(relativeFilePath) || ShouldExclude(relativeFilePath) || !ShouldInclude(relativeFilePath))
{
return;
}
//The entries in _buffer should be deleted before _logfiles and added after _logfiles
switch (e.ChangeType)
{
case WatcherChangeTypes.Deleted:
if (!File.Exists(e.FullPath)) //macOS sometimes fires this event when a file is created so we need this extra check.
{
RemoveFromBuffer(relativeFilePath);
_logFiles.Remove(relativeFilePath);
BookmarkManager.RemoveBookmark(this.GetBookmarkName(e.FullPath));
}
break;
case WatcherChangeTypes.Created:
if (!_logFiles.ContainsKey(relativeFilePath))
{
_logFiles[relativeFilePath] = CreateLogSourceInfo(e.FullPath, 0);
if (this.bookmarkOnBufferFlush)
{
BookmarkManager.RegisterBookmark(this.GetBookmarkName(e.FullPath), 0, (pos) => this.SaveBookmark());
}
AddToBuffer(relativeFilePath);
}
break;
case WatcherChangeTypes.Changed:
AddToBuffer(relativeFilePath);
break;
}
_logger?.LogDebug($"ThreadId{Thread.CurrentThread.ManagedThreadId} File: {e.FullPath} ChangeType: {e.ChangeType}");
}
catch (Exception ex)
{
_logger?.LogError(ex.ToMinimized());
}
}
protected virtual void OnRenamed(object source, RenamedEventArgs e)
{
try
{
// this is for Subdirectories check only
if (this.includeSubdirectories && this.ShouldSkip(e.FullPath))
{
return;
}
//Sometimes we receive event where e.name is null so we should just skip it
if (string.IsNullOrEmpty(e.Name) || string.IsNullOrEmpty(e.OldName) ||
ShouldExclude(e.Name) || ShouldExclude(e.OldName) ||
(!ShouldInclude(e.Name) && !ShouldInclude(e.OldName)))
{
return;
}
//File name rotation
RemoveFromBuffer(e.OldName);
if (_logFiles.ContainsKey(e.OldName))
{
var newSourceInfo = CreateLogSourceInfo(e.FullPath, _logFiles[e.OldName].Position);
newSourceInfo.LineNumber = _logFiles[e.OldName].LineNumber;
_logFiles[e.Name] = newSourceInfo;
_logFiles.Remove(e.OldName);
var bookmark = BookmarkManager.GetBookmark(this.GetBookmarkName(e.OldFullPath));
if (bookmark != null)
{
BookmarkManager.RemoveBookmark(bookmark.Id);
BookmarkManager.RegisterBookmark(this.GetBookmarkName(e.FullPath), bookmark.Position, (id) => this.SaveBookmark());
}
}
else
{
var newSource = CreateLogSourceInfo(e.FullPath, 0);
_logFiles.Add(e.Name, newSource);
BookmarkManager.RegisterBookmark(this.GetBookmarkName(e.FullPath), 0, (id) => this.SaveBookmark());
}
}
catch (Exception ex)
{
_logger?.LogError(ex.ToMinimized());
}
finally
{
AddToBuffer(e.Name);
_logger?.LogInformation("File: {0} renamed to {1}", e.OldFullPath, e.FullPath);
}
}
public void LoadSavedBookmark()
{
lock (_bookmarkFileLock)
{
try
{
using (var fs = new FileStream(this.bookmarkPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (var sr = new StreamReader(fs))
{
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
if (!string.IsNullOrWhiteSpace(line))
{
try
{
string[] parts = line.Split(',');
var logSource = CreateLogSourceInfo(parts[0], long.Parse(parts[1]));
_logFiles[GetRelativeFilePath(parts[0], _directory)] = logSource;
if (this.bookmarkOnBufferFlush)
{
BookmarkManager.RegisterBookmark(this.GetBookmarkName(logSource.FilePath), logSource.Position, (id) => this.SaveBookmark());
}
}
catch (Exception ex)
{
//Allow continue processing because it is legitimate for system to remove log files while the agent is stopped
_logger?.LogWarning($"Fail to process bookmark {line}: {ex.ToMinimized()}");
}
}
}
}
}
catch (Exception ex)
{
_logger?.LogError($"Failed loading bookmark: {ex.ToMinimized()}");
throw; //Inform caller the error
}
}
}
private void ReadBookmarkFromLogFiles()
{
var candidateFiles = _fileFilters.SelectMany(filter => this.GetFiles(_directory, filter))
.Where(file => !ShouldExclude(file));
if (_fileFilters.Length > 1)
{
//If there are multiple filters, they may overlap so we need to dedupe
candidateFiles = candidateFiles.Distinct();
}
string[] files = candidateFiles.ToArray();
foreach (string filePath in files)
{
FileInfo fi = new FileInfo(filePath);
var relativeFilePath = GetRelativeFilePath(filePath, _directory);
long fileSize = fi.Length;
if (_hasBookmark && this.InitialPosition != InitialPositionEnum.EOS)
{
ProcessNewOrExpandedFiles(filePath, relativeFilePath, fileSize);
continue;
}
switch (this.InitialPosition)
{
case InitialPositionEnum.EOS:
_logFiles[relativeFilePath] = CreateLogSourceInfo(filePath, fi.Length);
break;
case InitialPositionEnum.BOS:
//Process all files
_logFiles[relativeFilePath] = CreateLogSourceInfo(filePath, 0);
if (this.bookmarkOnBufferFlush)
{
BookmarkManager.RegisterBookmark(this.GetBookmarkName(filePath), 0, (pos) => this.SaveBookmark());
}
AddToBuffer(relativeFilePath);
break;
case InitialPositionEnum.Bookmark:
_logFiles[relativeFilePath] = CreateLogSourceInfo(filePath, fi.Length);
if (this.bookmarkOnBufferFlush)
{
BookmarkManager.RegisterBookmark(this.GetBookmarkName(filePath), fi.Length, (pos) => this.SaveBookmark());
}
break;
case InitialPositionEnum.Timestamp:
if (fi.LastWriteTimeUtc > this.InitialPositionTimestamp)
{
_logFiles[relativeFilePath] = CreateLogSourceInfo(filePath, 0);
if (this.bookmarkOnBufferFlush)
{
BookmarkManager.RegisterBookmark(this.GetBookmarkName(filePath), 0, (pos) => this.SaveBookmark());
}
AddToBuffer(relativeFilePath);
}
else
{
_logFiles[relativeFilePath] = CreateLogSourceInfo(filePath, fi.Length);
if (this.bookmarkOnBufferFlush)
{
BookmarkManager.RegisterBookmark(this.GetBookmarkName(filePath), fi.Length, (pos) => this.SaveBookmark());
}
}
break;
default:
throw new NotImplementedException($"Initial Position {this.InitialPosition} is not supported");
}
}
}
protected virtual (long recordsRead, long bytesRead) ParseLogFile(string relativeFilePath, string fullPath)
{
long recordsRead = 0;
long bytesRead = 0;
int bookmarkId;
if (!_logFiles.TryGetValue(relativeFilePath, out TContext sourceInfo))
{
sourceInfo = CreateLogSourceInfo(fullPath, 0);
_logFiles.Add(relativeFilePath, sourceInfo);
bookmarkId = this.bookmarkOnBufferFlush ? BookmarkManager.RegisterBookmark(this.GetBookmarkName(fullPath), 0, (pos) => this.SaveBookmark()).Id : 0;
}
else
{
bookmarkId = BookmarkManager.GetBookmarkId(this.GetBookmarkName(fullPath));
}
try
{
using (var fs = new FileStream(fullPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (var sr = CreateStreamReader(fs, _encoding))
{
var records = _recordParser.ParseRecords(sr, sourceInfo);
foreach (var record in records)
{
ILogEnvelope envelope = (ILogEnvelope)record;
if (envelope != null &&
record.Timestamp > (this.InitialPositionTimestamp ?? DateTime.MinValue) &&
envelope.LineNumber > _skipLines)
{
record.BookmarkId = bookmarkId;
_recordSubject.OnNext(record);
recordsRead++;
}
if (!_started)
{
break;
}
}
//Need to grab the position before disposing the reader because disposing the reader will dispose the stream
bytesRead = fs.Position - sourceInfo.Position;
sourceInfo.Position = fs.Position;
sourceInfo.ConsecutiveIOExceptionCount = 0;
}
}
catch (IOException ex)
{
//Add it back to buffer for processing
AddToBuffer(relativeFilePath);
sourceInfo.ConsecutiveIOExceptionCount++;
if (sourceInfo.ConsecutiveIOExceptionCount >= this.NumberOfConsecutiveIOExceptionsToLogError)
{
_logger?.LogError(ex.ToMinimized());
}
}
catch (Exception ex)
{
_logger?.LogError(ex.ToMinimized());
}
return(recordsRead, bytesRead);
}
