/// <inheritdoc />
public bool SetValue(IReadOnlyPropertyDescriptor property, object value)
{
lock (_syncRoot)
{
return(_storage.SetValue(property, value));
}
}
/// <summary>
/// Initializes this text log file.
/// </summary>
/// <param name="taskScheduler"></param>
/// <param name="fileName"></param>
/// <param name="format"></param>
/// <param name="encoding"></param>
internal StreamingTextLogSource(ITaskScheduler taskScheduler,
string fileName,
ILogFileFormat format,
Encoding encoding)
{
_taskScheduler = taskScheduler;
_encoding = encoding;
_listeners = new LogSourceListenerCollection(this);
_sourceDoesNotExist = new SourceDoesNotExist(fileName);
_sourceCannotBeAccessed = new SourceCannotBeAccessed(fileName);
_fileName = fileName ?? throw new ArgumentNullException(nameof(fileName));
_index = new LogBufferList(StreamingTextLogSource.LineOffsetInBytes);
_propertiesBuffer = new PropertiesBufferList();
_propertiesBuffer.SetValue(Core.Properties.Name, _fileName);
_propertiesBuffer.SetValue(Core.Properties.Format, format);
_propertiesBuffer.SetValue(TextProperties.RequiresBuffer, true);
_propertiesBuffer.SetValue(TextProperties.LineCount, 0);
_properties = new ConcurrentPropertiesList(Core.Properties.Minimum);
SynchronizeProperties();
_cancellationTokenSource = new CancellationTokenSource();
_columns = new IColumnDescriptor[] { Core.Columns.Index, StreamingTextLogSource.LineOffsetInBytes, Core.Columns.RawContent };
_pendingReadRequests = new ConcurrentQueue <IReadRequest>();
_fileScanTask = _taskScheduler.StartPeriodic(() => RunFileScan(_cancellationTokenSource.Token));
_fileReadTask = _taskScheduler.StartPeriodic(() => RunFileRead(_cancellationTokenSource.Token));
}
private void UpdateProperties()
{
// First we want to retrieve all properties from the source
_source.GetAllProperties(_propertiesBuffer);
// Then we'll add / overwrite properties
_propertiesBuffer.SetValue(Core.Properties.PercentageProcessed, ComputePercentageProcessed());
_propertiesBuffer.SetValue(Core.Properties.LogEntryCount, _indices.Count);
// And last but not least we'll update our own properties to the new values
// It's important we do this in one go so clients can retrieve all those properties
// in a consistent state
_properties.CopyFrom(_propertiesBuffer);
}
private void UpdateProperties()
{
Size? size = null;
DateTime? lastModified = null;
DateTime? startTimestamp = null;
DateTime? endTimestamp = null;
int maxCharactersPerLine = 0;
Percentage processed = Percentage.HundredPercent;
for (int n = 0; n < _sources.Count; ++n)
{
var source = _sources[n];
source.GetAllProperties(_propertiesBuffer);
var sourceSize = _propertiesBuffer.GetValue(Core.Properties.Size);
if (size == null)
{
size = sourceSize;
}
else if (sourceSize != null)
{
size += sourceSize;
}
var last = _propertiesBuffer.GetValue(Core.Properties.LastModified);
if (last != null && (last > lastModified || lastModified == null))
{
lastModified = last;
}
var start = _propertiesBuffer.GetValue(Core.Properties.StartTimestamp);
if (start != null && (start < startTimestamp || startTimestamp == null))
{
startTimestamp = start;
}
var end = _propertiesBuffer.GetValue(Core.Properties.EndTimestamp);
if (end != null && (end > endTimestamp || endTimestamp == null))
{
endTimestamp = end;
}
maxCharactersPerLine = Math.Max(maxCharactersPerLine, _propertiesBuffer.GetValue(TextProperties.MaxCharactersInLine));
var sourceProcessed = _propertiesBuffer.GetValue(Core.Properties.PercentageProcessed);
processed *= sourceProcessed;
}
_propertiesBuffer.SetValue(Core.Properties.LogEntryCount, _index.Count);
_propertiesBuffer.SetValue(TextProperties.MaxCharactersInLine, maxCharactersPerLine);
_propertiesBuffer.SetValue(Core.Properties.PercentageProcessed, processed);
_propertiesBuffer.SetValue(Core.Properties.LastModified, lastModified);
_propertiesBuffer.SetValue(Core.Properties.Size, size);
_propertiesBuffer.SetValue(Core.Properties.StartTimestamp, startTimestamp);
_propertiesBuffer.SetValue(Core.Properties.EndTimestamp, endTimestamp);
_propertiesBuffer.SetValue(Core.Properties.Duration, endTimestamp - startTimestamp);
// We want to ensure that we modify all properties at once so that users of this log file don't
// see an inconsistent state of properties when they retrieve them.
_properties.CopyFrom(_propertiesBuffer);
}
public FileLogSource(IServiceContainer services, string fileName, TimeSpan maximumWaitTime)
: base(services.Retrieve <ITaskScheduler>())
{
_syncRoot = new object();
_filesystem = services.Retrieve <IFilesystem>();
_services = services;
_fullFilename = Path.IsPathRooted(fileName)
? fileName
: Path.Combine(Directory.GetCurrentDirectory(), fileName);
_maximumWaitTime = maximumWaitTime;
_sourceDoesNotExist = new SourceDoesNotExist(fileName);
_sourceCannotBeAccessed = new SourceCannotBeAccessed(fileName);
var formatMatcher = services.Retrieve <ILogFileFormatMatcher>();
_encodingDetector = new EncodingDetector();
_formatDetector = new FileFormatDetector(formatMatcher);
_buffer = new LogBufferArray(MaximumLineCount, Core.Columns.RawContent);
_pendingSections = new ConcurrentQueue <KeyValuePair <ILogSource, LogSourceModification> >();
_propertiesBuffer = new PropertiesBufferList();
_propertiesBuffer.SetValue(Core.Properties.Name, _fullFilename);
_properties = new ConcurrentPropertiesList();
StartTask();
}
/// <summary>
/// Initializes this object.
/// </summary>
/// <remarks>
/// Plugin authors are deliberately prevented from calling this constructor directly because it's signature may change
/// over time. In order to create an instance of this type, simply call <see cref="ILogSourceFactory.CreateMultiLineLogFile"/>.
/// </remarks>
/// <param name="taskScheduler"></param>
/// <param name="source"></param>
/// <param name="maximumWaitTime"></param>
public MultiLineLogSource(ITaskScheduler taskScheduler, ILogSource source, TimeSpan maximumWaitTime)
: base(taskScheduler)
{
if (source == null)
{
throw new ArgumentNullException(nameof(source));
}
_maximumWaitTime = maximumWaitTime;
_pendingModifications = new ConcurrentQueue <LogSourceModification>();
_syncRoot = new object();
_specialColumns = new HashSet <IColumnDescriptor> {
Core.Columns.LogEntryIndex, Core.Columns.Timestamp, Core.Columns.LogLevel
};
_indices = new List <LogEntryInfo>();
// The log file we were given might offer even more properties than the minimum set and we
// want to expose those as well.
_propertiesBuffer = new PropertiesBufferList(Core.Properties.CombineWithMinimum(source.Properties));
_propertiesBuffer.SetValue(Core.Properties.EmptyReason, null);
_properties = new ConcurrentPropertiesList(Core.Properties.CombineWithMinimum(source.Properties));
_properties.CopyFrom(_propertiesBuffer);
_currentLogEntry = new LogEntryInfo(-1, 0);
_source = source;
_source.AddListener(this, maximumWaitTime, MaximumBatchSize);
StartTask();
}
private void SynchronizeProperties()
{
_source.GetAllProperties(_propertiesBufferView);
var sourceProcessed = _propertiesBuffer.GetValue(Core.Properties.PercentageProcessed);
var sourceCount = _propertiesBuffer.GetValue(Core.Properties.LogEntryCount);
var ownProgress = sourceCount > 0
? Percentage.Of(_count, sourceCount).Clamped()
: Percentage.HundredPercent;
var totalProgress = (sourceProcessed * ownProgress).Clamped();
_propertiesBuffer.SetValue(Core.Properties.PercentageProcessed, totalProgress);
_propertiesBuffer.SetValue(Core.Properties.LogEntryCount, _count);
GetOverwrittenProperties(_propertiesBuffer);
_properties.CopyFrom(_propertiesBuffer);
}
/// <summary>
/// Initializes this text log file.
/// </summary>
/// <param name="filesystem"></param>
/// <param name="taskScheduler"></param>
/// <param name="fileName"></param>
/// <param name="format"></param>
/// <param name="encoding"></param>
internal TextLogSource(IFilesystem filesystem,
ITaskScheduler taskScheduler,
string fileName,
ILogFileFormat format,
Encoding encoding)
: base(taskScheduler)
{
_filesystem = filesystem;
_fileName = fileName ?? throw new ArgumentNullException(nameof(fileName));
_encoding = encoding ?? throw new ArgumentNullException(nameof(encoding));
_entries = new LogBufferList(Core.Columns.RawContent);
_columns = new IColumnDescriptor[]
{
Core.Columns.Index,
Core.Columns.OriginalIndex,
Core.Columns.LogEntryIndex,
Core.Columns.LineNumber,
Core.Columns.OriginalLineNumber,
Core.Columns.OriginalDataSourceName,
Core.Columns.RawContent,
PageBufferedLogSource.RetrievalState
};
_sourceDoesNotExist = new SourceDoesNotExist(fileName);
_sourceCannotBeAccessed = new SourceCannotBeAccessed(fileName);
_localProperties = new PropertiesBufferList(Core.Properties.Minimum);
_localProperties.SetValue(Core.Properties.Name, _fileName);
_localProperties.Add(TextProperties.LineCount);
_localProperties.SetValue(Core.Properties.Format, format);
_localProperties.SetValue(TextProperties.LineCount, 0);
_localProperties.SetValue(TextProperties.RequiresBuffer, false);
_properties = new ConcurrentPropertiesList(Core.Properties.Minimum);
SynchronizePropertiesWithUser();
_syncRoot = new object();
_properties.SetValue(TextProperties.AutoDetectedEncoding, encoding);
Log.DebugFormat("Log File '{0}' is interpreted using {1}", _fileName, _encoding.EncodingName);
StartTask();
}
/// <summary>
/// Initializes this object.
/// </summary>
/// <param name="columns"></param>
/// <param name="properties"></param>
public InMemoryLogSource(IEnumerable <IColumnDescriptor> columns, IReadOnlyDictionary <IReadOnlyPropertyDescriptor, object> properties)
{
if (columns == null)
{
throw new ArgumentNullException(nameof(columns));
}
_syncRoot = new object();
_logBuffer = new LogBufferList(Core.Columns.CombineWithMinimum(columns));
_listeners = new LogSourceListenerCollection(this);
_properties = new PropertiesBufferList(Core.Properties.Minimum);
_properties.SetValue(Core.Properties.Size, Size.Zero);
_properties.SetValue(Core.Properties.PercentageProcessed, Percentage.HundredPercent);
foreach (var pair in properties)
{
_properties.SetValue(pair.Key, pair.Value);
}
}
/// <inheritdoc />
protected override TimeSpan RunOnce(CancellationToken token)
{
bool read = false;
try
{
if (!File.Exists(_fileName))
{
SetDoesNotExist();
}
else
{
var info = new FileInfo(_fileName);
var fileSize = info.Length;
_localProperties.SetValue(Core.Properties.LastModified, info.LastWriteTimeUtc);
_localProperties.SetValue(Core.Properties.Created, info.CreationTimeUtc);
_localProperties.SetValue(Core.Properties.Size, Size.FromBytes(fileSize));
UpdatePercentageProcessed(_lastStreamPosition, fileSize, allow100Percent: true);
SynchronizePropertiesWithUser();
using (var stream = new FileStream(_fileName,
FileMode.Open,
FileAccess.Read,
FileShare.ReadWrite))
{
using (var reader = new StreamReaderEx(stream, _encoding))
{
// We change the error flag explicitly AFTER opening
// the stream because that operation might throw if we're
// not allowed to access the file (in which case a different
// error must be set).
_localProperties.SetValue(Core.Properties.EmptyReason, null);
if (stream.Length >= _lastStreamPosition)
{
stream.Position = _lastStreamPosition;
}
else
{
OnReset(stream, out _numberOfLinesRead, out _lastStreamPosition);
}
int numProcessed = 0;
string currentLine;
while ((currentLine = reader.ReadLine()) != null)
{
token.ThrowIfCancellationRequested();
bool lastLineHadNewline = _lastLineHadNewline;
var trimmedLine = currentLine.TrimNewlineEnd(out _lastLineHadNewline);
var entryCount = _entries.Count;
if (entryCount > 0 && !lastLineHadNewline)
{
// We need to remove the last line and create a new line
// that consists of the entire content.
RemoveLast();
trimmedLine = _untrimmedLastLine + trimmedLine;
_untrimmedLastLine = _untrimmedLastLine + currentLine;
}
else
{
_untrimmedLastLine = currentLine;
++_numberOfLinesRead;
read = true;
}
Add(trimmedLine,
_numberOfLinesRead);
if (++numProcessed % 1000 == 0)
{
// Here's the deal: Since we're processing the file in chunks, we advance the underlying
// stream faster than we're actually consuming lines. This means that it's quite likely
// that at the end of the file, we have moved the stream to the end, but have not quite
// yet processed the underlying buffer from StreamReaderEx. The percentage processed
// should be accurate enough so that if it is at 100%, then no more log entries are added.
// We can only guarantee that when we have processed all lines and therefore we reserve
// setting the percentage to 100% ONLY when we can read no more lines
// (See the SetEndOfSourceReached() call below, outside the loop).
UpdatePercentageProcessed(stream.Position, fileSize, allow100Percent: false);
SynchronizePropertiesWithUser();
}
}
_lastStreamPosition = stream.Position;
_localProperties.SetValue(TextProperties.LineCount, _entries.Count);
_localProperties.SetValue(Core.Properties.LogEntryCount, _entries.Count);
}
}
Listeners.OnRead(_numberOfLinesRead);
SetEndOfSourceReached();
}
}
catch (FileNotFoundException e)
{
SetError(_sourceDoesNotExist);
Log.Debug(e);
}
catch (DirectoryNotFoundException e)
{
SetError(_sourceDoesNotExist);
Log.Debug(e);
}
catch (OperationCanceledException e)
{
Log.Debug(e);
}
catch (UnauthorizedAccessException e)
{
SetError(_sourceCannotBeAccessed);
Log.Debug(e);
}
catch (IOException e)
{
SetError(_sourceCannotBeAccessed);
Log.Debug(e);
}
catch (Exception e)
{
Log.Debug(e);
}
if (read)
{
return(TimeSpan.Zero);
}
return(TimeSpan.FromMilliseconds(100));
}
private void Process(LogSourceSection section)
{
DateTime?startTime = _propertiesBuffer.GetValue(Core.Properties.StartTimestamp);
DateTime?endTime = _propertiesBuffer.GetValue(Core.Properties.EndTimestamp);
int traceCount = _propertiesBuffer.GetValue(Core.Properties.TraceLogEntryCount);
int debugCount = _propertiesBuffer.GetValue(Core.Properties.DebugLogEntryCount);
int infoCount = _propertiesBuffer.GetValue(Core.Properties.InfoLogEntryCount);
int warningCount = _propertiesBuffer.GetValue(Core.Properties.WarningLogEntryCount);
int errorCount = _propertiesBuffer.GetValue(Core.Properties.ErrorLogEntryCount);
int fatalCount = _propertiesBuffer.GetValue(Core.Properties.FatalLogEntryCount);
int otherCount = _propertiesBuffer.GetValue(Core.Properties.OtherLogEntryCount);
_source.GetEntries(section, _buffer, 0, LogSourceQueryOptions.Default);
bool evaluateTimestamp = _requiredColumns.Contains(Core.Columns.Timestamp);
bool evaluateLevel = _requiredColumns.Contains(Core.Columns.LogLevel);
foreach (var entry in _buffer)
{
if (!entry.Index.IsValid)
{
break;
}
if (evaluateTimestamp)
{
var timestamp = entry.Timestamp;
if (timestamp != null)
{
if (startTime == null)
{
startTime = timestamp;
}
else if (timestamp < startTime)
{
startTime = timestamp;
}
if (endTime == null)
{
endTime = timestamp;
}
else if (timestamp > endTime)
{
endTime = timestamp;
}
}
}
if (evaluateLevel)
{
var level = entry.LogLevel;
switch (level)
{
case LevelFlags.Fatal:
++fatalCount;
break;
case LevelFlags.Error:
++errorCount;
break;
case LevelFlags.Warning:
++warningCount;
break;
case LevelFlags.Info:
++infoCount;
break;
case LevelFlags.Debug:
++debugCount;
break;
case LevelFlags.Trace:
++traceCount;
break;
case LevelFlags.Other:
++otherCount;
break;
}
}
}
if (evaluateTimestamp)
{
_propertiesBuffer.SetValue(Core.Properties.StartTimestamp, startTime);
_propertiesBuffer.SetValue(Core.Properties.EndTimestamp, endTime);
_propertiesBuffer.SetValue(Core.Properties.Duration, endTime - startTime);
}
if (evaluateLevel)
{
_propertiesBuffer.SetValue(Core.Properties.TraceLogEntryCount, traceCount);
_propertiesBuffer.SetValue(Core.Properties.DebugLogEntryCount, debugCount);
_propertiesBuffer.SetValue(Core.Properties.InfoLogEntryCount, infoCount);
_propertiesBuffer.SetValue(Core.Properties.WarningLogEntryCount, warningCount);
_propertiesBuffer.SetValue(Core.Properties.ErrorLogEntryCount, errorCount);
_propertiesBuffer.SetValue(Core.Properties.FatalLogEntryCount, fatalCount);
_propertiesBuffer.SetValue(Core.Properties.OtherLogEntryCount, otherCount);
}
}
private void UpdateFormat()
{
try
{
if (DetectFileChange(out var overwrittenEncoding))
{
if (!_filesystem.FileExists(_fullFilename))
{
SetError(_sourceDoesNotExist);
}
else
{
using (var stream = TryOpenRead(_fullFilename, out var error))
{
if (stream == null)
{
SetError(error);
}
else
{
var autoDetectedEncoding = _encodingDetector.TryFindEncoding(stream);
var defaultEncoding = _services.TryRetrieve <Encoding>() ?? Encoding.Default;
var format = _formatDetector.TryDetermineFormat(_fullFilename, stream, overwrittenEncoding ?? autoDetectedEncoding ?? defaultEncoding);
var encoding = PickEncoding(overwrittenEncoding, format, autoDetectedEncoding, defaultEncoding);
var formatChanged = _propertiesBuffer.SetValue(Core.Properties.Format, format);
_propertiesBuffer.SetValue(TextProperties.AutoDetectedEncoding, autoDetectedEncoding);
_propertiesBuffer.SetValue(TextProperties.ByteOrderMark, autoDetectedEncoding != null);
_propertiesBuffer.SetValue(Core.Properties.EmptyReason, error);
var encodingChanged = _propertiesBuffer.SetValue(TextProperties.Encoding, encoding);
var currentLogSources = _logSources;
if (currentLogSources == null || currentLogSources.Count == 0 ||
formatChanged || encodingChanged)
{
Dispose(currentLogSources);
// Depending on the log file we're actually opening, the plugins we have installed, the final log source
// that we expose here could be anything. It could be the raw log source which reads everything line by line,
// but it also could be a plugin's ILogSource implementation which does all kinds of magic.
var newLogSources = CreateAllLogSources(_services, _fullFilename, format, encoding, _maximumWaitTime);
if (!StartListenTo(newLogSources))
{
Dispose(newLogSources);
}
}
}
}
}
}
}
catch (IOException e)
{
Log.DebugFormat("Caught exception while reading '{0}': {1}", _fullFilename, e);
}
catch (Exception e)
{
Log.DebugFormat("Caught unexpected exception while reading '{0}': {1}", _fullFilename, e);
}
}
public EmptyLogSource()
{
_properties.SetValue(Core.Properties.PercentageProcessed, Percentage.HundredPercent);
_properties.SetValue(Core.Properties.Size, Size.Zero);
}
/// <inheritdoc />
public void SetProperty(IPropertyDescriptor property, object value)
{
_properties.SetValue(property, value);
}