/// <summary>
/// Initializes this text log file.
/// </summary>
/// <param name="taskScheduler"></param>
/// <param name="fileName"></param>
/// <param name="format"></param>
/// <param name="encoding"></param>
internal StreamingTextLogSource(ITaskScheduler taskScheduler,
string fileName,
ILogFileFormat format,
Encoding encoding)
{
_taskScheduler = taskScheduler;
_encoding = encoding;
_listeners = new LogSourceListenerCollection(this);
_sourceDoesNotExist = new SourceDoesNotExist(fileName);
_sourceCannotBeAccessed = new SourceCannotBeAccessed(fileName);
_fileName = fileName ?? throw new ArgumentNullException(nameof(fileName));
_index = new LogBufferList(StreamingTextLogSource.LineOffsetInBytes);
_propertiesBuffer = new PropertiesBufferList();
_propertiesBuffer.SetValue(Core.Properties.Name, _fileName);
_propertiesBuffer.SetValue(Core.Properties.Format, format);
_propertiesBuffer.SetValue(TextProperties.RequiresBuffer, true);
_propertiesBuffer.SetValue(TextProperties.LineCount, 0);
_properties = new ConcurrentPropertiesList(Core.Properties.Minimum);
SynchronizeProperties();
_cancellationTokenSource = new CancellationTokenSource();
_columns = new IColumnDescriptor[] { Core.Columns.Index, StreamingTextLogSource.LineOffsetInBytes, Core.Columns.RawContent };
_pendingReadRequests = new ConcurrentQueue <IReadRequest>();
_fileScanTask = _taskScheduler.StartPeriodic(() => RunFileScan(_cancellationTokenSource.Token));
_fileReadTask = _taskScheduler.StartPeriodic(() => RunFileRead(_cancellationTokenSource.Token));
}
public LogEntryCache(params IColumnDescriptor[] columns)
{
_maxSize = 10000;
_vacuumBuffer1 = new KeyValuePair <int, DateTime> [_maxSize];
_vacuumBuffer2 = new int[_maxSize];
_buffer = new LogBufferList(columns.Concat(new[] { LastAccessed }));
_bufferIndexByLineIndex = new Dictionary <LogLineIndex, int>();
}
protected override int ReadLines(LogBufferList index, StreamReader reader)
{
var lines = new string[_sourceIndices.Length];
var lineOffsets = GetLineOffsets(index);
var linesRead = ReadData(reader, lineOffsets, lines);
CopyDataToDestination(_sourceIndices, lines, linesRead);
return(linesRead);
}
private long[] GetLineOffsets(LogBufferList index)
{
// We need to find out where to reposition the file stream.
// For a non-contiguous read-request, we'll have to look up
// the offset for every requested line since it might jump all over the place.
var indices = new long[_sourceIndices.Length];
lock (index)
{
index.CopyTo(StreamingTextLogSource.LineOffsetInBytes, _sourceIndices, indices, 0);
}
return(indices);
}
private long GetFirstLineOffset(LogBufferList index)
{
// We need to find out where to reposition the file stream.
// This is super easy for contiguous read-requests: All we have to do is
// to find the offset of the first requested line and then read from there...
var indices = new long[1];
lock (index)
{
index.CopyTo(StreamingTextLogSource.LineOffsetInBytes, new LogSourceSection(_sourceSection.Index, 1), indices, 0);
}
return(indices[0]);
}
public FullyBufferedLogSource(ITaskScheduler taskScheduler,
ILogSource source,
IReadOnlyList <IColumnDescriptor> bufferedColumns,
TimeSpan maximumWaitTime)
: base(taskScheduler,
source,
bufferedColumns,
new IReadOnlyPropertyDescriptor[0],
maximumWaitTime)
{
_buffer = new LogBufferList(bufferedColumns);
_syncRoot = new object();
StartTask();
}
private static void RemoveLinesFrom(LogBufferList lastLogBuffer, int currentSourceIndex)
{
while (lastLogBuffer.Count > 0)
{
int i = lastLogBuffer.Count - 1;
var logEntry = lastLogBuffer[i];
if (logEntry.Index >= currentSourceIndex)
{
lastLogBuffer.RemoveAt(i);
}
else
{
break;
}
}
}
public void Serve(LogBufferList index, StreamReader reader)
{
try
{
_startTime = DateTime.Now;
var linesRead = ReadLines(index, reader);
Complete(linesRead);
}
catch (Exception e)
{
if (!_taskSource.TrySetException(e))
{
Log.Warn("Unable to fail pending task, here's the exception intended for the task: {0}", e);
}
}
}
protected override int ReadLines(LogBufferList index, StreamReader reader)
{
var lines = new string[_sourceSection.Count];
var firstLineOffset = GetFirstLineOffset(index);
if (firstLineOffset < 0) //< We don't know the offset for that index
{
return(0);
}
var linesRead = ReadData(reader, firstLineOffset, lines);
// TODO: Should we try to avoid this unnecessary copy?
CopyDataToDestination(((IReadOnlyList <LogLineIndex>)_sourceSection).ToArray(), lines, linesRead);
return(linesRead);
}
/// <summary>
/// Initializes this text log file.
/// </summary>
/// <param name="filesystem"></param>
/// <param name="taskScheduler"></param>
/// <param name="fileName"></param>
/// <param name="format"></param>
/// <param name="encoding"></param>
internal TextLogSource(IFilesystem filesystem,
ITaskScheduler taskScheduler,
string fileName,
ILogFileFormat format,
Encoding encoding)
: base(taskScheduler)
{
_filesystem = filesystem;
_fileName = fileName ?? throw new ArgumentNullException(nameof(fileName));
_encoding = encoding ?? throw new ArgumentNullException(nameof(encoding));
_entries = new LogBufferList(Core.Columns.RawContent);
_columns = new IColumnDescriptor[]
{
Core.Columns.Index,
Core.Columns.OriginalIndex,
Core.Columns.LogEntryIndex,
Core.Columns.LineNumber,
Core.Columns.OriginalLineNumber,
Core.Columns.OriginalDataSourceName,
Core.Columns.RawContent,
PageBufferedLogSource.RetrievalState
};
_sourceDoesNotExist = new SourceDoesNotExist(fileName);
_sourceCannotBeAccessed = new SourceCannotBeAccessed(fileName);
_localProperties = new PropertiesBufferList(Core.Properties.Minimum);
_localProperties.SetValue(Core.Properties.Name, _fileName);
_localProperties.Add(TextProperties.LineCount);
_localProperties.SetValue(Core.Properties.Format, format);
_localProperties.SetValue(TextProperties.LineCount, 0);
_localProperties.SetValue(TextProperties.RequiresBuffer, false);
_properties = new ConcurrentPropertiesList(Core.Properties.Minimum);
SynchronizePropertiesWithUser();
_syncRoot = new object();
_properties.SetValue(TextProperties.AutoDetectedEncoding, encoding);
Log.DebugFormat("Log File '{0}' is interpreted using {1}", _fileName, _encoding.EncodingName);
StartTask();
}
/// <summary>
/// Initializes this object.
/// </summary>
/// <param name="columns"></param>
/// <param name="properties"></param>
public InMemoryLogSource(IEnumerable <IColumnDescriptor> columns, IReadOnlyDictionary <IReadOnlyPropertyDescriptor, object> properties)
{
if (columns == null)
{
throw new ArgumentNullException(nameof(columns));
}
_syncRoot = new object();
_logBuffer = new LogBufferList(Core.Columns.CombineWithMinimum(columns));
_listeners = new LogSourceListenerCollection(this);
_properties = new PropertiesBufferList(Core.Properties.Minimum);
_properties.SetValue(Core.Properties.Size, Size.Zero);
_properties.SetValue(Core.Properties.PercentageProcessed, Percentage.HundredPercent);
foreach (var pair in properties)
{
_properties.SetValue(pair.Key, pair.Value);
}
}
/// <summary>
/// Initializes this object.
/// </summary>
/// <param name="scheduler"></param>
/// <param name="maximumWaitTime"></param>
/// <param name="source"></param>
/// <param name="logLineFilter"></param>
/// <param name="logEntryFilter"></param>
public FilteredLogSource(ITaskScheduler scheduler,
TimeSpan maximumWaitTime,
ILogSource source,
ILogLineFilter logLineFilter,
ILogEntryFilter logEntryFilter)
: base(scheduler)
{
_source = source ?? throw new ArgumentNullException(nameof(source));
_properties = new ConcurrentPropertiesList(source.Properties);
_propertiesBuffer = new PropertiesBufferList(); //< Will be used as temporary storage to hold the properties from the source
_logLineFilter = logLineFilter ?? new NoFilter();
_logEntryFilter = logEntryFilter ?? new NoFilter();
_pendingModifications = new ConcurrentQueue <LogSourceModification>();
_indices = new List <int>();
_logEntryIndices = new Dictionary <int, int>();
_array = new LogBufferArray(BatchSize, Core.Columns.Minimum);
_lastLogBuffer = new LogBufferList(Core.Columns.Minimum);
_maximumWaitTime = maximumWaitTime;
_source.AddListener(this, maximumWaitTime, BatchSize);
StartTask();
}
protected abstract int ReadLines(LogBufferList index, StreamReader reader);
public LogEntryAccessor(LogBufferList list, int index)
{
_list = list;
_index = index;
}
