public void TestPrefetchAsyncBatch()
{
var pageSize = 100;
var buffer = new PageBufferedLogSource(_taskScheduler, _source.Object, TimeSpan.Zero, pageSize: pageSize);
var destination = new LogBufferArray(4, new IColumnDescriptor[] { Core.Columns.Index, Core.Columns.RawContent });
var queryOptions = new LogSourceQueryOptions(LogSourceQueryMode.FromCache | LogSourceQueryMode.FetchForLater);
buffer.OnLogFileModified(_source.Object, LogSourceModification.Appended(0, 10));
var section1ToQuery = new LogSourceSection(2, 4);
buffer.GetEntries(section1ToQuery, destination, 0, queryOptions);
_source.Verify(x => x.GetEntries(It.IsAny <IReadOnlyList <LogLineIndex> >(), It.IsAny <ILogBuffer>(), It.IsAny <int>(), It.IsAny <LogSourceQueryOptions>()),
Times.Never, "Because we didn't allow data to be retrieved on the calling thread");
var section2ToQuery = new LogSourceSection(7, 3);
buffer.GetEntries(section2ToQuery, destination, 0, queryOptions);
_source.Verify(x => x.GetEntries(It.IsAny <IReadOnlyList <LogLineIndex> >(), It.IsAny <ILogBuffer>(), It.IsAny <int>(), It.IsAny <LogSourceQueryOptions>()),
Times.Never, "Because we didn't allow data to be retrieved on the calling thread");
_taskScheduler.RunOnce();
_source.Verify(x => x.GetEntries(new LogSourceSection(0, pageSize), It.IsAny <ILogBuffer>(), It.IsAny <int>(), It.IsAny <LogSourceQueryOptions>()),
Times.Once, "Because the buffer should avoid reading the same data for the same page multiple times in a row");
}
private void Append(LogSourceSection section)
{
var buffer = new LogBufferArray(section.Count, Core.Columns.Index, Core.Columns.Timestamp, Core.Columns.LogLevel);
_source.GetEntries(section, buffer);
lock (_syncRoot)
{
for (var i = 0; i < section.Count; ++i)
{
var line = buffer[i];
if (_currentLogEntry.EntryIndex.IsInvalid ||
!AppendToCurrentLogEntry(line))
{
_currentLogEntry = _currentLogEntry.NextEntry(line.Index);
}
_indices.Add(_currentLogEntry);
}
}
_currentSourceIndex += section.Count;
_fullSourceSection = new LogSourceSection(0, _currentSourceIndex.Value);
}
public ContiguousReadRequest(LogSourceSection sourceSection,
ILogBuffer destination,
int destinationIndex)
: base(destination, destinationIndex)
{
_sourceSection = sourceSection;
}
public IReadOnlyList <MergedLogLineIndex> Get(LogSourceSection section)
{
var indices = new MergedLogLineIndex[section.Count];
lock (_syncRoot)
{
if (section.Index.Value < _indices.Count)
{
var count = Math.Min((section.Index + section.Count).Value, _indices.Count);
_indices.CopyTo(section.Index.Value, indices, 0, count);
for (var i = 0; i < section.Count - count; ++i)
{
indices[section.LastIndex + i] = MergedLogLineIndex.Invalid;
}
}
else
{
for (var i = 0; i < section.Count; ++i)
{
indices[i] = MergedLogLineIndex.Invalid;
}
}
}
return(indices);
}
private void AddToCache(ILogBuffer source, int destinationIndex, LogSourceSection contiguousSection)
{
lock (_syncRoot)
{
_buffer.TryAdd(contiguousSection, source, destinationIndex);
}
}
protected override void OnSectionAppended(LogSourceSection section, IReadOnlyLogBuffer data, int totalLogEntryCount)
{
lock (_syncRoot)
{
_buffer.AddRange(data, section.Count);
}
}
/// <summary>
/// Processes as many pending modifications as are available, removes existing indices if necessary and
/// establishes the boundaries of the source log file.
/// </summary>
/// <param name="token"></param>
/// <returns></returns>
private bool ProcessModifications(CancellationToken token)
{
bool performedWork = false;
while (_pendingModifications.TryDequeue(out var modification) && !token.IsCancellationRequested)
{
if (modification.IsReset())
{
Clear();
_lastLogBuffer.Clear();
_currentSourceIndex = 0;
}
else if (modification.IsRemoved(out var removedSection))
{
LogLineIndex startIndex = removedSection.Index;
_fullSourceSection = new LogSourceSection(0, (int)startIndex);
if (_currentSourceIndex > _fullSourceSection.LastIndex)
{
_currentSourceIndex = (int)removedSection.Index;
}
RemoveFrom(_currentSourceIndex);
RemoveLinesFrom(_lastLogBuffer, _currentSourceIndex);
}
else if (modification.IsAppended(out var appendedSection))
{
_fullSourceSection = LogSourceSection.MinimumBoundingLine(_fullSourceSection, appendedSection);
}
performedWork = true;
}
return(performedWork);
}
/// <summary>
/// Adds data from the given source to this cache.
/// </summary>
/// <param name="destinationSection">The destination indices of the log entries from the given source</param>
/// <param name="source">The source from which to copy data to this buffer</param>
/// <param name="sourceIndex">The index into the given source from which onwards log entries may be added to this cache.</param>
public void TryAdd(LogSourceSection destinationSection, IReadOnlyLogBuffer source, int sourceIndex)
{
if (!CanCache(source))
{
return;
}
// We want to make sure to only read that data, which is actually covered by the log source.
// This is because one thread might resize this buffer to a smaller size and then another thread
// might try to add data which has been previously read, but is now no longer supposed to be part of the source.
var sourceSectionEndIndex = Math.Min((int)(destinationSection.Index + destinationSection.Count), _sourceCount);
for (LogLineIndex i = destinationSection.Index; i < sourceSectionEndIndex;)
{
var pageIndex = GetPageIndex(i);
var page = GetOrCreatePageByIndex(pageIndex);
var remainingPageCount = (pageIndex + 1) * _pageSize - i;
var count = Math.Min(remainingPageCount, sourceSectionEndIndex - i);
var sourceStartIndex = (i - destinationSection.Index) + sourceIndex;
page.Add(sourceStartIndex, count, source, i);
// We want the next copy operation to start at the beginning of the next page (it's a contiguous write after all)
i += remainingPageCount;
}
}
public void TestGetCount([Values(0, 1)] int count)
{
var section = new LogSourceSection(9001, count);
section.Count.Should().Be(count);
((IReadOnlyList <LogLineIndex>)section).Count.Should().Be(count);
}
public void TestEnumerate([Range(0, 5)] int startIndex,
[Range(0, 5)] int count)
{
var section = new LogSourceSection(startIndex, count);
var expected = Enumerable.Range(startIndex, count).Select(x => (LogLineIndex)x).ToArray();
section.Should().Equal(expected);
}
/// <summary>
/// Processes all newly arrived log entries.
/// </summary>
/// <param name="token"></param>
private void ProcessNewLogEntries(CancellationToken token)
{
if (!_fullSourceSection.IsEndOfSection(_currentSourceIndex))
{
int remaining = _fullSourceSection.Index + _fullSourceSection.Count - _currentSourceIndex;
int nextCount = Math.Min(remaining, BatchSize);
var nextSection = new LogSourceSection(_currentSourceIndex, nextCount);
_source.GetEntries(nextSection, _array);
for (int i = 0; i < nextCount; ++i)
{
if (token.IsCancellationRequested)
{
break;
}
var logEntry = _array[i];
if (Log.IsDebugEnabled)
{
Log.DebugFormat("Processing: LineIndex={0}, OriginalLineIndex={1}, LogEntryIndex={2}, Message={3}",
logEntry.Index,
logEntry.OriginalIndex,
logEntry.LogEntryIndex,
logEntry.RawContent);
}
if (_lastLogBuffer.Count == 0 || _lastLogBuffer[0].LogEntryIndex == logEntry.LogEntryIndex)
{
TryAddLogLine(logEntry);
}
else if (logEntry.LogEntryIndex != _lastLogBuffer[0].LogEntryIndex)
{
TryAddLogEntry(_lastLogBuffer);
_lastLogBuffer.Clear();
TryAddLogLine(logEntry);
}
}
_currentSourceIndex += nextCount;
}
// Now that we've processes all newly added log entries, we can check if we're at the end just yet...
if (_fullSourceSection.IsEndOfSection(_currentSourceIndex))
{
TryAddLogEntry(_lastLogBuffer);
UpdateProperties(); //< we need to update our own properties after we've added the last entry, but before we notify listeners...
Listeners.OnRead(_indices.Count);
if (_properties.GetValue(Core.Properties.PercentageProcessed) == Percentage.HundredPercent)
{
Listeners.Flush();
}
}
else
{
UpdateProperties();
}
}
public void TestGetByIndex()
{
var section = new LogSourceSection(42, 10);
for (int i = 0; i < 10; ++i)
{
section[i].Should().Be(new LogLineIndex(42 + i));
}
}
public void TestGetColumn2()
{
var section = new LogSourceSection(42, 100);
var buffer = new string[100];
var logFile = new LogSourceProxy(_taskScheduler, TimeSpan.Zero);
logFile.GetColumn(section, Core.Columns.RawContent, buffer);
buffer.Should().OnlyContain(x => ReferenceEquals(x, null));
}
/// <inheritdoc />
public void GetEntries(LogSourceSection sourceSection, ILogBuffer destination, int destinationIndex, LogSourceQueryOptions queryOptions)
{
lock (_syncRoot)
{
foreach (var column in destination.Columns)
{
destination.CopyFrom(column, destinationIndex, _logBuffer, new Int32View(sourceSection));
}
}
}
private void AppendMatches(LogSourceSection section)
{
try
{
LogBufferArray lines;
lock (_syncRoot)
{
lines = _logLinesArray;
if (lines == null)
{
return;
}
}
// We've instructed the logfile to give us exactly up to
// _logLinesBuffer.Length amount of entries in the ctor, hence the following
// is correct:
_logSource.GetEntries(section, lines);
bool added = false;
for (int i = 0; i < section.Count; ++i)
{
var line = lines[i];
_filter.Match(line, _matchesBuffer);
if (_matchesBuffer.Count > 0)
{
lock (_syncRoot)
{
foreach (LogLineMatch logLineMatch in _matchesBuffer)
{
var match = new LogMatch(line.Index, logLineMatch);
_matches.Add(match);
}
}
_matchesBuffer.Clear();
added = true;
}
}
if (added)
{
_listeners.EmitSearchChanged(_matches);
}
}
catch (IndexOutOfRangeException e)
{
// This exception is usually thrown when we access a portion of the
// log file that has already been reset. This means that a reset event is
// either pending or soon to be. So not doing anything else to handle
// this exception is fine.
Log.DebugFormat("Caught exception while searching log file: {0}", e);
}
}
private void Clear()
{
_fullSourceSection = new LogSourceSection(0, 0);
_currentSourceIndex = 0;
_currentLogEntry = new LogEntryInfo(-1, 0);
lock (_syncRoot)
{
_indices.Clear();
}
Listeners.OnRead(-1);
}
private void Clear()
{
_fullSourceSection = new LogSourceSection();
lock (_indices)
{
_indices.Clear();
_logEntryIndices.Clear();
_currentLogEntryIndex = 0;
}
Listeners.OnRead(-1);
}
public Page(int index, int pageSize, IReadOnlyList <IColumnDescriptor> columns, IReadOnlyList <IColumnDescriptor> copiedColumns)
{
_index = index;
_pageSize = pageSize;
_copiedColumns = copiedColumns;
_section = new LogSourceSection(index * pageSize, pageSize);
_buffer = new LogBufferArray(pageSize, columns);
_lastAccessTime = DateTime.MinValue;
_numReads = 0;
_buffer.Fill(PageBufferedLogSource.RetrievalState, RetrievalState.NotInSource, 0, _pageSize);
}
private void Remove(LogSourceSection sectionToRemove)
{
var firstRemovedIndex = LogLineIndex.Min(_fullSourceSection.LastIndex, sectionToRemove.Index);
var lastRemovedIndex = LogLineIndex.Min(_fullSourceSection.LastIndex, sectionToRemove.LastIndex);
var removedCount = lastRemovedIndex - firstRemovedIndex + 1;
var previousSourceIndex = _currentSourceIndex;
_fullSourceSection = new LogSourceSection(0, (int)firstRemovedIndex);
if (_fullSourceSection.Count > 0)
{
// It's possible (likely) that we've received an invalidation for a region of the source
// that we've already processed (i.e. created indices for). If that's the case, then we need
// to rewind the index. Otherwise nothing needs to be done...
var newIndex = _fullSourceSection.LastIndex + 1;
if (newIndex < _currentSourceIndex)
{
_currentSourceIndex = newIndex;
}
}
else
{
_currentSourceIndex = 0;
}
lock (_syncRoot)
{
var toRemove = _indices.Count - lastRemovedIndex;
if (toRemove > 0)
{
_indices.RemoveRange((int)firstRemovedIndex, toRemove);
_currentLogEntry = new LogEntryInfo(firstRemovedIndex - 1, 0);
}
if (previousSourceIndex != _currentSourceIndex)
{
_indices.RemoveRange((int)_currentSourceIndex, _indices.Count - _currentSourceIndex);
}
}
if (_indices.Count != _currentSourceIndex)
{
Log.ErrorFormat("Inconsistency detected: We have {0} indices for {1} lines", _indices.Count,
_currentSourceIndex);
}
Listeners.Remove((int)firstRemovedIndex, removedCount);
if (_fullSourceSection.Count > firstRemovedIndex)
{
_fullSourceSection = new LogSourceSection(0, firstRemovedIndex.Value);
}
}
private void UpdateMaxWidth(LogSourceSection section, ILogSource logSource)
{
logSource.GetEntries(section, _buffer);
for (int i = 0; i < section.Count; ++i)
{
var rawContent = _buffer[i].RawContent;
if (rawContent == null)
{
break;
}
_maxCharactersInLine = Math.Max(_maxCharactersInLine, rawContent.Length);
}
}
public void Remove(int firstIndex, int count)
{
int lastIndex = Math.Min(firstIndex + count, _lastNumberOfLines);
int invalidateCount = lastIndex - firstIndex;
// When the start index of the invalidation is greater than the last reported index
// then this means that our listeners haven't even gotten the change yet and thus
// they don't need to be notified of the invalidation either.
if (invalidateCount > 0)
{
var section = new LogSourceSection(firstIndex, invalidateCount);
_listener.OnLogFileModified(_logSource, LogSourceModification.Removed(section));
_lastNumberOfLines = firstIndex;
}
}
private void RemoveSection(LogSourceSection section)
{
_count = (int)section.Index;
try
{
OnSectionRemoved(_count);
}
catch (Exception e)
{
Log.WarnFormat("Caught unexpected exception: {0}", e);
}
SynchronizeProperties();
Listeners.Remove((int)section.Index, section.Count);
}
public void UpdateDataSources(IDataSource dataSource, LogSourceSection visibleSection, double yOffset)
{
_dataSource = dataSource;
_visibleSection = visibleSection;
_yOffset = yOffset;
var multi = UnpackMultiDataSource(dataSource);
if (multi != null)
{
var dataSources = multi.OriginalSources;
var displayMode = DisplayMode;
var maximumCharacterCount = GetMaximumCharacterCount(displayMode, dataSources);
var texts = new FormattedText[dataSources.Count];
for (int i = 0; i < dataSources.Count; ++i)
{
var originalDataSource = dataSources[i];
texts[i] = CreateFormattedText(displayMode, GetDataSourceName(displayMode, originalDataSource));
}
_dataSourcesPerLogLine.Clear();
var entries = new LogBufferArray(visibleSection.Count, new[] { Columns.SourceId });
multi.FilteredLogSource.GetEntries(visibleSection, entries);
foreach (var logEntry in entries)
{
var dataSourceId = (int)logEntry.GetValue(Columns.SourceId);
if (dataSourceId >= 0 && dataSourceId < texts.Length)
{
var text = texts[dataSourceId];
_dataSourcesPerLogLine.Add(text);
}
else
{
_dataSourcesPerLogLine.Add(item: null);
}
}
Width = _textSettings.EstimateWidthUpperLimit(maximumCharacterCount);
}
else
{
_dataSourcesPerLogLine.Clear();
Width = 0;
}
InvalidateVisual();
}
public void TestGetColumn1()
{
var section = new LogSourceSection(42, 100);
var buffer = new string[142];
var logFile = new LogSourceProxy(_taskScheduler, TimeSpan.Zero, _logFile.Object);
var destinationIndex = 42;
var queryOptions = new LogSourceQueryOptions(LogSourceQueryMode.FromCache);
logFile.GetColumn(section, Core.Columns.RawContent, buffer, destinationIndex, queryOptions);
_logFile.Verify(x => x.GetColumn(It.Is <LogSourceSection>(y => y == section),
Core.Columns.RawContent,
buffer,
destinationIndex,
queryOptions),
Times.Once);
}
private void AppendSection(LogSourceSection section)
{
_count = (int)(section.Index + section.Count);
try
{
_source.GetEntries(section, _fetchBuffer);
OnSectionAppended(section, _fetchBuffer, _count);
}
catch (Exception e)
{
Log.WarnFormat("Caught unexpected exception: {0}", e);
}
SynchronizeProperties();
Listeners.OnRead(_count);
}
/// <inheritdoc />
public void GetColumn <T>(LogSourceSection sourceSection, IColumnDescriptor <T> column, T[] destination, int destinationIndex, LogSourceQueryOptions queryOptions)
{
if (column == null)
{
throw new ArgumentNullException(nameof(column));
}
if (destination == null)
{
throw new ArgumentNullException(nameof(destination));
}
if (destinationIndex < 0)
{
throw new ArgumentOutOfRangeException(nameof(destinationIndex));
}
_logBuffer.CopyTo(column, (int)sourceSection.Index, destination, destinationIndex, sourceSection.Count);
}
private void Report(int numberOfLinesRead, DateTime now)
{
// We may never report all lines in one go if the listener specified
// that he only wants to receive batches of N.
// Therefore we invoke the listener multiple times.
int count;
while ((count = numberOfLinesRead - _lastNumberOfLines) > 0)
{
count = Math.Min(count, _maximumCount);
var section = new LogSourceSection(_lastNumberOfLines, count);
_listener.OnLogFileModified(_logSource, LogSourceModification.Appended(section));
_lastNumberOfLines += count;
_lastReportedTime = now;
}
}
public void TestGetEntries1()
{
_logFile.Setup(x => x.GetEntries(It.IsAny <LogSourceSection>(), It.IsAny <ILogBuffer>(), It.IsAny <int>(), It.IsAny <LogSourceQueryOptions>())).Throws <SystemException>();
var section = new LogSourceSection(42, 100);
var buffer = new Mock <ILogBuffer>().Object;
var destinationIndex = 9001;
var queryOptions = new LogSourceQueryOptions(LogSourceQueryMode.FromCache);
new Action(() => _proxy.GetEntries(section, buffer, destinationIndex, queryOptions)).Should().NotThrow();
_logFile.Verify(x => x.GetEntries(It.Is <LogSourceSection>(y => y == section),
It.Is <ILogBuffer>(y => ReferenceEquals(y, buffer)),
destinationIndex,
queryOptions),
Times.Once);
}
public void TestGetColumn1()
{
_logFile.Setup(x => x.GetColumn(It.IsAny <LogSourceSection>(), It.IsAny <IColumnDescriptor <string> >(), It.IsAny <string[]>(), It.IsAny <int>(), It.IsAny <LogSourceQueryOptions>())).Throws <SystemException>();
var section = new LogSourceSection(42, 100);
var buffer = new string[9101];
var destinationIndex = 9001;
var queryOptions = new LogSourceQueryOptions(LogSourceQueryMode.FromCache);
new Action(() => _proxy.GetColumn(section, Columns.RawContent, buffer, destinationIndex, queryOptions)).Should().NotThrow();
_logFile.Verify(x => x.GetColumn(It.Is <LogSourceSection>(y => y == section),
Columns.RawContent,
buffer,
destinationIndex,
queryOptions),
Times.Once);
}
private bool TryGetEntriesContiguous(LogSourceSection sourceSection, ILogBuffer destination, int destinationIndex, out IReadOnlyList <LogSourceSection> accessedPageBoundaries)
{
bool fullyRead = true;
var sourceSectionEndIndex = Math.Min((int)(sourceSection.Index + sourceSection.Count), _sourceCount);
var numEntriesRead = 0;
var tmpAccessedPageBoundaries = new List <LogSourceSection>();
for (LogLineIndex i = sourceSection.Index; i < sourceSectionEndIndex;)
{
var pageIndex = GetPageIndex(i);
var remainingPageCount = (pageIndex + 1) * _pageSize - i;
var count = Math.Min(remainingPageCount, sourceSectionEndIndex - i);
var page = TryGetPage(pageIndex);
if (page != null)
{
fullyRead &= page.TryRead(i, count, destination, destinationIndex + numEntriesRead, fullyRead);
tmpAccessedPageBoundaries.Add(page.Section);
}
else
{
destination.FillDefault(destinationIndex + numEntriesRead, count);
if (destination.Contains(PageBufferedLogSource.RetrievalState))
{
destination.Fill(PageBufferedLogSource.RetrievalState, RetrievalState.NotCached, destinationIndex + numEntriesRead, count);
}
fullyRead = false;
tmpAccessedPageBoundaries.Add(GetSectionForPage(pageIndex));
}
numEntriesRead += count;
i += count;
}
if (numEntriesRead < sourceSection.Count)
{
var start = destinationIndex + numEntriesRead;
destination.FillDefault(start, sourceSection.Count - numEntriesRead);
fullyRead = false;
}
accessedPageBoundaries = tmpAccessedPageBoundaries;
return(fullyRead);
}
