public void GetEntries(IReadOnlyList <LogLineIndex> sourceIndices,
ILogBuffer destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
var source = _source;
if (source == null)
{
destination.FillDefault(destinationIndex, sourceIndices.Count);
return;
}
source.GetEntries(sourceIndices, destination.Except(MaxAdornedColumns), destinationIndex, queryOptions);
var augmentedColumns = FindAugmentedColumns(destination);
if (augmentedColumns.Count == 0)
{
return;
}
if (destinationIndex != 0)
{
throw new NotImplementedException();
}
foreach (var column in augmentedColumns)
{
destination.CopyFrom(column, this, sourceIndices, queryOptions);
}
}
public override void GetEntries(IReadOnlyList <LogLineIndex> sourceIndices,
ILogBuffer destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
_source.GetEntries(sourceIndices, destination, destinationIndex, queryOptions);
}
private void GetDeltaTime(IReadOnlyList <LogLineIndex> indices,
TimeSpan?[] destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
// The easiest way to serve random access to this column is to simply retrieve
// the timestamp for every requested index as well as for the preceding index.
var actualIndices = new LogLineIndex[indices.Count * 2];
for (int i = 0; i < indices.Count; ++i)
{
var index = indices[i];
actualIndices[i * 2 + 0] = index - 1;
actualIndices[i * 2 + 1] = index;
}
var timestamps = _source.GetColumn(actualIndices, Core.Columns.Timestamp, queryOptions);
for (int i = 0; i < indices.Count; ++i)
{
var previousTimestamp = timestamps[i * 2 + 0];
var currentTimestamp = timestamps[i * 2 + 1];
destination[destinationIndex + i] = currentTimestamp - previousTimestamp;
}
}
public void TestPrefetchAsyncBatch()
{
var pageSize = 100;
var buffer = new PageBufferedLogSource(_taskScheduler, _source.Object, TimeSpan.Zero, pageSize: pageSize);
var destination = new LogBufferArray(4, new IColumnDescriptor[] { Core.Columns.Index, Core.Columns.RawContent });
var queryOptions = new LogSourceQueryOptions(LogSourceQueryMode.FromCache | LogSourceQueryMode.FetchForLater);
buffer.OnLogFileModified(_source.Object, LogSourceModification.Appended(0, 10));
var section1ToQuery = new LogSourceSection(2, 4);
buffer.GetEntries(section1ToQuery, destination, 0, queryOptions);
_source.Verify(x => x.GetEntries(It.IsAny <IReadOnlyList <LogLineIndex> >(), It.IsAny <ILogBuffer>(), It.IsAny <int>(), It.IsAny <LogSourceQueryOptions>()),
Times.Never, "Because we didn't allow data to be retrieved on the calling thread");
var section2ToQuery = new LogSourceSection(7, 3);
buffer.GetEntries(section2ToQuery, destination, 0, queryOptions);
_source.Verify(x => x.GetEntries(It.IsAny <IReadOnlyList <LogLineIndex> >(), It.IsAny <ILogBuffer>(), It.IsAny <int>(), It.IsAny <LogSourceQueryOptions>()),
Times.Never, "Because we didn't allow data to be retrieved on the calling thread");
_taskScheduler.RunOnce();
_source.Verify(x => x.GetEntries(new LogSourceSection(0, pageSize), It.IsAny <ILogBuffer>(), It.IsAny <int>(), It.IsAny <LogSourceQueryOptions>()),
Times.Once, "Because the buffer should avoid reading the same data for the same page multiple times in a row");
}
public override void GetColumn <T>(IReadOnlyList <LogLineIndex> sourceIndices,
IColumnDescriptor <T> column,
T[] destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
var source = _finalLogSource;
if (source != null)
{
source.GetColumn(sourceIndices, column, destination, destinationIndex, queryOptions);
}
else
{
if (sourceIndices == null)
{
throw new ArgumentNullException(nameof(sourceIndices));
}
if (column == null)
{
throw new ArgumentNullException(nameof(column));
}
if (destination == null)
{
throw new ArgumentNullException(nameof(destination));
}
if (destinationIndex < 0)
{
throw new ArgumentOutOfRangeException(nameof(destinationIndex));
}
destination.Fill(column.DefaultValue, destinationIndex, sourceIndices.Count);
}
}
public void GetEntries(IReadOnlyList <LogLineIndex> sourceIndices,
ILogBuffer destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
if (TryReadFromCache(sourceIndices, destination, destinationIndex, queryOptions, out var accessedPageBoundaries))
{
return;
}
if ((queryOptions.QueryMode & LogSourceQueryMode.FetchForLater) == LogSourceQueryMode.FetchForLater)
{
FetchForLater(accessedPageBoundaries);
}
// For some clients (GUI) it is permissible to serve partial requests and to default out the rest.
// The ui will try again at a later date until it gets what it needs.
if ((queryOptions.QueryMode & LogSourceQueryMode.FromSource) == 0)
{
return;
}
// Whelp, we gotta fetch from the source instead.
// Note: We don't lock this part because this would block any other thread currently
// trying to read data from cache, etc.. We only block when we need to and only
// for a short amount of time!
_source.GetEntries(sourceIndices, destination, destinationIndex, queryOptions);
// However now that we got some data, we could try to add it to our cache
AddToCache(sourceIndices, destination, destinationIndex, queryOptions);
}
/// <inheritdoc />
public void GetColumn <T>(IReadOnlyList <LogLineIndex> sourceIndices,
IColumnDescriptor <T> column,
T[] destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
if (sourceIndices == null)
{
throw new ArgumentNullException(nameof(sourceIndices));
}
if (column == null)
{
throw new ArgumentNullException(nameof(column));
}
if (destination == null)
{
throw new ArgumentNullException(nameof(destination));
}
if (destinationIndex < 0)
{
throw new ArgumentOutOfRangeException(nameof(destinationIndex));
}
if (destinationIndex + sourceIndices.Count > destination.Length)
{
throw new ArgumentException("The given buffer must have an equal or greater length than destinationIndex+length");
}
GetEntries(sourceIndices,
new SingleColumnLogBufferView <T>(column, destination, destinationIndex, sourceIndices.Count),
0, queryOptions);
}
public void GetColumn <T>(IReadOnlyList <LogLineIndex> sourceIndices,
IColumnDescriptor <T> column,
T[] destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
if (ReferenceEquals(column, Core.Columns.RawContent))
{
var view = new SingleColumnLogBufferView <T>(column, destination, destinationIndex, sourceIndices.Count);
ReadRawData(sourceIndices, view, 0, queryOptions);
}
else if (ReferenceEquals(column, Core.Columns.Index))
{
GetIndices(sourceIndices, (LogLineIndex[])(object)destination, destinationIndex);
}
else if (ReferenceEquals(column, StreamingTextLogSource.LineOffsetInBytes))
{
lock (_index)
{
_index.CopyTo(column, sourceIndices, destination, destinationIndex);
}
}
else
{
throw new NoSuchColumnException(column);
}
}
/// <summary>
/// Retrieves a list of cells for a given column from this log file.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="logSource"></param>
/// <param name="sourceIndices"></param>
/// <param name="column"></param>
/// <param name="destination"></param>
/// <param name="queryOptions">Configures how the data is to be retrieved</param>
public static void GetColumn <T>(this ILogSource logSource,
IReadOnlyList <LogLineIndex> sourceIndices,
IColumnDescriptor <T> column,
T[] destination,
LogSourceQueryOptions queryOptions)
{
logSource.GetColumn(sourceIndices, column, destination, 0, queryOptions);
}
/// <inheritdoc />
public void CopyFrom(IColumnDescriptor column,
int destinationIndex,
ILogSource source,
IReadOnlyList <LogLineIndex> sourceIndices,
LogSourceQueryOptions queryOptions)
{
throw new System.NotImplementedException();
}
/// <inheritdoc />
public void GetColumn <T>(IReadOnlyList <LogLineIndex> sourceIndices,
IColumnDescriptor <T> column,
T[] destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
_buffer.GetColumn(sourceIndices, column, destination, destinationIndex, queryOptions);
}
public void GetColumn <T>(IReadOnlyList <LogLineIndex> sourceIndices,
IColumnDescriptor <T> column,
T[] destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
if (sourceIndices == null)
{
throw new ArgumentNullException(nameof(sourceIndices));
}
if (column == null)
{
throw new ArgumentNullException(nameof(column));
}
if (destination == null)
{
throw new ArgumentNullException(nameof(destination));
}
if (destinationIndex < 0)
{
throw new ArgumentOutOfRangeException(nameof(destinationIndex));
}
if (destinationIndex + sourceIndices.Count > destination.Length)
{
throw new ArgumentException("The given buffer must have an equal or greater length than destinationIndex+length");
}
if (Equals(column, Core.Columns.Index) || Equals(column, Core.Columns.OriginalIndex))
{
GetIndex(sourceIndices, (LogLineIndex[])(object)destination, destinationIndex);
}
else if (Equals(column, Core.Columns.LogEntryIndex))
{
GetLogEntryIndex(sourceIndices, (LogEntryIndex[])(object)destination, destinationIndex);
}
else if (Equals(column, Core.Columns.LineNumber))
{
GetLineNumber(sourceIndices, (int[])(object)destination, destinationIndex);
}
else if (Equals(column, Core.Columns.OriginalLineNumber))
{
GetLineNumber(sourceIndices, (int[])(object)destination, destinationIndex);
}
else if (IsAdorned(column, Core.Columns.ElapsedTime))
{
GetElapsedTime(sourceIndices, (TimeSpan?[])(object)destination, destinationIndex, queryOptions);
}
else if (IsAdorned(column, Core.Columns.DeltaTime))
{
GetDeltaTime(sourceIndices, (TimeSpan?[])(object)destination, destinationIndex, queryOptions);
}
else
{
_source.GetColumn(sourceIndices, column, destination, destinationIndex, queryOptions);
}
}
public void TestSkipSourceIfNotAllowed()
{
var buffer = new PageBufferedLogSource(_taskScheduler, _source.Object, TimeSpan.Zero);
var destination = new LogBufferArray(4, new IColumnDescriptor[] { Core.Columns.Index, Core.Columns.RawContent });
var queryOptions = new LogSourceQueryOptions(LogSourceQueryMode.FromCache);
buffer.GetEntries(new LogSourceSection(10, 4), destination, 0, queryOptions);
_source.Verify(x => x.GetEntries(It.IsAny <IReadOnlyList <LogLineIndex> >(), It.IsAny <ILogBuffer>(), It.IsAny <int>(), It.IsAny <LogSourceQueryOptions>()),
Times.Never, "because we didn't allow the data to be retrieved from the source under any circumstances");
}
public void TestFetchFromSourceWhenNotAllowedFromCache()
{
var buffer = new PageBufferedLogSource(_taskScheduler, _source.Object, TimeSpan.Zero);
var destination = new LogBufferArray(4, new IColumnDescriptor[] { Core.Columns.Index, Core.Columns.RawContent });
var queryOptions = new LogSourceQueryOptions(LogSourceQueryMode.FromSource);
buffer.GetEntries(new LogSourceSection(10, 4), destination, 0, queryOptions);
_source.Verify(x => x.GetEntries(new LogSourceSection(10, 4), destination, 0, queryOptions),
Times.Once);
}
public override void GetEntries(IReadOnlyList <LogLineIndex> sourceIndices,
ILogBuffer destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
lock (_syncRoot)
{
_buffer.CopyTo(new Int32View(sourceIndices), destination, destinationIndex);
}
}
public void TestCopyFromLogFile_Contiguous_NoSuchColumn()
{
var inner = new Mock <ILogBuffer>();
var view = new LogBufferView(inner.Object, Core.Columns.LogLevel, Core.Columns.Message);
var source = new Mock <ILogSource>();
var queryOptions = new LogSourceQueryOptions(LogSourceQueryMode.FromCache);
new Action(() => view.CopyFrom(Core.Columns.Timestamp, 42, source.Object, new LogSourceSection(2, 98), queryOptions)).Should().Throw <NoSuchColumnException>();
inner.Verify(x => x.CopyFrom(Core.Columns.Timestamp, It.IsAny <int>(), It.IsAny <ILogSource>(), It.IsAny <LogSourceSection>(), It.IsAny <LogSourceQueryOptions>()), Times.Never);
}
public void TestCopyFromLogFile_Contiguous()
{
var inner = new Mock <ILogBuffer>();
var view = new LogBufferView(inner.Object, Core.Columns.LogLevel, Core.Columns.Message);
var source = new Mock <ILogSource>();
var queryOptions = new LogSourceQueryOptions(LogSourceQueryMode.FromCache);
view.CopyFrom(Core.Columns.LogLevel, 42, source.Object, new LogSourceSection(2, 98), queryOptions);
inner.Verify(x => x.CopyFrom(Core.Columns.LogLevel, 42, source.Object, new LogSourceSection(2, 98), queryOptions), Times.Once);
}
public void TestCopyFromLogFile_Noncontiguous()
{
var inner = new Mock <ILogBuffer>();
var view = new LogBufferView(inner.Object, Core.Columns.LogLevel, Core.Columns.Message);
var source = new Mock <ILogSource>();
var sourceIndices = new[] { new LogLineIndex(1), new LogLineIndex(42) };
var queryOptions = new LogSourceQueryOptions(LogSourceQueryMode.FromCache);
view.CopyFrom(Core.Columns.LogLevel, 42, source.Object, sourceIndices, queryOptions);
inner.Verify(x => x.CopyFrom(Core.Columns.LogLevel, 42, source.Object, sourceIndices, queryOptions), Times.Once);
}
public void GetColumn <T>(IReadOnlyList <LogLineIndex> sourceIndices,
IColumnDescriptor <T> column,
T[] destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
if (sourceIndices == null)
{
throw new ArgumentNullException(nameof(sourceIndices));
}
GetEntries(sourceIndices, new SingleColumnLogBufferView <T>(column, destination, destinationIndex, sourceIndices.Count), 0, queryOptions);
}
private void AddToCache(IReadOnlyList <LogLineIndex> sourceIndices,
ILogBuffer source,
int sourceIndex,
LogSourceQueryOptions queryOptions)
{
if ((queryOptions.QueryMode & LogSourceQueryMode.DontCache) == LogSourceQueryMode.DontCache)
{
return;
}
if (sourceIndices is LogSourceSection contiguousSection)
{
AddToCache(source, sourceIndex, contiguousSection);
}
}
public void GetEntries(IReadOnlyList <LogLineIndex> sourceIndices,
ILogBuffer destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
foreach (var column in destination.Columns)
{
if (!_columns.Contains(column))
{
throw new NoSuchColumnException(column);
}
}
ReadRawData(sourceIndices, destination, destinationIndex, queryOptions);
}
public void TestGetColumn1()
{
var section = new LogSourceSection(42, 100);
var buffer = new string[142];
var logFile = new LogSourceProxy(_taskScheduler, TimeSpan.Zero, _logFile.Object);
var destinationIndex = 42;
var queryOptions = new LogSourceQueryOptions(LogSourceQueryMode.FromCache);
logFile.GetColumn(section, Core.Columns.RawContent, buffer, destinationIndex, queryOptions);
_logFile.Verify(x => x.GetColumn(It.Is <LogSourceSection>(y => y == section),
Core.Columns.RawContent,
buffer,
destinationIndex,
queryOptions),
Times.Once);
}
public override void GetEntries(IReadOnlyList <LogLineIndex> sourceIndices,
ILogBuffer destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
var source = _finalLogSource;
if (source != null)
{
source.GetEntries(sourceIndices, destination, destinationIndex, queryOptions);
}
else
{
destination.FillDefault(destinationIndex, sourceIndices.Count);
}
}
public void TestGetColumn2()
{
_logFile.Setup(x => x.GetColumn(It.IsAny <IReadOnlyList <LogLineIndex> >(), It.IsAny <IColumnDescriptor <string> >(), It.IsAny <string[]>(), It.IsAny <int>(), It.IsAny <LogSourceQueryOptions>())).Throws <SystemException>();
var indices = new LogLineIndex[] { 1, 2, 3 };
var buffer = new string[201];
var queryOptions = new LogSourceQueryOptions(LogSourceQueryMode.FromCache);
new Action(() => _proxy.GetColumn(indices, Columns.RawContent, buffer, 101, queryOptions)).Should().NotThrow();
_logFile.Verify(x => x.GetColumn(It.Is <IReadOnlyList <LogLineIndex> >(y => y == indices),
It.Is <IColumnDescriptor <string> >(y => Equals(y, Columns.RawContent)),
It.Is <string[]>(y => ReferenceEquals(y, buffer)),
It.Is <int>(y => y == 101),
queryOptions),
Times.Once);
}
public void TestGetEntries2()
{
_logFile.Setup(x => x.GetEntries(It.IsAny <IReadOnlyList <LogLineIndex> >(), It.IsAny <ILogBuffer>(), It.IsAny <int>(), It.IsAny <LogSourceQueryOptions>())).Throws <SystemException>();
var indices = new LogLineIndex[] { 1, 2, 3 };
var buffer = new Mock <ILogBuffer>().Object;
var destinationIndex = 101;
var queryOptions = new LogSourceQueryOptions(LogSourceQueryMode.FromCache);
new Action(() => _proxy.GetEntries(indices, buffer, destinationIndex, queryOptions)).Should().NotThrow();
_logFile.Verify(x => x.GetEntries(It.Is <IReadOnlyList <LogLineIndex> >(y => y == indices),
It.Is <ILogBuffer>(y => ReferenceEquals(y, buffer)),
destinationIndex,
queryOptions),
Times.Once);
}
public void TestGetEntries1()
{
_logFile.Setup(x => x.GetEntries(It.IsAny <LogSourceSection>(), It.IsAny <ILogBuffer>(), It.IsAny <int>(), It.IsAny <LogSourceQueryOptions>())).Throws <SystemException>();
var section = new LogSourceSection(42, 100);
var buffer = new Mock <ILogBuffer>().Object;
var destinationIndex = 9001;
var queryOptions = new LogSourceQueryOptions(LogSourceQueryMode.FromCache);
new Action(() => _proxy.GetEntries(section, buffer, destinationIndex, queryOptions)).Should().NotThrow();
_logFile.Verify(x => x.GetEntries(It.Is <LogSourceSection>(y => y == section),
It.Is <ILogBuffer>(y => ReferenceEquals(y, buffer)),
destinationIndex,
queryOptions),
Times.Once);
}
public void TestGetColumn1()
{
_logFile.Setup(x => x.GetColumn(It.IsAny <LogSourceSection>(), It.IsAny <IColumnDescriptor <string> >(), It.IsAny <string[]>(), It.IsAny <int>(), It.IsAny <LogSourceQueryOptions>())).Throws <SystemException>();
var section = new LogSourceSection(42, 100);
var buffer = new string[9101];
var destinationIndex = 9001;
var queryOptions = new LogSourceQueryOptions(LogSourceQueryMode.FromCache);
new Action(() => _proxy.GetColumn(section, Columns.RawContent, buffer, destinationIndex, queryOptions)).Should().NotThrow();
_logFile.Verify(x => x.GetColumn(It.Is <LogSourceSection>(y => y == section),
Columns.RawContent,
buffer,
destinationIndex,
queryOptions),
Times.Once);
}
public void TestGetOriginalIndexFrom2()
{
using (var proxy = new LogSourceProxy(_taskScheduler, TimeSpan.Zero, _logFile.Object))
{
var buffer = new LogLineIndex[100];
var destinationIndex = 47;
var queryOptions = new LogSourceQueryOptions(LogSourceQueryMode.FromCache);
proxy.GetColumn(new LogSourceSection(1, 42),
Core.Columns.OriginalIndex,
buffer,
destinationIndex,
queryOptions);
_logFile.Verify(x => x.GetColumn(It.Is <LogSourceSection>(y => y == new LogSourceSection(1, 42)),
Core.Columns.OriginalIndex,
buffer,
destinationIndex,
queryOptions),
Times.Once, "because the proxy should simply forward those calls to its source");
}
}
/// <inheritdoc />
public void GetEntries(IReadOnlyList <LogLineIndex> sourceIndices,
ILogBuffer destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
var source = _source;
if (source != null)
{
var columnsToCopy = new IColumnDescriptor[] { Core.Columns.Index, Core.Columns.RawContent };
var tmp = new LogBufferArray(sourceIndices.Count, columnsToCopy);
source.GetEntries(sourceIndices, tmp, 0, queryOptions);
foreach (var column in columnsToCopy)
{
if (destination.Contains(column))
{
destination.CopyFrom(column, destinationIndex, tmp, new Int32Range(0, sourceIndices.Count));
}
}
for (var i = 0; i < sourceIndices.Count; ++i)
{
var parsedLogEntry = _parser.Parse(tmp[i]);
if (parsedLogEntry != null)
{
destination[destinationIndex + i].CopyFrom(parsedLogEntry);
}
else
{
destination[destinationIndex + i].CopyFrom(_nothingParsed);
}
}
}
else
{
destination.FillDefault(destinationIndex, sourceIndices.Count);
}
}
public override void GetColumn <T>(IReadOnlyList <LogLineIndex> sourceIndices,
IColumnDescriptor <T> column,
T[] destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
lock (_syncRoot)
{
if (Equals(column, PageBufferedLogSource.RetrievalState))
{
var dest = (RetrievalState[])(object)destination;
if (sourceIndices is LogSourceSection section)
{
var totalCount = (int)(section.Index + section.Count);
var fillCount = Math.Min(totalCount, _buffer.Count);
dest.Fill(RetrievalState.Retrieved, destinationIndex, fillCount);
if (totalCount > fillCount)
{
dest.Fill(RetrievalState.NotInSource, fillCount, totalCount - fillCount);
}
}
else
{
for (int i = 0; i < sourceIndices.Count; ++i)
{
var index = sourceIndices[i];
dest[destinationIndex + i] = index < _buffer.Count
? RetrievalState.Retrieved
: RetrievalState.NotInSource;
}
}
}
else
{
_buffer.CopyTo(column, new Int32View(sourceIndices), destination, destinationIndex);
}
}
}
