public void GetEntries(IReadOnlyList <LogLineIndex> sourceIndices,
ILogBuffer destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
if (TryReadFromCache(sourceIndices, destination, destinationIndex, queryOptions, out var accessedPageBoundaries))
{
return;
}
if ((queryOptions.QueryMode & LogSourceQueryMode.FetchForLater) == LogSourceQueryMode.FetchForLater)
{
FetchForLater(accessedPageBoundaries);
}
// For some clients (GUI) it is permissible to serve partial requests and to default out the rest.
// The ui will try again at a later date until it gets what it needs.
if ((queryOptions.QueryMode & LogSourceQueryMode.FromSource) == 0)
{
return;
}
// Whelp, we gotta fetch from the source instead.
// Note: We don't lock this part because this would block any other thread currently
// trying to read data from cache, etc.. We only block when we need to and only
// for a short amount of time!
_source.GetEntries(sourceIndices, destination, destinationIndex, queryOptions);
// However now that we got some data, we could try to add it to our cache
AddToCache(sourceIndices, destination, destinationIndex, queryOptions);
}
public bool TryRead(LogLineIndex sourceStartIndex, int count, ILogBuffer destination, int destinationIndex, bool requiresValidityCheck)
{
++_numReads;
_lastAccessTime = DateTime.UtcNow;
var pageSourceIndex = sourceStartIndex - _section.Index;
var range = new Int32Range(pageSourceIndex, count);
foreach (var column in _buffer.Columns)
{
if (destination.Contains(column))
{
destination.CopyFrom(column, destinationIndex, _buffer, range);
}
}
if (requiresValidityCheck)
{
if (_buffer.ContainsAnyDefault(Columns.Index, range))
{
return(false);
}
}
return(true);
}
protected AbstractReadRequest(ILogBuffer destination, int destinationIndex)
{
_destination = destination;
_destinationIndex = destinationIndex;
_taskSource = new TaskCompletionSource <int>();
_creationTime = DateTime.Now;
}
public void GetEntries(IReadOnlyList <LogLineIndex> sourceIndices,
ILogBuffer destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
var source = _source;
if (source == null)
{
destination.FillDefault(destinationIndex, sourceIndices.Count);
return;
}
source.GetEntries(sourceIndices, destination.Except(MaxAdornedColumns), destinationIndex, queryOptions);
var augmentedColumns = FindAugmentedColumns(destination);
if (augmentedColumns.Count == 0)
{
return;
}
if (destinationIndex != 0)
{
throw new NotImplementedException();
}
foreach (var column in augmentedColumns)
{
destination.CopyFrom(column, this, sourceIndices, queryOptions);
}
}
public ContiguousReadRequest(LogSourceSection sourceSection,
ILogBuffer destination,
int destinationIndex)
: base(destination, destinationIndex)
{
_sourceSection = sourceSection;
}
public FragmentedReadRequest(IReadOnlyList <LogLineIndex> sourceIndices,
ILogBuffer destination,
int destinationIndex)
: base(destination, destinationIndex)
{
_sourceIndices = sourceIndices as LogLineIndex[] ?? sourceIndices.ToArray();
}
public override void GetEntries(IReadOnlyList <LogLineIndex> sourceIndices,
ILogBuffer destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
_source.GetEntries(sourceIndices, destination, destinationIndex, queryOptions);
}
private void AddToCache(ILogBuffer source, int destinationIndex, LogSourceSection contiguousSection)
{
lock (_syncRoot)
{
_buffer.TryAdd(contiguousSection, source, destinationIndex);
}
}
/// <inheritdoc />
public override void GetEntries(IReadOnlyList <LogLineIndex> sourceIndices, ILogBuffer destination, int destinationIndex, LogSourceQueryOptions queryOptions)
{
foreach (var column in destination.Columns)
{
destination.CopyFrom(column, destinationIndex, this, sourceIndices, queryOptions);
}
}
/// <summary>
/// Copies the given *non-contiguous* segment of data from the given log file into this buffer in a contiguous block.
/// </summary>
/// <remarks>
/// This buffer must be large enough already to accomodate the data.
/// </remarks>
/// <param name="that"></param>
/// <param name="column">The column to copy the data from the log file to this buffer</param>
/// <param name="destinationIndex">The first index in this buffer to which the data from the given <paramref name="source" /> is copied</param>
/// <param name="source">The log file from which data should be copied from</param>
/// <param name="sourceIndices">The non-contiguous section of the log file from which to copy from (e.g. from index 5, 10 entries)</param>
public static void CopyFrom(this ILogBuffer that,
IColumnDescriptor column,
int destinationIndex,
ILogSource source,
IReadOnlyList <LogLineIndex> sourceIndices)
{
that.CopyFrom(column, destinationIndex, source, sourceIndices, LogSourceQueryOptions.Default);
}
/// <inheritdoc />
public override void GetEntries(IReadOnlyList <LogLineIndex> sourceIndices, ILogBuffer destination, int destinationIndex, LogSourceQueryOptions queryOptions)
{
// TODO: This can probably be optimized (why are we translating indices each time for every column?!
foreach (var column in destination.Columns)
{
destination.CopyFrom(column, destinationIndex, this, sourceIndices, queryOptions);
}
}
/// <inheritdoc />
public void GetEntries(IReadOnlyList <LogLineIndex> sourceIndices, ILogBuffer destination, int destinationIndex, LogSourceQueryOptions queryOptions)
{
lock (_syncRoot)
{
foreach (var column in destination.Columns)
{
destination.CopyFrom(column, destinationIndex, _logBuffer, new Int32View(sourceIndices));
}
}
}
public bool TryCopyTo(IReadOnlyList <LogLineIndex> indices, ILogBuffer buffer, int destinationIndex)
{
if (!TryTranslateToBufferIndices(indices, out var bufferIndices))
{
return(false);
}
_buffer.CopyTo(bufferIndices, buffer, destinationIndex);
return(true);
}
public override void GetEntries(IReadOnlyList <LogLineIndex> sourceIndices,
ILogBuffer destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
lock (_syncRoot)
{
_buffer.CopyTo(new Int32View(sourceIndices), destination, destinationIndex);
}
}
/// <inheritdoc />
public void GetEntries(LogSourceSection sourceSection, ILogBuffer destination, int destinationIndex, LogSourceQueryOptions queryOptions)
{
lock (_syncRoot)
{
foreach (var column in destination.Columns)
{
destination.CopyFrom(column, destinationIndex, _logBuffer, new Int32View(sourceSection));
}
}
}
/// <summary>
/// Returns a new log buffer which acts as a view onto the original buffer.
/// If the original buffer ONLY contains the desired column, then the original buffer is returned.
/// Otherwise a new temporary buffer is returned with the same size as the original buffer containing
/// only the given column.
/// </summary>
/// <param name="that"></param>
/// <param name="column"></param>
/// <returns></returns>
public static ILogBuffer CreateViewOnlyWithColumn(this ILogBuffer that, IColumnDescriptor column)
{
if (that.Columns.Count == 1 && that.Contains(column))
{
return(that);
}
var temporaryBuffer = new LogBufferArray(that.Count, column);
return(temporaryBuffer);
}
/// <inheritdoc />
public void GetEntries(IReadOnlyList <LogLineIndex> sourceIndices, ILogBuffer destination, int destinationIndex, LogSourceQueryOptions queryOptions)
{
try
{
_logSource.GetEntries(sourceIndices, destination, destinationIndex, queryOptions);
}
catch (Exception e)
{
BlameExceptionOnPlugin(e);
}
}
/// <summary>
///
/// </summary>
/// <param name="sourceIndices"></param>
/// <param name="destination"></param>
/// <param name="destinationIndex"></param>
public void CopyTo(IReadOnlyList <int> sourceIndices, ILogBuffer destination, int destinationIndex)
{
foreach (var column in destination.Columns)
{
if (_dataByColumn.TryGetValue(column, out var data))
{
destination.CopyFrom(column, destinationIndex, this, sourceIndices);
}
else
{
destination.FillDefault(column, destinationIndex, sourceIndices.Count);
}
}
}
private static IReadOnlyList <IColumnDescriptor> FindAugmentedColumns(ILogBuffer destination)
{
var augmented = new List <IColumnDescriptor>(MaxAdornedColumns.Count);
foreach (var column in MaxAdornedColumns)
{
if (destination.Contains(column))
{
augmented.Add(column);
}
}
return(augmented);
}
public void GetEntries(IReadOnlyList <LogLineIndex> sourceIndices,
ILogBuffer destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
foreach (var column in destination.Columns)
{
if (!_columns.Contains(column))
{
throw new NoSuchColumnException(column);
}
}
ReadRawData(sourceIndices, destination, destinationIndex, queryOptions);
}
private void AddToCache(IReadOnlyList <LogLineIndex> sourceIndices,
ILogBuffer source,
int sourceIndex,
LogSourceQueryOptions queryOptions)
{
if ((queryOptions.QueryMode & LogSourceQueryMode.DontCache) == LogSourceQueryMode.DontCache)
{
return;
}
if (sourceIndices is LogSourceSection contiguousSection)
{
AddToCache(source, sourceIndex, contiguousSection);
}
}
public override void GetEntries(IReadOnlyList <LogLineIndex> sourceIndices,
ILogBuffer destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
var source = _finalLogSource;
if (source != null)
{
source.GetEntries(sourceIndices, destination, destinationIndex, queryOptions);
}
else
{
destination.FillDefault(destinationIndex, sourceIndices.Count);
}
}
/// <inheritdoc />
public void GetEntries(IReadOnlyList <LogLineIndex> sourceIndices, ILogBuffer destination, int destinationIndex, LogSourceQueryOptions queryOptions)
{
ILogSource logSource = _source;
if (logSource != null)
{
logSource.GetEntries(sourceIndices, destination, destinationIndex, queryOptions);
}
else
{
foreach (var column in destination.Columns)
{
destination.FillDefault(column, destinationIndex, sourceIndices.Count);
}
}
}
public void Cancel()
{
var elapsed = DateTime.Now - _creationTime;
Log.DebugFormat("Cancelling '{0}' after {1:F1}ms...", this, elapsed.TotalMilliseconds);
// This lock is important because we must make sure that
lock (this)
{
_destination = null;
}
if (!_taskSource.TrySetCanceled())
{
Log.Warn("Unable to cancel pending task (why though?!)");
}
}
private bool TryGetEntriesContiguous(LogSourceSection sourceSection, ILogBuffer destination, int destinationIndex, out IReadOnlyList <LogSourceSection> accessedPageBoundaries)
{
bool fullyRead = true;
var sourceSectionEndIndex = Math.Min((int)(sourceSection.Index + sourceSection.Count), _sourceCount);
var numEntriesRead = 0;
var tmpAccessedPageBoundaries = new List <LogSourceSection>();
for (LogLineIndex i = sourceSection.Index; i < sourceSectionEndIndex;)
{
var pageIndex = GetPageIndex(i);
var remainingPageCount = (pageIndex + 1) * _pageSize - i;
var count = Math.Min(remainingPageCount, sourceSectionEndIndex - i);
var page = TryGetPage(pageIndex);
if (page != null)
{
fullyRead &= page.TryRead(i, count, destination, destinationIndex + numEntriesRead, fullyRead);
tmpAccessedPageBoundaries.Add(page.Section);
}
else
{
destination.FillDefault(destinationIndex + numEntriesRead, count);
if (destination.Contains(PageBufferedLogSource.RetrievalState))
{
destination.Fill(PageBufferedLogSource.RetrievalState, RetrievalState.NotCached, destinationIndex + numEntriesRead, count);
}
fullyRead = false;
tmpAccessedPageBoundaries.Add(GetSectionForPage(pageIndex));
}
numEntriesRead += count;
i += count;
}
if (numEntriesRead < sourceSection.Count)
{
var start = destinationIndex + numEntriesRead;
destination.FillDefault(start, sourceSection.Count - numEntriesRead);
fullyRead = false;
}
accessedPageBoundaries = tmpAccessedPageBoundaries;
return(fullyRead);
}
/// <summary>
/// Returns a new log buffer which acts as a view onto the original buffer.
/// If the original buffer already contains the desired column, then the original buffer is returned.
/// If it does not, then an additional temporary buffer of equal length is created which
/// holds data for the given <paramref name="columns"/> and the returned buffer combines both the
/// <paramref name="that" /> as well as the temporary buffer into one view.
/// </summary>
/// <remarks>
/// This method exists for those cases, where, in order to fulfill a certain <see cref="ILogSource.GetEntries(System.Collections.Generic.IReadOnlyList{LogLineIndex}, ILogBuffer, int, LogSourceQueryOptions)"/>
/// request, one has to make certain to retrieve a particular column from the source. Nothing needs to be done
/// if the caller is also interested in said column, however if the caller is not, then it becomes a one liner
/// to create a combined buffer which acts as a proxy for the original for every column but the desired one,
/// which is copied into an additional buffer.
///
/// If it were permissible to query the source log file countless times, then this method wouldn't need to exist,
/// however since we don't want to do exactly that, we'll have to do this procedure to improve performance, now
/// that log files are streamed into memory on-demand.
/// </remarks>
/// <param name="that"></param>
/// <param name="columns"></param>
/// <returns></returns>
public static ILogBuffer CreateViewWithAdditionalColumns(this ILogBuffer that, IReadOnlyList <IColumnDescriptor> columns)
{
if (columns.Count == 0)
{
return(that);
}
var missingColumns = columns.Except(that.Columns).ToList();
if (missingColumns.Count == 0)
{
return(that);
}
var temporaryBuffer = new LogBufferArray(that.Count, missingColumns);
var combinedView = new CombinedLogBufferView(new[] { that, temporaryBuffer });
return(combinedView);
}
/// <summary>
/// Tries to retrieve the given entries from this buffer.
/// </summary>
/// <remarks>
/// This method will fill the index column of the given buffer (if it has one) with the indices of the log entries,
/// or <see cref="LogLineIndex.Invalid"/> in case the data isn't part of the cache or it outside of the valid section
/// of the log source.
/// </remarks>
/// <param name="sourceIndices"></param>
/// <param name="destination"></param>
/// <param name="destinationIndex"></param>
/// <param name="accessedPageBoundaries"></param>
/// <returns>True when *all* entries could be retrieved from this buffer, false otherwise</returns>
public bool TryGetEntries(IReadOnlyList <LogLineIndex> sourceIndices,
ILogBuffer destination,
int destinationIndex,
out IReadOnlyList <LogSourceSection> accessedPageBoundaries)
{
if (destination == null)
{
throw new ArgumentNullException(nameof(destination));
}
if (destinationIndex < 0)
{
throw new ArgumentOutOfRangeException(nameof(destinationIndex), destinationIndex, "The destination index must 0 or greater");
}
if (sourceIndices is LogSourceSection contiguousSection)
{
return(TryGetEntriesContiguous(contiguousSection, destination, destinationIndex, out accessedPageBoundaries));
}
return(TryGetEntriesSegmented(sourceIndices, destination, destinationIndex, out accessedPageBoundaries));
}
/// <inheritdoc />
public void GetEntries(IReadOnlyList <LogLineIndex> sourceIndices,
ILogBuffer destination,
int destinationIndex,
LogSourceQueryOptions queryOptions)
{
var source = _source;
if (source != null)
{
var columnsToCopy = new IColumnDescriptor[] { Core.Columns.Index, Core.Columns.RawContent };
var tmp = new LogBufferArray(sourceIndices.Count, columnsToCopy);
source.GetEntries(sourceIndices, tmp, 0, queryOptions);
foreach (var column in columnsToCopy)
{
if (destination.Contains(column))
{
destination.CopyFrom(column, destinationIndex, tmp, new Int32Range(0, sourceIndices.Count));
}
}
for (var i = 0; i < sourceIndices.Count; ++i)
{
var parsedLogEntry = _parser.Parse(tmp[i]);
if (parsedLogEntry != null)
{
destination[destinationIndex + i].CopyFrom(parsedLogEntry);
}
else
{
destination[destinationIndex + i].CopyFrom(_nothingParsed);
}
}
}
else
{
destination.FillDefault(destinationIndex, sourceIndices.Count);
}
}
private bool TryReadFromCache(IReadOnlyList <LogLineIndex> sourceIndices,
ILogBuffer destination, int destinationIndex,
LogSourceQueryOptions queryOptions,
out IReadOnlyList <LogSourceSection> accessedPageBoundaries)
{
accessedPageBoundaries = null;
// For whatever reason some people prefer to read directly from the source (this might not be a real use case but whatever..)
if ((queryOptions.QueryMode & LogSourceQueryMode.FromCache) != LogSourceQueryMode.FromCache)
{
return(false);
}
lock (_syncRoot)
{
if (_buffer.TryGetEntries(sourceIndices, destination, destinationIndex, out accessedPageBoundaries))
{
return(true);
}
}
return(false);
}
/// <inheritdoc />
public override void GetEntries(IReadOnlyList <LogLineIndex> sourceIndices, ILogBuffer destination, int destinationIndex, LogSourceQueryOptions queryOptions)
{
if (IsDisposed)
{
destination.FillDefault(destinationIndex, sourceIndices.Count);
return;
}
var remainingColumns = new List <IColumnDescriptor>();
bool partiallyRetrieved = false;
foreach (var column in destination.Columns)
{
if (_specialColumns.Contains(column))
{
destination.CopyFrom(column, destinationIndex, this, sourceIndices, queryOptions);
partiallyRetrieved = true;
}
else
{
remainingColumns.Add(column);
}
}
if (remainingColumns.Count > 0)
{
if (partiallyRetrieved)
{
var view = new LogBufferView(destination, remainingColumns);
_source.GetEntries(sourceIndices, view, destinationIndex, queryOptions);
}
else
{
_source.GetEntries(sourceIndices, destination, destinationIndex, queryOptions);
}
}
}
