public void Export(IProgress <Percentage> progressReporter = null)
{
const int bufferSize = 1000;
var buffer = new LogBufferArray(bufferSize, Columns.Index, Columns.RawContent);
var count = _logSource.GetProperty(Properties.LogEntryCount);
var index = 0;
using (var writer = new StreamWriter(_stream, Encoding.UTF8, 1024, true))
{
while (index < count)
{
var remaining = count - index;
if (remaining > bufferSize)
{
remaining = bufferSize;
}
if (!ExportPortion(writer, buffer, index, remaining))
{
break;
}
index += remaining;
var progress = Percentage.Of(index, count);
progressReporter?.Report(progress);
}
}
}
public void Export(IProgress <Percentage> progressReporter = null)
{
const int bufferSize = 1000;
var buffer = new LogLine[bufferSize];
var count = _logFile.Count;
var index = 0;
using (var writer = new StreamWriter(_stream, Encoding.UTF8, 1024, true))
{
while (index < count)
{
var remaining = count - index;
if (remaining > bufferSize)
{
remaining = bufferSize;
}
if (!ExportPortion(writer, buffer, index, remaining))
{
break;
}
index += remaining;
var progress = Percentage.Of(index, count);
progressReporter?.Report(progress);
}
}
}
private void UpdatePercentageProcessed(long streamPosition, long fileSize, bool allow100Percent)
{
var processed = Percentage.Of(streamPosition, fileSize).Clamped();
if (processed >= Percentage.FromPercent(99) && !allow100Percent)
{
processed = Percentage.FromPercent(99);
}
_localProperties.SetValue(Core.Properties.PercentageProcessed, processed);
}
public void TestSetToDefaultPartial()
{
var properties = new PropertiesBufferList();
var sourceCannotBeAccessed = new SourceDoesNotExist("wdawdwaw.txt");
properties.SetValue(Core.Properties.EmptyReason, sourceCannotBeAccessed);
properties.SetValue(Core.Properties.PercentageProcessed, Percentage.Of(50, 100));
properties.SetToDefault(new [] { Core.Properties.PercentageProcessed });
properties.GetValue(Core.Properties.EmptyReason).Should().Be(sourceCannotBeAccessed, "because only the PercentageProcessed property may have been reset");
properties.GetValue(Core.Properties.PercentageProcessed).Should().Be(Core.Properties.PercentageProcessed.DefaultValue);
}
public void TestSetToDefault()
{
var properties = new PropertiesBufferList();
var sourceCannotBeAccessed = new SourceDoesNotExist("wdawdwaw.txt");
properties.SetValue(Core.Properties.EmptyReason, sourceCannotBeAccessed);
properties.SetValue(Core.Properties.PercentageProcessed, Percentage.Of(50, 100));
properties.SetToDefault();
properties.GetValue(Core.Properties.EmptyReason).Should().Be(Core.Properties.EmptyReason.DefaultValue);
properties.GetValue(Core.Properties.PercentageProcessed).Should().Be(Percentage.Zero);
}
public void TestSetToDefaultNull()
{
var properties = new PropertiesBufferList();
var sourceCannotBeAccessed = new SourceDoesNotExist("wdawdwaw.txt");
properties.SetValue(Core.Properties.EmptyReason, sourceCannotBeAccessed);
properties.SetValue(Core.Properties.PercentageProcessed, Percentage.Of(50, 100));
new Action(() => properties.SetToDefault(null)).Should().Throw <ArgumentNullException>();
properties.GetValue(Core.Properties.EmptyReason).Should().Be(sourceCannotBeAccessed);
properties.GetValue(Core.Properties.PercentageProcessed).Should().Be(Percentage.Of(50, 100));
}
private void SynchronizeProperties()
{
_source.GetAllProperties(_propertiesBuffer.Except(_adornedProperties));
var sourceProcessed = _propertiesBuffer.GetValue(Core.Properties.PercentageProcessed);
var sourceCount = _propertiesBuffer.GetValue(Core.Properties.LogEntryCount);
var ownProgress = sourceCount > 0
? Percentage.Of(_count, sourceCount).Clamped()
: Percentage.HundredPercent;
var totalProgress = (sourceProcessed * ownProgress).Clamped();
_propertiesBuffer.SetValue(Core.Properties.PercentageProcessed, totalProgress);
_propertiesBuffer.SetValue(Core.Properties.LogEntryCount, _count);
_properties.CopyFrom(_propertiesBuffer);
}
private Percentage ComputePercentageProcessed()
{
if (!_propertiesBuffer.TryGetValue(Core.Properties.PercentageProcessed, out var sourcePercentage))
{
return(Percentage.Zero);
}
if (_fullSourceSection.Count <= 0)
{
return(sourcePercentage);
}
var ownPercentage = Percentage.Of(_currentSourceIndex, _fullSourceSection.Count);
var totalPercentage = (ownPercentage * sourcePercentage).Clamped();
return(totalPercentage);
}
private void UpdateProperties()
{
Size? size = null;
DateTime?lastModified = null;
DateTime?startTimestamp = null;
DateTime?endTimestamp = null;
for (int n = 0; n < _sources.Count; ++n)
{
var source = _sources[n];
var sourceSize = source.GetValue(LogFileProperties.Size);
if (size == null)
{
size = sourceSize;
}
else if (sourceSize != null)
{
size += sourceSize;
}
var last = source.GetValue(LogFileProperties.LastModified);
if (last != null && (last > lastModified || lastModified == null))
{
lastModified = last;
}
var start = source.GetValue(LogFileProperties.StartTimestamp);
if (start != null && (start < startTimestamp || startTimestamp == null))
{
startTimestamp = start;
}
var end = source.GetValue(LogFileProperties.EndTimestamp);
if (end != null && (end > endTimestamp || endTimestamp == null))
{
endTimestamp = end;
}
}
_properties.SetValue(LogFileProperties.LastModified, lastModified);
_properties.SetValue(LogFileProperties.Size, size);
_properties.SetValue(LogFileProperties.StartTimestamp, startTimestamp);
_properties.SetValue(LogFileProperties.EndTimestamp, endTimestamp);
_progress = Percentage.Of(_indices.Count, _totalLineCount);
}
private void UpdateProperties()
{
// Now we can perform a block-copy of all properties and then update our own as desired..
_source.GetAllProperties(_propertiesBuffer);
var sourceProcessed = _propertiesBuffer.GetValue(Core.Properties.PercentageProcessed);
var sourceCount = _propertiesBuffer.GetValue(Core.Properties.LogEntryCount);
var ownProgress = sourceCount > 0
? Percentage.Of(_indices.Count, sourceCount).Clamped()
: Percentage.HundredPercent;
var totalProgress = (sourceProcessed * ownProgress).Clamped();
_propertiesBuffer.SetValue(Core.Properties.PercentageProcessed, totalProgress);
_propertiesBuffer.SetValue(Core.Properties.LogEntryCount, (int)_currentSourceIndex);
// We want to update all properties at once, hence we modify _sourceProperties where necessary and then
// move them to our properties in a single call
_properties.CopyFrom(_propertiesBuffer);
}
/// <inheritdoc />
protected override TimeSpan RunOnce(CancellationToken token)
{
PendingModification modification;
while (_pendingModifications.TryDequeue(out modification))
{
if (token.IsCancellationRequested)
{
return(TimeSpan.Zero);
}
_totalLineCount = CalculateTotalLogLineCount();
_progress = Percentage.Of(_indices.Count, _totalLineCount);
if (modification.Section.IsReset)
{
Clear(modification.LogFile);
}
else if (modification.Section.IsInvalidate)
{
throw new NotImplementedException();
}
else
{
for (var i = 0; i < modification.Section.Count; ++i)
{
var sourceIndex = modification.Section.Index + i;
var newLogLine = modification.LogFile.GetLine((int)sourceIndex);
if (newLogLine.Timestamp != null)
{
// We need to find out where this new entry (or entries) is/are to be inserted.
var insertionIndex = _indices.Count;
byte logFileIndex;
_logFileIndices.TryGetValue(modification.LogFile, out logFileIndex);
for (var n = _indices.Count - 1; n >= 0; --n)
{
var idx = _indices[n];
var logFile = _sources[idx.LogFileIndex];
var entry = logFile.GetLine(idx.SourceLineIndex);
if (entry.Timestamp <= newLogLine.Timestamp)
{
insertionIndex = n + 1;
break;
}
if (entry.Timestamp > newLogLine.Timestamp)
{
insertionIndex = n;
}
}
var mergedLogEntryIndex = GetMergedLogEntryIndex(modification.LogFile, insertionIndex, newLogLine);
var index = new Index((int)sourceIndex,
mergedLogEntryIndex,
newLogLine.LogEntryIndex,
logFileIndex);
if (insertionIndex < _indices.Count)
{
InvalidateOnward(insertionIndex, modification.LogFile, newLogLine);
}
lock (_syncRoot)
{
_indices.Insert(insertionIndex, index);
_maxCharactersPerLine = Math.Max(_maxCharactersPerLine, newLogLine.Message?.Length ?? 0);
}
Listeners.OnRead(_indices.Count);
// There's no need to frantically update every time, but every
// once in a while would be nice...
if (i % 100 == 0)
{
UpdateProperties();
}
}
}
}
}
UpdateProperties();
Listeners.OnRead(_indices.Count);
SetEndOfSourceReached();
return(_maximumWaitTime);
}
private bool ReadEntireFile(CancellationToken cancellationToken)
{
if (!HasFileChanged(out var currentFingerprint, out var previousFingerprint))
{
if (Log.IsDebugEnabled)
{
Log.DebugFormat("File {0} remains unchanged (Fingerprint: {1}), nothing to do...", _fileName, currentFingerprint);
}
return(false);
}
var start = DateTime.UtcNow;
if (Log.IsDebugEnabled)
{
Log.DebugFormat("File {0} change detected (Fingerprint: {1}), scanning for new lines...", _fileName, currentFingerprint);
}
if (previousFingerprint?.Size > currentFingerprint.Size)
{
if (Log.IsDebugEnabled)
{
Log.DebugFormat("File {0} size reduced from {1}bytes to {2} bytes, starting all over", _fileName, previousFingerprint.Size, currentFingerprint.Size);
}
Clear();
}
using (var stream = new FileStream(_fileName,
FileMode.Open,
FileAccess.Read,
FileShare.ReadWrite | FileShare.Delete))
{
_propertiesBuffer.SetValue(Core.Properties.EmptyReason, null);
var startOffset = _lastLineOffsetStreamPosition;
if (startOffset == 0)
{
AddFirstLineIfNecessary(stream);
}
else
{
stream.Position = startOffset;
}
var detector = new LineOffsetDetector(stream, _encoding);
var buffer = new FixedSizeList <long>(1000);
long lineOffset;
while ((lineOffset = detector.FindNextLineOffset()) >= 0)
{
if (cancellationToken.IsCancellationRequested)
{
return(false);
}
if (!buffer.TryAdd(lineOffset))
{
AddLines(buffer, Percentage.Of(stream.Position, currentFingerprint.Size), startOffset);
buffer.TryAdd(lineOffset);
}
}
AddLines(buffer, Percentage.HundredPercent, startOffset);
_lastStreamPosition = stream.Position;
}
if (Log.IsDebugEnabled)
{
var elapsed = DateTime.UtcNow - start;
Log.DebugFormat("File {0} scan finished, took: {1:F1}ms", _fileName, elapsed.TotalMilliseconds);
}
return(true);
}
