public void Process(CancellationToken token)
{
if (_used)
{
throw new InvalidOperationException("This log processor cannot be reused.");
}
_used = true;
_reader = new LogReader(Stream, true);
StreamHeader = new LogStreamHeader(_reader);
var events = new List <LogEvent> (Environment.ProcessorCount * 1000);
while (!Stream.EndOfStream)
{
token.ThrowIfCancellationRequested();
_bufferHeader = new LogBufferHeader(StreamHeader, _reader);
// Read the entire buffer into a MemoryStream ahead of time to
// reduce the amount of I/O system calls we do. This should be
// fine since the profiler tries to keep buffers small and
// flushes them every second at minimum. This also has the
// advantage that we can use the Position and Length properties
// even if the stream we read the buffer from is actually
// non-seekable.
var stream = new MemoryStream(_reader.ReadBytes(_bufferHeader.Length), false);
using (var reader = new LogReader(stream, false)) {
var oldReader = _reader;
_reader = reader;
while (stream.Position < stream.Length)
{
token.ThrowIfCancellationRequested();
var ev = ReadEvent();
ProcessEvent(ImmediateVisitor, ev);
events.Add(ev);
if (ev is SynchronizationPointEvent)
{
ProcessEvents(events, token);
}
}
_reader = oldReader;
}
}
ProcessEvents(events, token);
}
public void Process(CancellationToken token)
{
if (_used)
{
throw new InvalidOperationException("This log processor cannot be reused.");
}
_used = true;
StreamHeader = new LogStreamHeader(Reader);
var events = new List <LogEvent> (Environment.ProcessorCount * 1000);
while (!Reader.BaseStream.EndOfStream)
{
token.ThrowIfCancellationRequested();
_bufferHeader = new LogBufferHeader(StreamHeader, Reader);
// Use the manual position tracking in LogReader so we're
// compatible with non-seekable streams.
var goal = Reader.Position + _bufferHeader.Length;
while (Reader.Position < goal)
{
token.ThrowIfCancellationRequested();
var ev = ReadEvent();
ProcessEvent(ImmediateVisitor, ev);
events.Add(ev);
if (ev is SynchronizationPointEvent)
{
ProcessEvents(events, token);
}
}
}
ProcessEvents(events, token);
}
void ParsingManager()
{
var _reader = new LogReader(Stream, true);
StreamHeader = new LogStreamHeader(_reader);
var avaibleWorkers = new Queue <Worker>();
for (int i = 0; i < 3; i++)
{
avaibleWorkers.Enqueue(new Worker(token, this));
}
var workingWorkers = new List <Worker>();
var unreportedEvents = new Dictionary <int, List <LogEvent> >();
int bufferId = 0;
int lastReportedId = 0;
startLength = Stream.Length;
while (true)
{
while (avaibleWorkers.Count > 0)
{
if (!Wait(48))
{
avaibleWorkers.Dequeue().Stop();
continue;
}
var _bufferHeader = new LogBufferHeader(StreamHeader, _reader);
if (!Wait(_bufferHeader.Length))
{
avaibleWorkers.Dequeue().Stop();
continue;
}
var worker = avaibleWorkers.Dequeue();
worker.BufferId = bufferId++;
worker.bufferHeader = _bufferHeader;
worker.memoryStream.Position = 0;
worker.memoryStream.SetLength(_bufferHeader.Length);
if (Stream.Read(worker.memoryStream.GetBuffer(), 0, _bufferHeader.Length) != _bufferHeader.Length)
{
throw new InvalidOperationException();
}
worker.done = new TaskCompletionSource <bool>();
worker.list = listsPool.GetObject();
workingWorkers.Add(worker);
worker.StartWork();
if (bufferId == 1)
{
worker.done.Task.Wait(); //Temporary workaround to make sure 1st event time is set correctly
}
}
if (workingWorkers.Count == 0)
{
fileFinished = true;
eventsReady.Set();
return;
}
if (queue.Count > 20)
{
needMoreEvents.Reset();
needMoreEvents.WaitOne();
}
foreach (var workingWorker in workingWorkers.ToArray())
{
if (workingWorker.done.Task.IsCompleted)
{
workingWorkers.Remove(workingWorker);
unreportedEvents.Add(workingWorker.BufferId, workingWorker.list);
while (unreportedEvents.ContainsKey(lastReportedId))
{
queue.Enqueue(unreportedEvents[lastReportedId]);
eventsReady.Set();
unreportedEvents.Remove(lastReportedId++);
}
avaibleWorkers.Enqueue(workingWorker);
}
}
}
}
public void Process(CancellationToken token, bool live = false)
{
if (_used)
{
throw new InvalidOperationException("This log processor cannot be reused.");
}
_used = true;
_reader = new LogReader(Stream, true);
StreamHeader = new LogStreamHeader(_reader);
List <LogEvent> events = null;
if (SortedVisitor != null)
{
events = new List <LogEvent> (Environment.ProcessorCount * 1000);
}
var memoryStream = new MemoryStream(4096 * 16);
while (live || (Stream.Position < Stream.Length))
{
token.ThrowIfCancellationRequested();
Wait(48, live, token);
_bufferHeader = new LogBufferHeader(StreamHeader, _reader);
Wait(_bufferHeader.Length, live, token);
memoryStream.Position = 0;
memoryStream.SetLength(_bufferHeader.Length);
if (Stream.Read(memoryStream.GetBuffer(), 0, _bufferHeader.Length) != _bufferHeader.Length)
{
throw new InvalidOperationException();
}
using (var reader = new LogReader(memoryStream, true)) {
var oldReader = _reader;
_reader = reader;
while (memoryStream.Position < memoryStream.Length)
{
var ev = ReadEvent();
ProcessEvent(ImmediateVisitor, ev);
if (SortedVisitor != null)
{
events.Add(ev);
if (ev is SynchronizationPointEvent)
{
ProcessEvents(events);
}
}
}
_reader = oldReader;
}
}
if (SortedVisitor != null)
{
ProcessEvents(events);
}
}
