public void Process(CancellationToken token) { if (_used) { throw new InvalidOperationException("This log processor cannot be reused."); } _used = true; _reader = new LogReader(Stream, true); StreamHeader = new LogStreamHeader(_reader); var events = new List <LogEvent> (Environment.ProcessorCount * 1000); while (!Stream.EndOfStream) { token.ThrowIfCancellationRequested(); _bufferHeader = new LogBufferHeader(StreamHeader, _reader); // Read the entire buffer into a MemoryStream ahead of time to // reduce the amount of I/O system calls we do. This should be // fine since the profiler tries to keep buffers small and // flushes them every second at minimum. This also has the // advantage that we can use the Position and Length properties // even if the stream we read the buffer from is actually // non-seekable. var stream = new MemoryStream(_reader.ReadBytes(_bufferHeader.Length), false); using (var reader = new LogReader(stream, false)) { var oldReader = _reader; _reader = reader; while (stream.Position < stream.Length) { token.ThrowIfCancellationRequested(); var ev = ReadEvent(); ProcessEvent(ImmediateVisitor, ev); events.Add(ev); if (ev is SynchronizationPointEvent) { ProcessEvents(events, token); } } _reader = oldReader; } } ProcessEvents(events, token); }
public void Process(CancellationToken token) { if (_used) { throw new InvalidOperationException("This log processor cannot be reused."); } _used = true; StreamHeader = new LogStreamHeader(Reader); var events = new List <LogEvent> (Environment.ProcessorCount * 1000); while (!Reader.BaseStream.EndOfStream) { token.ThrowIfCancellationRequested(); _bufferHeader = new LogBufferHeader(StreamHeader, Reader); // Use the manual position tracking in LogReader so we're // compatible with non-seekable streams. var goal = Reader.Position + _bufferHeader.Length; while (Reader.Position < goal) { token.ThrowIfCancellationRequested(); var ev = ReadEvent(); ProcessEvent(ImmediateVisitor, ev); events.Add(ev); if (ev is SynchronizationPointEvent) { ProcessEvents(events, token); } } } ProcessEvents(events, token); }
void ParsingManager() { var _reader = new LogReader(Stream, true); StreamHeader = new LogStreamHeader(_reader); var avaibleWorkers = new Queue <Worker>(); for (int i = 0; i < 3; i++) { avaibleWorkers.Enqueue(new Worker(token, this)); } var workingWorkers = new List <Worker>(); var unreportedEvents = new Dictionary <int, List <LogEvent> >(); int bufferId = 0; int lastReportedId = 0; startLength = Stream.Length; while (true) { while (avaibleWorkers.Count > 0) { if (!Wait(48)) { avaibleWorkers.Dequeue().Stop(); continue; } var _bufferHeader = new LogBufferHeader(StreamHeader, _reader); if (!Wait(_bufferHeader.Length)) { avaibleWorkers.Dequeue().Stop(); continue; } var worker = avaibleWorkers.Dequeue(); worker.BufferId = bufferId++; worker.bufferHeader = _bufferHeader; worker.memoryStream.Position = 0; worker.memoryStream.SetLength(_bufferHeader.Length); if (Stream.Read(worker.memoryStream.GetBuffer(), 0, _bufferHeader.Length) != _bufferHeader.Length) { throw new InvalidOperationException(); } worker.done = new TaskCompletionSource <bool>(); worker.list = listsPool.GetObject(); workingWorkers.Add(worker); worker.StartWork(); if (bufferId == 1) { worker.done.Task.Wait(); //Temporary workaround to make sure 1st event time is set correctly } } if (workingWorkers.Count == 0) { fileFinished = true; eventsReady.Set(); return; } if (queue.Count > 20) { needMoreEvents.Reset(); needMoreEvents.WaitOne(); } foreach (var workingWorker in workingWorkers.ToArray()) { if (workingWorker.done.Task.IsCompleted) { workingWorkers.Remove(workingWorker); unreportedEvents.Add(workingWorker.BufferId, workingWorker.list); while (unreportedEvents.ContainsKey(lastReportedId)) { queue.Enqueue(unreportedEvents[lastReportedId]); eventsReady.Set(); unreportedEvents.Remove(lastReportedId++); } avaibleWorkers.Enqueue(workingWorker); } } } }
public void Process(CancellationToken token, bool live = false) { if (_used) { throw new InvalidOperationException("This log processor cannot be reused."); } _used = true; _reader = new LogReader(Stream, true); StreamHeader = new LogStreamHeader(_reader); List <LogEvent> events = null; if (SortedVisitor != null) { events = new List <LogEvent> (Environment.ProcessorCount * 1000); } var memoryStream = new MemoryStream(4096 * 16); while (live || (Stream.Position < Stream.Length)) { token.ThrowIfCancellationRequested(); Wait(48, live, token); _bufferHeader = new LogBufferHeader(StreamHeader, _reader); Wait(_bufferHeader.Length, live, token); memoryStream.Position = 0; memoryStream.SetLength(_bufferHeader.Length); if (Stream.Read(memoryStream.GetBuffer(), 0, _bufferHeader.Length) != _bufferHeader.Length) { throw new InvalidOperationException(); } using (var reader = new LogReader(memoryStream, true)) { var oldReader = _reader; _reader = reader; while (memoryStream.Position < memoryStream.Length) { var ev = ReadEvent(); ProcessEvent(ImmediateVisitor, ev); if (SortedVisitor != null) { events.Add(ev); if (ev is SynchronizationPointEvent) { ProcessEvents(events); } } } _reader = oldReader; } } if (SortedVisitor != null) { ProcessEvents(events); } }