public static IEnumerable <PostprocessedMessage> MergePostprocessedMessage(IEnumerable <PostprocessedMessage>[] enums) { var comparer = new EnumeratorsComparer(); var iters = new VCSKicksCollection.PriorityQueue <IEnumerator <PostprocessedMessage> >(comparer); try { foreach (var e in enums) { var i = e.GetEnumerator(); if (i.MoveNext()) { iters.Enqueue(i); } } for (; iters.Count > 0;) { var i = iters.Dequeue(); try { yield return(i.Current); if (i.MoveNext()) { iters.Enqueue(i); i = null; } } finally { if (i != null) { i.Dispose(); } } } } finally { while (iters.Count != 0) { iters.Dequeue().Dispose(); } } }
static async Task EnumMessagesAndMerge( ILogSource[] sources, Action <IMessage> callback, Progress.IProgressAggregator progress, CancellationToken cancellation ) { var queue = new VCSKicksCollection.PriorityQueue <EnumMessagesHelper>( new EnumMessagesHelper.Comparer()); var helpers = sources.Select(s => new EnumMessagesHelper( s, cancellation, progress.CreateProgressSink())).ToList(); try { await Task.WhenAll(helpers.Select(h => h.FillBuffer())); Action <EnumMessagesHelper> enqueueOfKill = h => { if (h.Peek() != null) { queue.Enqueue(h); } else { h.Dispose(); } }; helpers.ForEach(enqueueOfKill); while (queue.Count > 0) { var h = queue.Dequeue(); callback(h.Peek()); h.Dequeue(); if (h.Peek() == null) { await h.FillBuffer(); } enqueueOfKill(h); } } finally { helpers.ForEach(h => h.Dispose()); } }
public static IEnumerable <T> MergeSortedSequences <T>(this IEnumerable <T>[] enums, IComparer <T> valueComparer) { var comparer = new EnumeratorsComparer <T>(valueComparer); var iters = new VCSKicksCollection.PriorityQueue <IEnumerator <T> >(comparer); try { foreach (var e in enums) { var i = e.GetEnumerator(); if (i.MoveNext()) { iters.Enqueue(i); } } for (; iters.Count > 0;) { var i = iters.Dequeue(); try { yield return(i.Current); if (i.MoveNext()) { iters.Enqueue(i); i = null; } } finally { if (i != null) { i.Dispose(); } } } } finally { while (iters.Count != 0) { iters.Dequeue().Dispose(); } } }
/// <summary> /// Makes sure events have correct timestamp and go in correct order. /// Chromedriver log has a problem that timestamps at beginning of lines like [1525678451.879] /// seem to be rounded up to next 100ms boundary which results to inaccurate views. /// Most important messages like Network.requestWillBeSent have "timestamp" as a json field. /// That "timestamp" is nr os seconds from unknown origin. Luckily some messages also have /// "wallTime" in json payload that can help interpret "timestamp". /// Another problem is that "timestamp" of Network.requestWillBeSent might not match /// timing.requestTime in Network.responseReceived. The latter seems to be more accurate. /// However if a request is served from cache its timing.requestTime is totally wrong and /// should be ignored in favor of Network.requestWillBeSent's "timestamp". /// </summary> static IEnumerableAsync <Message[]> FixTimestamps(IEnumerableAsync <Message[]> messages) { DateTime?timestampBase = null; var pendingMessages = new List <MessageEntry>(); var queue = new VCSKicksCollection.PriorityQueue <MessageEntry>(new Comparer()); var queuedRequestStarts = new Dictionary <string, MessageEntry>(); double? lastDequeuedTimestamp = null; Action <Queue <Message> > dequeue = (outputQueue) => { var entry = queue.Dequeue(); if (!entry.IsInvalidated) { if (lastDequeuedTimestamp == null || entry.Timestamp != null) { lastDequeuedTimestamp = entry.Timestamp; } outputQueue.Enqueue(entry.Msg); if (entry.Type == MessageEntry.EntryType.StartRequest) { queuedRequestStarts.Remove(entry.RequestId); } } }; Action <MessageEntry> enqueue = null; enqueue = (r) => { var rebased = r.Rebase(timestampBase.Value); queue.Enqueue(rebased); if (r.Type == MessageEntry.EntryType.StartRequest) { queuedRequestStarts[r.RequestId] = rebased; } else if (r.Type == MessageEntry.EntryType.ResponseWithTiming) { MessageEntry queuedRequest; if (queuedRequestStarts.TryGetValue(r.RequestId, out queuedRequest) && !queuedRequest.IsInvalidated && !queuedRequest.IsServedFromCache) { if (lastDequeuedTimestamp == null || r.ResponseTiming_RequestTime > lastDequeuedTimestamp.Value) { enqueue(queuedRequest.InvalidateAndMakeFixedStartRequest(r.ResponseTiming_RequestTime)); } } } else if (r.Type == MessageEntry.EntryType.ServedFromCache) { MessageEntry queuedRequest; if (queuedRequestStarts.TryGetValue(r.RequestId, out queuedRequest)) { queuedRequest.IsServedFromCache = true; } } }; Action flushPendingMessages = () => { foreach (var pendingMessage in pendingMessages) { enqueue(pendingMessage); } pendingMessages.Clear(); }; int queueSize = 4096; return(messages.Select <Message, Message>((m, outputQueue) => { var newEntry = new MessageEntry(m); if (timestampBase == null && newEntry.Timestamp != null && newEntry.WallTime != null) { timestampBase = TimeUtils.UnixTimestampMillisToDateTime( newEntry.WallTime.Value * 1000d).ToUnspecifiedTime().AddSeconds( -newEntry.Timestamp.Value); flushPendingMessages(); } if (timestampBase == null) { pendingMessages.Add(newEntry); } else { enqueue(newEntry); } if (queue.Count >= queueSize * 2) { while (queue.Count > queueSize) { dequeue(outputQueue); } } }, (outputQueue) => { while (queue.Count > 0) { dequeue(outputQueue); } })); }
public static async Task SerializePostprocessorOutput <Evt, Serializer, EvtVisitor>( this IEnumerableAsync <Evt[]> events, Func <Action <object, XElement>, Serializer> serializerFactory, Task <ILogPartToken> rotatedLogPartToken, ILogPartTokenFactories rotatedLogPartFactories, Func <object, TextLogEventTrigger> triggersConverter, string contentsEtagAttr, string rootElementName, string outputFileName, ITempFilesManager tempFiles, CancellationToken cancellation ) where Evt : IVisitable <EvtVisitor> where Serializer : class, IEventsSerializer, EvtVisitor { rotatedLogPartToken = rotatedLogPartToken ?? Task.FromResult <ILogPartToken>(null); var sortKeyAttr = XName.Get("__key"); var chunks = new List <string>(); Serializer serializer = null; Action resetSerializer = () => { if (serializer?.Output?.Count > 0) { string chunkFileName = tempFiles.GenerateNewName(); chunks.Add(chunkFileName); using (var writer = XmlWriter.Create(chunkFileName, new XmlWriterSettings() { OmitXmlDeclaration = true, ConformanceLevel = ConformanceLevel.Fragment })) { foreach (var e in serializer.Output.OrderBy(e => e.Attribute(sortKeyAttr).Value)) { e.WriteTo(writer); } } } serializer = serializerFactory((trigger, elt) => { triggersConverter(trigger).Save(elt); elt.SetAttributeValue(sortKeyAttr, ((IOrderedTrigger)trigger).Index.ToString("x8")); }); }; resetSerializer(); await events.ForEach(batch => { foreach (var e in batch) { e.Visit(serializer); if (serializer.Output.Count >= 8 * 1024) { resetSerializer(); } } return(Task.FromResult(!cancellation.IsCancellationRequested)); }); resetSerializer(); if (cancellation.IsCancellationRequested) { return; } using (var outputWriter = XmlWriter.Create(outputFileName, new XmlWriterSettings() { Indent = true })) { outputWriter.WriteStartElement(rootElementName); new PostprocessorOutputETag(contentsEtagAttr).Write(outputWriter); rotatedLogPartFactories.SafeWriteTo(await rotatedLogPartToken, outputWriter); var readersSettings = new XmlReaderSettings() { ConformanceLevel = ConformanceLevel.Fragment }; var readers = chunks.Select(chunkFileName => XmlReader.Create(chunkFileName, readersSettings)).ToList(); try { var q = new VCSKicksCollection.PriorityQueue <KeyValuePair <XmlReader, XElement> >(Comparer <KeyValuePair <XmlReader, XElement> > .Create((item1, item2) => { return(string.CompareOrdinal(item1.Value.Attribute(sortKeyAttr).Value, item2.Value.Attribute(sortKeyAttr).Value)); })); Action <XmlReader> enqueueReader = reader => { if (!reader.EOF) { if (reader.MoveToContent() != XmlNodeType.Element) { throw new InvalidOperationException("bad chunk"); } q.Enqueue(new KeyValuePair <XmlReader, XElement>(reader, (XElement)XNode.ReadFrom(reader))); } }; readers.ForEach(enqueueReader); while (q.Count > 0) { var item = q.Dequeue(); item.Value.Attribute(sortKeyAttr).Remove(); item.Value.WriteTo(outputWriter); enqueueReader(item.Key); } } finally { readers.ForEach(r => r.Dispose()); chunks.ForEach(chunkFileName => tempFiles.DeleteIfTemporary(chunkFileName)); } outputWriter.WriteEndElement(); // end of root node } }
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback) { await callback.BecomeLongRunning(); string factoryName = @params.Argument; callback.TempFilesCleanupList.Add(@params.Location); Action <double?> setStepDescription = prctComplete => { var str = new StringBuilder(); str.Append(@params.FullPath); str.Append(": fixing timestamp anomalies..."); if (prctComplete != null) { str.AppendFormat(" {0}%", (int)(prctComplete.Value * 100)); } callback.SetStepDescription(str.ToString()); }; setStepDescription(null); string tmpFileName = callback.TempFilesManager.GenerateNewName(); var factoryNameSplit = factoryName.Split('\\'); if (factoryNameSplit.Length != 2) { throw new InvalidFormatException(); } var factory = logProviderFactoryRegistry.Find(factoryNameSplit[0], factoryNameSplit[1]); if (factory == null) { throw new InvalidDataException("factory not found: " + factoryName); } var readerFactory = factory as IMediaBasedReaderFactory; if (readerFactory == null) { throw new InvalidDataException("bad factory: " + factoryName); } using (ILogMedia fileMedia = await SimpleFileMedia.Create(fileSystem, SimpleFileMedia.CreateConnectionParamsFromFileName(@params.Location))) using (ILogSourceThreadsInternal threads = new LogSourceThreads()) using (var reader = readerFactory.CreateMessagesReader( new MediaBasedReaderParams(threads, fileMedia))) { var readerImpl = reader as MediaBasedPositionedMessagesReader; // todo: do not use real classes; have stream encoding in an interface. if (readerImpl == null) { throw new InvalidDataException("bad reader was made by factory " + factoryName); } await reader.UpdateAvailableBounds(false); var range = new FileRange.Range(reader.BeginPosition, reader.EndPosition); double rangeLen = range.Length; using (var progress = progressAggregator.CreateProgressSink()) using (var writer = new StreamWriter(tmpFileName, false, readerImpl.StreamEncoding)) await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(reader.BeginPosition, flags: MessagesParserFlag.DisableDejitter | MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading)), async parser => { var queue = new VCSKicksCollection.PriorityQueue <IMessage>( new MessagesComparer(ignoreConnectionIds: true)); Action dequeue = () => writer.WriteLine(queue.Dequeue().RawText.ToString()); double lastPrctComplete = 0; var cancellation = callback.Cancellation; for (long msgIdx = 0;; ++msgIdx) { if (cancellation.IsCancellationRequested) { break; } var msg = await parser.ReadNext(); if (msg == null) { break; } if ((msgIdx % progressUpdateThreshold) == 0 && rangeLen > 0) { var prctComplete = (double)(msg.Position - range.Begin) / rangeLen; progress.SetValue(prctComplete); if (prctComplete - lastPrctComplete > 0.05) { setStepDescription(prctComplete); lastPrctComplete = prctComplete; } } queue.Enqueue(msg); if (queue.Count > queueSize) { dequeue(); } } while (queue.Count > 0) { dequeue(); } }); } return(new PreprocessingStepParams( tmpFileName, @params.FullPath + " (reordered)", @params.PreprocessingHistory.Add(new PreprocessingHistoryItem(name, factoryName)) )); }
public IEnumerable <MergingCollectionEntry> Forward(int startPos, int endPosition) { Lock(); try { int totalCount = 0; var queueComparer = new QueueEntriesComparer(reverse: false); var queue = new VCSKicksCollection.PriorityQueue <QueueEntry>(queueComparer); try { int collectionsCount = 0; foreach (IMessagesCollection l in GetCollectionsToMerge()) { ++collectionsCount; int localCount = l.Count; totalCount += localCount; IEnumerator <IndexedMessage> i = l.Forward(0, localCount).GetEnumerator(); if (i.MoveNext()) { queue.Enqueue(new QueueEntry(i, l)); } } startPos = RangeUtils.PutInRange(0, totalCount, startPos); endPosition = RangeUtils.PutInRange(0, totalCount, endPosition); if (collectionsCount == 1) // optimized version for the case when there is only one collection to merge { var entry = queue.Dequeue(); using (IEnumerator <IndexedMessage> i = entry.enumerator) { for (int idx = 0; idx < endPosition; ++idx) { if (idx >= startPos) { yield return(new MergingCollectionEntry( new IndexedMessage(idx, i.Current.Message), entry.collection, i.Current.Index)); } if (!i.MoveNext()) { break; } } } } else { for (int idx = 0; idx < endPosition; ++idx) { var entry = queue.Dequeue(); var i = entry.enumerator; try { if (idx >= startPos) { yield return(new MergingCollectionEntry( new IndexedMessage(idx, i.Current.Message), entry.collection, i.Current.Index)); } if (i.MoveNext()) { queue.Enqueue(entry); i = null; } } finally { if (i != null) { i.Dispose(); } } } } } finally { while (queue.Count != 0) { queue.Dequeue().enumerator.Dispose(); } } } finally { Unlock(); } }
public IEnumerable <MergingCollectionEntry> Reverse(int startPos, int endPosition) { Lock(); try { var queueComparer = new QueueEntriesComparer(reverse: true); var queue = new VCSKicksCollection.PriorityQueue <QueueEntry>(queueComparer); try { int collectionsCount = 0; int c = 0; foreach (IMessagesCollection l in GetCollectionsToMerge()) { ++collectionsCount; int lc = l.Count; c += lc; IEnumerator <IndexedMessage> i = l.Reverse(lc - 1, -1).GetEnumerator(); if (i.MoveNext()) { queue.Enqueue(new QueueEntry(i, l)); } } startPos = RangeUtils.PutInRange(-1, c - 1, startPos); endPosition = RangeUtils.PutInRange(-1, c - 1, endPosition); for (int idx = c - 1; idx > endPosition; --idx) { var entry = queue.Dequeue(); var i = entry.enumerator; try { if (idx <= startPos) { yield return(new MergingCollectionEntry( new IndexedMessage(idx, i.Current.Message), entry.collection, i.Current.Index)); } if (i.MoveNext()) { queue.Enqueue(entry); i = null; } } finally { if (i != null) { i.Dispose(); } } } } finally { while (queue.Count != 0) { queue.Dequeue().enumerator.Dispose(); } } } finally { Unlock(); } }