Beispiel #1
0
        public static IEnumerable <PostprocessedMessage> MergePostprocessedMessage(IEnumerable <PostprocessedMessage>[] enums)
        {
            var comparer = new EnumeratorsComparer();
            var iters    = new VCSKicksCollection.PriorityQueue <IEnumerator <PostprocessedMessage> >(comparer);

            try
            {
                foreach (var e in enums)
                {
                    var i = e.GetEnumerator();
                    if (i.MoveNext())
                    {
                        iters.Enqueue(i);
                    }
                }
                for (; iters.Count > 0;)
                {
                    var i = iters.Dequeue();
                    try
                    {
                        yield return(i.Current);

                        if (i.MoveNext())
                        {
                            iters.Enqueue(i);
                            i = null;
                        }
                    }
                    finally
                    {
                        if (i != null)
                        {
                            i.Dispose();
                        }
                    }
                }
            }
            finally
            {
                while (iters.Count != 0)
                {
                    iters.Dequeue().Dispose();
                }
            }
        }
Beispiel #2
0
        static async Task EnumMessagesAndMerge(
            ILogSource[] sources,
            Action <IMessage> callback,
            Progress.IProgressAggregator progress,
            CancellationToken cancellation
            )
        {
            var queue = new VCSKicksCollection.PriorityQueue <EnumMessagesHelper>(
                new EnumMessagesHelper.Comparer());
            var helpers = sources.Select(s => new EnumMessagesHelper(
                                             s, cancellation, progress.CreateProgressSink())).ToList();

            try
            {
                await Task.WhenAll(helpers.Select(h => h.FillBuffer()));

                Action <EnumMessagesHelper> enqueueOfKill = h =>
                {
                    if (h.Peek() != null)
                    {
                        queue.Enqueue(h);
                    }
                    else
                    {
                        h.Dispose();
                    }
                };
                helpers.ForEach(enqueueOfKill);
                while (queue.Count > 0)
                {
                    var h = queue.Dequeue();
                    callback(h.Peek());
                    h.Dequeue();
                    if (h.Peek() == null)
                    {
                        await h.FillBuffer();
                    }
                    enqueueOfKill(h);
                }
            }
            finally
            {
                helpers.ForEach(h => h.Dispose());
            }
        }
Beispiel #3
0
        public static IEnumerable <T> MergeSortedSequences <T>(this IEnumerable <T>[] enums, IComparer <T> valueComparer)
        {
            var comparer = new EnumeratorsComparer <T>(valueComparer);
            var iters    = new VCSKicksCollection.PriorityQueue <IEnumerator <T> >(comparer);

            try
            {
                foreach (var e in enums)
                {
                    var i = e.GetEnumerator();
                    if (i.MoveNext())
                    {
                        iters.Enqueue(i);
                    }
                }
                for (; iters.Count > 0;)
                {
                    var i = iters.Dequeue();
                    try
                    {
                        yield return(i.Current);

                        if (i.MoveNext())
                        {
                            iters.Enqueue(i);
                            i = null;
                        }
                    }
                    finally
                    {
                        if (i != null)
                        {
                            i.Dispose();
                        }
                    }
                }
            }
            finally
            {
                while (iters.Count != 0)
                {
                    iters.Dequeue().Dispose();
                }
            }
        }
Beispiel #4
0
        /// <summary>
        /// Makes sure events have correct timestamp and go in correct order.
        /// Chromedriver log has a problem that timestamps at beginning of lines like [1525678451.879]
        /// seem to be rounded up to next 100ms boundary which results to inaccurate views.
        /// Most important messages like Network.requestWillBeSent have "timestamp" as a json field.
        /// That "timestamp" is nr os seconds from unknown origin. Luckily some messages also have
        /// "wallTime" in json payload that can help interpret "timestamp".
        /// Another problem is that "timestamp" of Network.requestWillBeSent might not match
        /// timing.requestTime in Network.responseReceived. The latter seems to be more accurate.
        /// However if a request is served from cache its timing.requestTime is totally wrong and
        /// should be ignored in favor of  Network.requestWillBeSent's "timestamp".
        /// </summary>
        static IEnumerableAsync <Message[]> FixTimestamps(IEnumerableAsync <Message[]> messages)
        {
            DateTime?timestampBase            = null;
            var      pendingMessages          = new List <MessageEntry>();
            var      queue                    = new VCSKicksCollection.PriorityQueue <MessageEntry>(new Comparer());
            var      queuedRequestStarts      = new Dictionary <string, MessageEntry>();
            double?  lastDequeuedTimestamp    = null;
            Action <Queue <Message> > dequeue = (outputQueue) =>
            {
                var entry = queue.Dequeue();
                if (!entry.IsInvalidated)
                {
                    if (lastDequeuedTimestamp == null || entry.Timestamp != null)
                    {
                        lastDequeuedTimestamp = entry.Timestamp;
                    }
                    outputQueue.Enqueue(entry.Msg);
                    if (entry.Type == MessageEntry.EntryType.StartRequest)
                    {
                        queuedRequestStarts.Remove(entry.RequestId);
                    }
                }
            };
            Action <MessageEntry> enqueue = null;

            enqueue = (r) =>
            {
                var rebased = r.Rebase(timestampBase.Value);
                queue.Enqueue(rebased);
                if (r.Type == MessageEntry.EntryType.StartRequest)
                {
                    queuedRequestStarts[r.RequestId] = rebased;
                }
                else if (r.Type == MessageEntry.EntryType.ResponseWithTiming)
                {
                    MessageEntry queuedRequest;
                    if (queuedRequestStarts.TryGetValue(r.RequestId, out queuedRequest) && !queuedRequest.IsInvalidated && !queuedRequest.IsServedFromCache)
                    {
                        if (lastDequeuedTimestamp == null || r.ResponseTiming_RequestTime > lastDequeuedTimestamp.Value)
                        {
                            enqueue(queuedRequest.InvalidateAndMakeFixedStartRequest(r.ResponseTiming_RequestTime));
                        }
                    }
                }
                else if (r.Type == MessageEntry.EntryType.ServedFromCache)
                {
                    MessageEntry queuedRequest;
                    if (queuedRequestStarts.TryGetValue(r.RequestId, out queuedRequest))
                    {
                        queuedRequest.IsServedFromCache = true;
                    }
                }
            };
            Action flushPendingMessages = () =>
            {
                foreach (var pendingMessage in pendingMessages)
                {
                    enqueue(pendingMessage);
                }
                pendingMessages.Clear();
            };
            int queueSize = 4096;

            return(messages.Select <Message, Message>((m, outputQueue) => {
                var newEntry = new MessageEntry(m);

                if (timestampBase == null && newEntry.Timestamp != null && newEntry.WallTime != null)
                {
                    timestampBase = TimeUtils.UnixTimestampMillisToDateTime(
                        newEntry.WallTime.Value * 1000d).ToUnspecifiedTime().AddSeconds(
                        -newEntry.Timestamp.Value);
                    flushPendingMessages();
                }

                if (timestampBase == null)
                {
                    pendingMessages.Add(newEntry);
                }
                else
                {
                    enqueue(newEntry);
                }

                if (queue.Count >= queueSize * 2)
                {
                    while (queue.Count > queueSize)
                    {
                        dequeue(outputQueue);
                    }
                }
            },
                                                      (outputQueue) =>
            {
                while (queue.Count > 0)
                {
                    dequeue(outputQueue);
                }
            }));
        }
        public static async Task SerializePostprocessorOutput <Evt, Serializer, EvtVisitor>(
            this IEnumerableAsync <Evt[]> events,
            Func <Action <object, XElement>, Serializer> serializerFactory,
            Task <ILogPartToken> rotatedLogPartToken,
            ILogPartTokenFactories rotatedLogPartFactories,
            Func <object, TextLogEventTrigger> triggersConverter,
            string contentsEtagAttr,
            string rootElementName,
            string outputFileName,
            ITempFilesManager tempFiles,
            CancellationToken cancellation
            ) where Evt : IVisitable <EvtVisitor> where Serializer : class, IEventsSerializer, EvtVisitor
        {
            rotatedLogPartToken = rotatedLogPartToken ?? Task.FromResult <ILogPartToken>(null);
            var        sortKeyAttr     = XName.Get("__key");
            var        chunks          = new List <string>();
            Serializer serializer      = null;
            Action     resetSerializer = () =>
            {
                if (serializer?.Output?.Count > 0)
                {
                    string chunkFileName = tempFiles.GenerateNewName();
                    chunks.Add(chunkFileName);
                    using (var writer = XmlWriter.Create(chunkFileName, new XmlWriterSettings()
                    {
                        OmitXmlDeclaration = true,
                        ConformanceLevel = ConformanceLevel.Fragment
                    }))
                    {
                        foreach (var e in serializer.Output.OrderBy(e => e.Attribute(sortKeyAttr).Value))
                        {
                            e.WriteTo(writer);
                        }
                    }
                }
                serializer = serializerFactory((trigger, elt) =>
                {
                    triggersConverter(trigger).Save(elt);
                    elt.SetAttributeValue(sortKeyAttr, ((IOrderedTrigger)trigger).Index.ToString("x8"));
                });
            };

            resetSerializer();
            await events.ForEach(batch =>
            {
                foreach (var e in batch)
                {
                    e.Visit(serializer);
                    if (serializer.Output.Count >= 8 * 1024)
                    {
                        resetSerializer();
                    }
                }
                return(Task.FromResult(!cancellation.IsCancellationRequested));
            });

            resetSerializer();

            if (cancellation.IsCancellationRequested)
            {
                return;
            }

            using (var outputWriter = XmlWriter.Create(outputFileName, new XmlWriterSettings()
            {
                Indent = true
            }))
            {
                outputWriter.WriteStartElement(rootElementName);
                new PostprocessorOutputETag(contentsEtagAttr).Write(outputWriter);
                rotatedLogPartFactories.SafeWriteTo(await rotatedLogPartToken, outputWriter);
                var readersSettings = new XmlReaderSettings()
                {
                    ConformanceLevel = ConformanceLevel.Fragment
                };
                var readers = chunks.Select(chunkFileName => XmlReader.Create(chunkFileName, readersSettings)).ToList();
                try
                {
                    var q = new VCSKicksCollection.PriorityQueue <KeyValuePair <XmlReader, XElement> >(Comparer <KeyValuePair <XmlReader, XElement> > .Create((item1, item2) =>
                    {
                        return(string.CompareOrdinal(item1.Value.Attribute(sortKeyAttr).Value, item2.Value.Attribute(sortKeyAttr).Value));
                    }));
                    Action <XmlReader> enqueueReader = reader =>
                    {
                        if (!reader.EOF)
                        {
                            if (reader.MoveToContent() != XmlNodeType.Element)
                            {
                                throw new InvalidOperationException("bad chunk");
                            }
                            q.Enqueue(new KeyValuePair <XmlReader, XElement>(reader, (XElement)XNode.ReadFrom(reader)));
                        }
                    };
                    readers.ForEach(enqueueReader);
                    while (q.Count > 0)
                    {
                        var item = q.Dequeue();
                        item.Value.Attribute(sortKeyAttr).Remove();
                        item.Value.WriteTo(outputWriter);
                        enqueueReader(item.Key);
                    }
                }
                finally
                {
                    readers.ForEach(r => r.Dispose());
                    chunks.ForEach(chunkFileName => tempFiles.DeleteIfTemporary(chunkFileName));
                }
                outputWriter.WriteEndElement();                 // end of root node
            }
        }
        async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
        {
            await callback.BecomeLongRunning();

            string factoryName = @params.Argument;

            callback.TempFilesCleanupList.Add(@params.Location);
            Action <double?> setStepDescription = prctComplete =>
            {
                var str = new StringBuilder();
                str.Append(@params.FullPath);
                str.Append(": fixing timestamp anomalies...");
                if (prctComplete != null)
                {
                    str.AppendFormat(" {0}%", (int)(prctComplete.Value * 100));
                }
                callback.SetStepDescription(str.ToString());
            };

            setStepDescription(null);

            string tmpFileName = callback.TempFilesManager.GenerateNewName();

            var factoryNameSplit = factoryName.Split('\\');

            if (factoryNameSplit.Length != 2)
            {
                throw new InvalidFormatException();
            }
            var factory = logProviderFactoryRegistry.Find(factoryNameSplit[0], factoryNameSplit[1]);

            if (factory == null)
            {
                throw new InvalidDataException("factory not found: " + factoryName);
            }
            var readerFactory = factory as IMediaBasedReaderFactory;

            if (readerFactory == null)
            {
                throw new InvalidDataException("bad factory: " + factoryName);
            }
            using (ILogMedia fileMedia = await SimpleFileMedia.Create(fileSystem,
                                                                      SimpleFileMedia.CreateConnectionParamsFromFileName(@params.Location)))
                using (ILogSourceThreadsInternal threads = new LogSourceThreads())
                    using (var reader = readerFactory.CreateMessagesReader(
                               new MediaBasedReaderParams(threads, fileMedia)))
                    {
                        var readerImpl = reader as MediaBasedPositionedMessagesReader;         // todo: do not use real classes; have stream encoding in an interface.
                        if (readerImpl == null)
                        {
                            throw new InvalidDataException("bad reader was made by factory " + factoryName);
                        }
                        await reader.UpdateAvailableBounds(false);

                        var    range    = new FileRange.Range(reader.BeginPosition, reader.EndPosition);
                        double rangeLen = range.Length;
                        using (var progress = progressAggregator.CreateProgressSink())
                            using (var writer = new StreamWriter(tmpFileName, false, readerImpl.StreamEncoding))
                                await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(reader.BeginPosition,
                                                                                                             flags: MessagesParserFlag.DisableDejitter | MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading)), async parser =>
                                {
                                    var queue = new VCSKicksCollection.PriorityQueue <IMessage>(
                                        new MessagesComparer(ignoreConnectionIds: true));
                                    Action dequeue          = () => writer.WriteLine(queue.Dequeue().RawText.ToString());
                                    double lastPrctComplete = 0;
                                    var cancellation        = callback.Cancellation;
                                    for (long msgIdx = 0;; ++msgIdx)
                                    {
                                        if (cancellation.IsCancellationRequested)
                                        {
                                            break;
                                        }
                                        var msg = await parser.ReadNext();
                                        if (msg == null)
                                        {
                                            break;
                                        }
                                        if ((msgIdx % progressUpdateThreshold) == 0 && rangeLen > 0)
                                        {
                                            var prctComplete = (double)(msg.Position - range.Begin) / rangeLen;
                                            progress.SetValue(prctComplete);
                                            if (prctComplete - lastPrctComplete > 0.05)
                                            {
                                                setStepDescription(prctComplete);
                                                lastPrctComplete = prctComplete;
                                            }
                                        }
                                        queue.Enqueue(msg);
                                        if (queue.Count > queueSize)
                                        {
                                            dequeue();
                                        }
                                    }
                                    while (queue.Count > 0)
                                    {
                                        dequeue();
                                    }
                                });
                    }

            return(new PreprocessingStepParams(
                       tmpFileName,
                       @params.FullPath + " (reordered)",
                       @params.PreprocessingHistory.Add(new PreprocessingHistoryItem(name, factoryName))
                       ));
        }
        public IEnumerable <MergingCollectionEntry> Forward(int startPos, int endPosition)
        {
            Lock();
            try
            {
                int totalCount    = 0;
                var queueComparer = new QueueEntriesComparer(reverse: false);
                var queue         = new VCSKicksCollection.PriorityQueue <QueueEntry>(queueComparer);
                try
                {
                    int collectionsCount = 0;
                    foreach (IMessagesCollection l in GetCollectionsToMerge())
                    {
                        ++collectionsCount;
                        int localCount = l.Count;
                        totalCount += localCount;
                        IEnumerator <IndexedMessage> i = l.Forward(0, localCount).GetEnumerator();
                        if (i.MoveNext())
                        {
                            queue.Enqueue(new QueueEntry(i, l));
                        }
                    }
                    startPos    = RangeUtils.PutInRange(0, totalCount, startPos);
                    endPosition = RangeUtils.PutInRange(0, totalCount, endPosition);

                    if (collectionsCount == 1)                     // optimized version for the case when there is only one collection to merge
                    {
                        var entry = queue.Dequeue();
                        using (IEnumerator <IndexedMessage> i = entry.enumerator)
                        {
                            for (int idx = 0; idx < endPosition; ++idx)
                            {
                                if (idx >= startPos)
                                {
                                    yield return(new MergingCollectionEntry(
                                                     new IndexedMessage(idx, i.Current.Message), entry.collection, i.Current.Index));
                                }
                                if (!i.MoveNext())
                                {
                                    break;
                                }
                            }
                        }
                    }
                    else
                    {
                        for (int idx = 0; idx < endPosition; ++idx)
                        {
                            var entry = queue.Dequeue();
                            var i     = entry.enumerator;
                            try
                            {
                                if (idx >= startPos)
                                {
                                    yield return(new MergingCollectionEntry(
                                                     new IndexedMessage(idx, i.Current.Message), entry.collection, i.Current.Index));
                                }
                                if (i.MoveNext())
                                {
                                    queue.Enqueue(entry);
                                    i = null;
                                }
                            }
                            finally
                            {
                                if (i != null)
                                {
                                    i.Dispose();
                                }
                            }
                        }
                    }
                }
                finally
                {
                    while (queue.Count != 0)
                    {
                        queue.Dequeue().enumerator.Dispose();
                    }
                }
            }
            finally
            {
                Unlock();
            }
        }
 public IEnumerable <MergingCollectionEntry> Reverse(int startPos, int endPosition)
 {
     Lock();
     try
     {
         var queueComparer = new QueueEntriesComparer(reverse: true);
         var queue         = new VCSKicksCollection.PriorityQueue <QueueEntry>(queueComparer);
         try
         {
             int collectionsCount = 0;
             int c = 0;
             foreach (IMessagesCollection l in GetCollectionsToMerge())
             {
                 ++collectionsCount;
                 int lc = l.Count;
                 c += lc;
                 IEnumerator <IndexedMessage> i = l.Reverse(lc - 1, -1).GetEnumerator();
                 if (i.MoveNext())
                 {
                     queue.Enqueue(new QueueEntry(i, l));
                 }
             }
             startPos    = RangeUtils.PutInRange(-1, c - 1, startPos);
             endPosition = RangeUtils.PutInRange(-1, c - 1, endPosition);
             for (int idx = c - 1; idx > endPosition; --idx)
             {
                 var entry = queue.Dequeue();
                 var i     = entry.enumerator;
                 try
                 {
                     if (idx <= startPos)
                     {
                         yield return(new MergingCollectionEntry(
                                          new IndexedMessage(idx, i.Current.Message), entry.collection, i.Current.Index));
                     }
                     if (i.MoveNext())
                     {
                         queue.Enqueue(entry);
                         i = null;
                     }
                 }
                 finally
                 {
                     if (i != null)
                     {
                         i.Dispose();
                     }
                 }
             }
         }
         finally
         {
             while (queue.Count != 0)
             {
                 queue.Dequeue().enumerator.Dispose();
             }
         }
     }
     finally
     {
         Unlock();
     }
 }