public static IEnumerable <PostprocessedMessage> MergePostprocessedMessage(IEnumerable <PostprocessedMessage>[] enums)
{
var comparer = new EnumeratorsComparer();
var iters = new VCSKicksCollection.PriorityQueue <IEnumerator <PostprocessedMessage> >(comparer);
try
{
foreach (var e in enums)
{
var i = e.GetEnumerator();
if (i.MoveNext())
{
iters.Enqueue(i);
}
}
for (; iters.Count > 0;)
{
var i = iters.Dequeue();
try
{
yield return(i.Current);
if (i.MoveNext())
{
iters.Enqueue(i);
i = null;
}
}
finally
{
if (i != null)
{
i.Dispose();
}
}
}
}
finally
{
while (iters.Count != 0)
{
iters.Dequeue().Dispose();
}
}
}
static async Task EnumMessagesAndMerge(
ILogSource[] sources,
Action <IMessage> callback,
Progress.IProgressAggregator progress,
CancellationToken cancellation
)
{
var queue = new VCSKicksCollection.PriorityQueue <EnumMessagesHelper>(
new EnumMessagesHelper.Comparer());
var helpers = sources.Select(s => new EnumMessagesHelper(
s, cancellation, progress.CreateProgressSink())).ToList();
try
{
await Task.WhenAll(helpers.Select(h => h.FillBuffer()));
Action <EnumMessagesHelper> enqueueOfKill = h =>
{
if (h.Peek() != null)
{
queue.Enqueue(h);
}
else
{
h.Dispose();
}
};
helpers.ForEach(enqueueOfKill);
while (queue.Count > 0)
{
var h = queue.Dequeue();
callback(h.Peek());
h.Dequeue();
if (h.Peek() == null)
{
await h.FillBuffer();
}
enqueueOfKill(h);
}
}
finally
{
helpers.ForEach(h => h.Dispose());
}
}
public static IEnumerable <T> MergeSortedSequences <T>(this IEnumerable <T>[] enums, IComparer <T> valueComparer)
{
var comparer = new EnumeratorsComparer <T>(valueComparer);
var iters = new VCSKicksCollection.PriorityQueue <IEnumerator <T> >(comparer);
try
{
foreach (var e in enums)
{
var i = e.GetEnumerator();
if (i.MoveNext())
{
iters.Enqueue(i);
}
}
for (; iters.Count > 0;)
{
var i = iters.Dequeue();
try
{
yield return(i.Current);
if (i.MoveNext())
{
iters.Enqueue(i);
i = null;
}
}
finally
{
if (i != null)
{
i.Dispose();
}
}
}
}
finally
{
while (iters.Count != 0)
{
iters.Dequeue().Dispose();
}
}
}
/// <summary>
/// Makes sure events have correct timestamp and go in correct order.
/// Chromedriver log has a problem that timestamps at beginning of lines like [1525678451.879]
/// seem to be rounded up to next 100ms boundary which results to inaccurate views.
/// Most important messages like Network.requestWillBeSent have "timestamp" as a json field.
/// That "timestamp" is nr os seconds from unknown origin. Luckily some messages also have
/// "wallTime" in json payload that can help interpret "timestamp".
/// Another problem is that "timestamp" of Network.requestWillBeSent might not match
/// timing.requestTime in Network.responseReceived. The latter seems to be more accurate.
/// However if a request is served from cache its timing.requestTime is totally wrong and
/// should be ignored in favor of Network.requestWillBeSent's "timestamp".
/// </summary>
static IEnumerableAsync <Message[]> FixTimestamps(IEnumerableAsync <Message[]> messages)
{
DateTime?timestampBase = null;
var pendingMessages = new List <MessageEntry>();
var queue = new VCSKicksCollection.PriorityQueue <MessageEntry>(new Comparer());
var queuedRequestStarts = new Dictionary <string, MessageEntry>();
double? lastDequeuedTimestamp = null;
Action <Queue <Message> > dequeue = (outputQueue) =>
{
var entry = queue.Dequeue();
if (!entry.IsInvalidated)
{
if (lastDequeuedTimestamp == null || entry.Timestamp != null)
{
lastDequeuedTimestamp = entry.Timestamp;
}
outputQueue.Enqueue(entry.Msg);
if (entry.Type == MessageEntry.EntryType.StartRequest)
{
queuedRequestStarts.Remove(entry.RequestId);
}
}
};
Action <MessageEntry> enqueue = null;
enqueue = (r) =>
{
var rebased = r.Rebase(timestampBase.Value);
queue.Enqueue(rebased);
if (r.Type == MessageEntry.EntryType.StartRequest)
{
queuedRequestStarts[r.RequestId] = rebased;
}
else if (r.Type == MessageEntry.EntryType.ResponseWithTiming)
{
MessageEntry queuedRequest;
if (queuedRequestStarts.TryGetValue(r.RequestId, out queuedRequest) && !queuedRequest.IsInvalidated && !queuedRequest.IsServedFromCache)
{
if (lastDequeuedTimestamp == null || r.ResponseTiming_RequestTime > lastDequeuedTimestamp.Value)
{
enqueue(queuedRequest.InvalidateAndMakeFixedStartRequest(r.ResponseTiming_RequestTime));
}
}
}
else if (r.Type == MessageEntry.EntryType.ServedFromCache)
{
MessageEntry queuedRequest;
if (queuedRequestStarts.TryGetValue(r.RequestId, out queuedRequest))
{
queuedRequest.IsServedFromCache = true;
}
}
};
Action flushPendingMessages = () =>
{
foreach (var pendingMessage in pendingMessages)
{
enqueue(pendingMessage);
}
pendingMessages.Clear();
};
int queueSize = 4096;
return(messages.Select <Message, Message>((m, outputQueue) => {
var newEntry = new MessageEntry(m);
if (timestampBase == null && newEntry.Timestamp != null && newEntry.WallTime != null)
{
timestampBase = TimeUtils.UnixTimestampMillisToDateTime(
newEntry.WallTime.Value * 1000d).ToUnspecifiedTime().AddSeconds(
-newEntry.Timestamp.Value);
flushPendingMessages();
}
if (timestampBase == null)
{
pendingMessages.Add(newEntry);
}
else
{
enqueue(newEntry);
}
if (queue.Count >= queueSize * 2)
{
while (queue.Count > queueSize)
{
dequeue(outputQueue);
}
}
},
(outputQueue) =>
{
while (queue.Count > 0)
{
dequeue(outputQueue);
}
}));
}
public static async Task SerializePostprocessorOutput <Evt, Serializer, EvtVisitor>(
this IEnumerableAsync <Evt[]> events,
Func <Action <object, XElement>, Serializer> serializerFactory,
Task <ILogPartToken> rotatedLogPartToken,
ILogPartTokenFactories rotatedLogPartFactories,
Func <object, TextLogEventTrigger> triggersConverter,
string contentsEtagAttr,
string rootElementName,
string outputFileName,
ITempFilesManager tempFiles,
CancellationToken cancellation
) where Evt : IVisitable <EvtVisitor> where Serializer : class, IEventsSerializer, EvtVisitor
{
rotatedLogPartToken = rotatedLogPartToken ?? Task.FromResult <ILogPartToken>(null);
var sortKeyAttr = XName.Get("__key");
var chunks = new List <string>();
Serializer serializer = null;
Action resetSerializer = () =>
{
if (serializer?.Output?.Count > 0)
{
string chunkFileName = tempFiles.GenerateNewName();
chunks.Add(chunkFileName);
using (var writer = XmlWriter.Create(chunkFileName, new XmlWriterSettings()
{
OmitXmlDeclaration = true,
ConformanceLevel = ConformanceLevel.Fragment
}))
{
foreach (var e in serializer.Output.OrderBy(e => e.Attribute(sortKeyAttr).Value))
{
e.WriteTo(writer);
}
}
}
serializer = serializerFactory((trigger, elt) =>
{
triggersConverter(trigger).Save(elt);
elt.SetAttributeValue(sortKeyAttr, ((IOrderedTrigger)trigger).Index.ToString("x8"));
});
};
resetSerializer();
await events.ForEach(batch =>
{
foreach (var e in batch)
{
e.Visit(serializer);
if (serializer.Output.Count >= 8 * 1024)
{
resetSerializer();
}
}
return(Task.FromResult(!cancellation.IsCancellationRequested));
});
resetSerializer();
if (cancellation.IsCancellationRequested)
{
return;
}
using (var outputWriter = XmlWriter.Create(outputFileName, new XmlWriterSettings()
{
Indent = true
}))
{
outputWriter.WriteStartElement(rootElementName);
new PostprocessorOutputETag(contentsEtagAttr).Write(outputWriter);
rotatedLogPartFactories.SafeWriteTo(await rotatedLogPartToken, outputWriter);
var readersSettings = new XmlReaderSettings()
{
ConformanceLevel = ConformanceLevel.Fragment
};
var readers = chunks.Select(chunkFileName => XmlReader.Create(chunkFileName, readersSettings)).ToList();
try
{
var q = new VCSKicksCollection.PriorityQueue <KeyValuePair <XmlReader, XElement> >(Comparer <KeyValuePair <XmlReader, XElement> > .Create((item1, item2) =>
{
return(string.CompareOrdinal(item1.Value.Attribute(sortKeyAttr).Value, item2.Value.Attribute(sortKeyAttr).Value));
}));
Action <XmlReader> enqueueReader = reader =>
{
if (!reader.EOF)
{
if (reader.MoveToContent() != XmlNodeType.Element)
{
throw new InvalidOperationException("bad chunk");
}
q.Enqueue(new KeyValuePair <XmlReader, XElement>(reader, (XElement)XNode.ReadFrom(reader)));
}
};
readers.ForEach(enqueueReader);
while (q.Count > 0)
{
var item = q.Dequeue();
item.Value.Attribute(sortKeyAttr).Remove();
item.Value.WriteTo(outputWriter);
enqueueReader(item.Key);
}
}
finally
{
readers.ForEach(r => r.Dispose());
chunks.ForEach(chunkFileName => tempFiles.DeleteIfTemporary(chunkFileName));
}
outputWriter.WriteEndElement(); // end of root node
}
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
string factoryName = @params.Argument;
callback.TempFilesCleanupList.Add(@params.Location);
Action <double?> setStepDescription = prctComplete =>
{
var str = new StringBuilder();
str.Append(@params.FullPath);
str.Append(": fixing timestamp anomalies...");
if (prctComplete != null)
{
str.AppendFormat(" {0}%", (int)(prctComplete.Value * 100));
}
callback.SetStepDescription(str.ToString());
};
setStepDescription(null);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
var factoryNameSplit = factoryName.Split('\\');
if (factoryNameSplit.Length != 2)
{
throw new InvalidFormatException();
}
var factory = logProviderFactoryRegistry.Find(factoryNameSplit[0], factoryNameSplit[1]);
if (factory == null)
{
throw new InvalidDataException("factory not found: " + factoryName);
}
var readerFactory = factory as IMediaBasedReaderFactory;
if (readerFactory == null)
{
throw new InvalidDataException("bad factory: " + factoryName);
}
using (ILogMedia fileMedia = await SimpleFileMedia.Create(fileSystem,
SimpleFileMedia.CreateConnectionParamsFromFileName(@params.Location)))
using (ILogSourceThreadsInternal threads = new LogSourceThreads())
using (var reader = readerFactory.CreateMessagesReader(
new MediaBasedReaderParams(threads, fileMedia)))
{
var readerImpl = reader as MediaBasedPositionedMessagesReader; // todo: do not use real classes; have stream encoding in an interface.
if (readerImpl == null)
{
throw new InvalidDataException("bad reader was made by factory " + factoryName);
}
await reader.UpdateAvailableBounds(false);
var range = new FileRange.Range(reader.BeginPosition, reader.EndPosition);
double rangeLen = range.Length;
using (var progress = progressAggregator.CreateProgressSink())
using (var writer = new StreamWriter(tmpFileName, false, readerImpl.StreamEncoding))
await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(reader.BeginPosition,
flags: MessagesParserFlag.DisableDejitter | MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading)), async parser =>
{
var queue = new VCSKicksCollection.PriorityQueue <IMessage>(
new MessagesComparer(ignoreConnectionIds: true));
Action dequeue = () => writer.WriteLine(queue.Dequeue().RawText.ToString());
double lastPrctComplete = 0;
var cancellation = callback.Cancellation;
for (long msgIdx = 0;; ++msgIdx)
{
if (cancellation.IsCancellationRequested)
{
break;
}
var msg = await parser.ReadNext();
if (msg == null)
{
break;
}
if ((msgIdx % progressUpdateThreshold) == 0 && rangeLen > 0)
{
var prctComplete = (double)(msg.Position - range.Begin) / rangeLen;
progress.SetValue(prctComplete);
if (prctComplete - lastPrctComplete > 0.05)
{
setStepDescription(prctComplete);
lastPrctComplete = prctComplete;
}
}
queue.Enqueue(msg);
if (queue.Count > queueSize)
{
dequeue();
}
}
while (queue.Count > 0)
{
dequeue();
}
});
}
return(new PreprocessingStepParams(
tmpFileName,
@params.FullPath + " (reordered)",
@params.PreprocessingHistory.Add(new PreprocessingHistoryItem(name, factoryName))
));
}
public IEnumerable <MergingCollectionEntry> Forward(int startPos, int endPosition)
{
Lock();
try
{
int totalCount = 0;
var queueComparer = new QueueEntriesComparer(reverse: false);
var queue = new VCSKicksCollection.PriorityQueue <QueueEntry>(queueComparer);
try
{
int collectionsCount = 0;
foreach (IMessagesCollection l in GetCollectionsToMerge())
{
++collectionsCount;
int localCount = l.Count;
totalCount += localCount;
IEnumerator <IndexedMessage> i = l.Forward(0, localCount).GetEnumerator();
if (i.MoveNext())
{
queue.Enqueue(new QueueEntry(i, l));
}
}
startPos = RangeUtils.PutInRange(0, totalCount, startPos);
endPosition = RangeUtils.PutInRange(0, totalCount, endPosition);
if (collectionsCount == 1) // optimized version for the case when there is only one collection to merge
{
var entry = queue.Dequeue();
using (IEnumerator <IndexedMessage> i = entry.enumerator)
{
for (int idx = 0; idx < endPosition; ++idx)
{
if (idx >= startPos)
{
yield return(new MergingCollectionEntry(
new IndexedMessage(idx, i.Current.Message), entry.collection, i.Current.Index));
}
if (!i.MoveNext())
{
break;
}
}
}
}
else
{
for (int idx = 0; idx < endPosition; ++idx)
{
var entry = queue.Dequeue();
var i = entry.enumerator;
try
{
if (idx >= startPos)
{
yield return(new MergingCollectionEntry(
new IndexedMessage(idx, i.Current.Message), entry.collection, i.Current.Index));
}
if (i.MoveNext())
{
queue.Enqueue(entry);
i = null;
}
}
finally
{
if (i != null)
{
i.Dispose();
}
}
}
}
}
finally
{
while (queue.Count != 0)
{
queue.Dequeue().enumerator.Dispose();
}
}
}
finally
{
Unlock();
}
}
public IEnumerable <MergingCollectionEntry> Reverse(int startPos, int endPosition)
{
Lock();
try
{
var queueComparer = new QueueEntriesComparer(reverse: true);
var queue = new VCSKicksCollection.PriorityQueue <QueueEntry>(queueComparer);
try
{
int collectionsCount = 0;
int c = 0;
foreach (IMessagesCollection l in GetCollectionsToMerge())
{
++collectionsCount;
int lc = l.Count;
c += lc;
IEnumerator <IndexedMessage> i = l.Reverse(lc - 1, -1).GetEnumerator();
if (i.MoveNext())
{
queue.Enqueue(new QueueEntry(i, l));
}
}
startPos = RangeUtils.PutInRange(-1, c - 1, startPos);
endPosition = RangeUtils.PutInRange(-1, c - 1, endPosition);
for (int idx = c - 1; idx > endPosition; --idx)
{
var entry = queue.Dequeue();
var i = entry.enumerator;
try
{
if (idx <= startPos)
{
yield return(new MergingCollectionEntry(
new IndexedMessage(idx, i.Current.Message), entry.collection, i.Current.Index));
}
if (i.MoveNext())
{
queue.Enqueue(entry);
i = null;
}
}
finally
{
if (i != null)
{
i.Dispose();
}
}
}
}
finally
{
while (queue.Count != 0)
{
queue.Dequeue().enumerator.Dispose();
}
}
}
finally
{
Unlock();
}
}