public static async Task SerializePostprocessorOutput(
Task <ILogPartToken> logPartToken,
ILogPartTokenFactories logPartTokenFactories,
IEnumerableAsync <M.Event[]> events,
Task <ISameNodeDetectionToken> sameNodeDetectionTokenTask,
ISameNodeDetectionTokenFactories nodeDetectionTokenFactories,
Func <object, TextLogEventTrigger> triggersConverter,
string contentsEtagAttr,
Func <Task <Stream> > openOutputStream,
ITempFilesManager tempFiles,
CancellationToken cancellation
)
{
events = events ?? new List <M.Event[]>().ToAsync();
logPartToken = logPartToken ?? Task.FromResult <ILogPartToken>(null);
sameNodeDetectionTokenTask = sameNodeDetectionTokenTask ?? Task.FromResult <ISameNodeDetectionToken>(null);
var eventsTmpFile = tempFiles.GenerateNewName();
Func <Task <Stream> > openTempFile(string fileName) => () => Task.FromResult <Stream>(new FileStream(fileName, FileMode.OpenOrCreate));
var serializeMessagingEvents = events.SerializePostprocessorOutput <M.Event, M.EventsSerializer, M.IEventsVisitor>(
triggerSerializer => new M.EventsSerializer(triggerSerializer),
null, logPartTokenFactories, triggersConverter, null, messagingEventsElementName, openTempFile(eventsTmpFile), tempFiles, cancellation
);
await Task.WhenAll(serializeMessagingEvents, logPartToken, sameNodeDetectionTokenTask);
using (var outputWriter = XmlWriter.Create(await openOutputStream(), new XmlWriterSettings()
{
Indent = true, Async = true, CloseOutput = true
}))
using (var messagingEventsReader = XmlReader.Create(eventsTmpFile))
{
outputWriter.WriteStartElement("root");
new PostprocessorOutputETag(contentsEtagAttr).Write(outputWriter);
logPartTokenFactories.SafeWriteTo(await logPartToken, outputWriter);
nodeDetectionTokenFactories.SafeWriteTo(await sameNodeDetectionTokenTask, outputWriter);
messagingEventsReader.ReadToFollowing(messagingEventsElementName);
await outputWriter.WriteNodeAsync(messagingEventsReader, false);
outputWriter.WriteEndElement(); // root
}
File.Delete(eventsTmpFile);
}
public static async Task SerializePostprocessorOutput <Evt, Serializer, EvtVisitor>(
this IEnumerableAsync <Evt[]> events,
Func <Action <object, XElement>, Serializer> serializerFactory,
Task <ILogPartToken> rotatedLogPartToken,
ILogPartTokenFactories rotatedLogPartFactories,
Func <object, TextLogEventTrigger> triggersConverter,
string contentsEtagAttr,
string rootElementName,
string outputFileName,
ITempFilesManager tempFiles,
CancellationToken cancellation
) where Evt : IVisitable <EvtVisitor> where Serializer : class, IEventsSerializer, EvtVisitor
{
rotatedLogPartToken = rotatedLogPartToken ?? Task.FromResult <ILogPartToken>(null);
var sortKeyAttr = XName.Get("__key");
var chunks = new List <string>();
Serializer serializer = null;
Action resetSerializer = () =>
{
if (serializer?.Output?.Count > 0)
{
string chunkFileName = tempFiles.GenerateNewName();
chunks.Add(chunkFileName);
using (var writer = XmlWriter.Create(chunkFileName, new XmlWriterSettings()
{
OmitXmlDeclaration = true,
ConformanceLevel = ConformanceLevel.Fragment
}))
{
foreach (var e in serializer.Output.OrderBy(e => e.Attribute(sortKeyAttr).Value))
{
e.WriteTo(writer);
}
}
}
serializer = serializerFactory((trigger, elt) =>
{
triggersConverter(trigger).Save(elt);
elt.SetAttributeValue(sortKeyAttr, ((IOrderedTrigger)trigger).Index.ToString("x8"));
});
};
resetSerializer();
await events.ForEach(batch =>
{
foreach (var e in batch)
{
e.Visit(serializer);
if (serializer.Output.Count >= 8 * 1024)
{
resetSerializer();
}
}
return(Task.FromResult(!cancellation.IsCancellationRequested));
});
resetSerializer();
if (cancellation.IsCancellationRequested)
{
return;
}
using (var outputWriter = XmlWriter.Create(outputFileName, new XmlWriterSettings()
{
Indent = true
}))
{
outputWriter.WriteStartElement(rootElementName);
new PostprocessorOutputETag(contentsEtagAttr).Write(outputWriter);
rotatedLogPartFactories.SafeWriteTo(await rotatedLogPartToken, outputWriter);
var readersSettings = new XmlReaderSettings()
{
ConformanceLevel = ConformanceLevel.Fragment
};
var readers = chunks.Select(chunkFileName => XmlReader.Create(chunkFileName, readersSettings)).ToList();
try
{
var q = new VCSKicksCollection.PriorityQueue <KeyValuePair <XmlReader, XElement> >(Comparer <KeyValuePair <XmlReader, XElement> > .Create((item1, item2) =>
{
return(string.CompareOrdinal(item1.Value.Attribute(sortKeyAttr).Value, item2.Value.Attribute(sortKeyAttr).Value));
}));
Action <XmlReader> enqueueReader = reader =>
{
if (!reader.EOF)
{
if (reader.MoveToContent() != XmlNodeType.Element)
{
throw new InvalidOperationException("bad chunk");
}
q.Enqueue(new KeyValuePair <XmlReader, XElement>(reader, (XElement)XNode.ReadFrom(reader)));
}
};
readers.ForEach(enqueueReader);
while (q.Count > 0)
{
var item = q.Dequeue();
item.Value.Attribute(sortKeyAttr).Remove();
item.Value.WriteTo(outputWriter);
enqueueReader(item.Key);
}
}
finally
{
readers.ForEach(r => r.Dispose());
chunks.ForEach(chunkFileName => tempFiles.DeleteIfTemporary(chunkFileName));
}
outputWriter.WriteEndElement(); // end of root node
}
}
public static async Task SerializePostprocessorOutput(
IEnumerableAsync <M.Event[]> events,
IEnumerableAsync <TLBlock.Event[]> timelineComments,
IEnumerableAsync <SIBlock.Event[]> stateInspectorComments,
Task <ILogPartToken> logPartToken,
ILogPartTokenFactories logPartTokenFactories,
Func <object, TextLogEventTrigger> triggersConverter,
string contentsEtagAttr,
string outputFileName,
ITempFilesManager tempFiles,
CancellationToken cancellation
)
{
events = events ?? new List <M.Event[]>().ToAsync();
timelineComments = timelineComments ?? new List <TLBlock.Event[]>().ToAsync();
stateInspectorComments = stateInspectorComments ?? new List <SIBlock.Event[]>().ToAsync();
logPartToken = logPartToken ?? Task.FromResult <ILogPartToken>(null);
var eventsTmpFile = tempFiles.GenerateNewName();
var timelineCommentsTmpFile = tempFiles.GenerateNewName();
var stateInsectorCommentsTmpFile = tempFiles.GenerateNewName();
var serializeMessagingEvents = events.SerializePostprocessorOutput <M.Event, M.EventsSerializer, M.IEventsVisitor>(
triggerSerializer => new M.EventsSerializer(triggerSerializer),
null, logPartTokenFactories, triggersConverter, null, messagingEventsElementName, eventsTmpFile, tempFiles, cancellation
);
var serializeTimelineComments = timelineComments.SerializePostprocessorOutput <TLBlock.Event, TLBlock.EventsSerializer, TLBlock.IEventsVisitor>(
triggerSerializer => new TLBlock.EventsSerializer(triggerSerializer),
null, logPartTokenFactories, triggersConverter, null, timelineCommentsElementName, timelineCommentsTmpFile, tempFiles, cancellation
);
var serializeStateInspectorComments = stateInspectorComments.SerializePostprocessorOutput <SIBlock.Event, SIBlock.EventsSerializer, SIBlock.IEventsVisitor>(
triggerSerializer => new SIBlock.EventsSerializer(triggerSerializer),
null, logPartTokenFactories, triggersConverter, null, stateCommentsElementName, stateInsectorCommentsTmpFile, tempFiles, cancellation
);
await Task.WhenAll(serializeMessagingEvents, serializeTimelineComments, serializeStateInspectorComments, logPartToken);
using (var outputWriter = XmlWriter.Create(outputFileName, new XmlWriterSettings()
{
Indent = true, Async = true
}))
using (var messagingEventsReader = XmlReader.Create(eventsTmpFile))
using (var timelineCommentsReader = XmlReader.Create(timelineCommentsTmpFile))
using (var stateInspectorCommentsReader = XmlReader.Create(stateInsectorCommentsTmpFile))
{
outputWriter.WriteStartElement("root");
new PostprocessorOutputETag(contentsEtagAttr).Write(outputWriter);
logPartTokenFactories.SafeWriteTo(await logPartToken, outputWriter);
messagingEventsReader.ReadToFollowing(messagingEventsElementName);
await outputWriter.WriteNodeAsync(messagingEventsReader, false);
timelineCommentsReader.ReadToFollowing(timelineCommentsElementName);
await outputWriter.WriteNodeAsync(timelineCommentsReader, false);
stateInspectorCommentsReader.ReadToFollowing(stateCommentsElementName);
await outputWriter.WriteNodeAsync(stateInspectorCommentsReader, false);
outputWriter.WriteEndElement(); // root
}
File.Delete(eventsTmpFile);
File.Delete(timelineCommentsTmpFile);
File.Delete(stateInsectorCommentsTmpFile);
}