public OutputDataDeserializer(TimeSeries.ITimeSeriesTypesAccess timeSeriesTypesAccess,
ILogPartTokenFactories logPartTokenFactories, Correlation.ISameNodeDetectionTokenFactories nodeDetectionTokenFactories)
{
this.timeSeriesTypesAccess = timeSeriesTypesAccess;
this.logPartTokenFactories = logPartTokenFactories;
this.nodeDetectionTokenFactories = nodeDetectionTokenFactories;
}
public StateInspectorOutput(LogSourcePostprocessorDeserializationParams p, ILogPartTokenFactories rotatedLogPartFactories)
{
this.logSource = p.LogSource;
if (!p.Reader.ReadToFollowing("root"))
{
throw new FormatException();
}
etag.Read(p.Reader);
var eventsDeserializer = new EventsDeserializer(TextLogEventTrigger.DeserializerFunction);
foreach (var elt in p.Reader.ReadChildrenElements())
{
if (eventsDeserializer.TryDeserialize(elt, out var evt))
{
events.Add(evt);
}
else if (rotatedLogPartFactories.TryReadLogPartToken(elt, out var tmp))
{
this.rotatedLogPartToken = tmp;
}
p.Cancellation.ThrowIfCancellationRequested();
}
}
public Model(ITempFilesManager tempFiles,
ILogPartTokenFactories logPartTokenFactories, ISameNodeDetectionTokenFactories nodeDetectionTokenFactories)
{
this.tempFiles = tempFiles;
this.logPartTokenFactories = logPartTokenFactories;
this.nodeDetectionTokenFactories = nodeDetectionTokenFactories;
}
public PostprocessorsManager(
ILogSourcesManager logSources,
Telemetry.ITelemetryCollector telemetry,
ISynchronizationContext modelSyncContext,
ISynchronizationContext threadPoolSyncContext,
IHeartBeatTimer heartbeat,
Progress.IProgressAggregator progressAggregator,
Settings.IGlobalSettingsAccessor settingsAccessor,
IOutputDataDeserializer outputDataDeserializer,
ITraceSourceFactory traceSourceFactory,
ILogPartTokenFactories logPartTokenFactories,
Correlation.ISameNodeDetectionTokenFactories sameNodeDetectionTokenFactories,
IChangeNotification changeNotification,
LogMedia.IFileSystem logFileSystem
)
{
this.logSources = logSources;
this.telemetry = telemetry;
this.progressAggregator = progressAggregator;
this.settingsAccessor = settingsAccessor;
this.modelSyncContext = modelSyncContext;
this.threadPoolSyncContext = threadPoolSyncContext;
this.heartbeat = heartbeat;
this.outputDataDeserializer = outputDataDeserializer;
this.logPartTokenFactories = logPartTokenFactories;
this.sameNodeDetectionTokenFactories = sameNodeDetectionTokenFactories;
this.changeNotification = changeNotification;
this.logFileSystem = logFileSystem;
this.tracer = traceSourceFactory.CreateTraceSource("App", "ppm");
this.updater = new AsyncInvokeHelper(modelSyncContext, Refresh);
logSources.OnLogSourceAdded += (sender, args) => updater.Invoke();
logSources.OnLogSourceRemoved += (sender, args) => updater.Invoke();
logSources.OnLogSourceAnnotationChanged += (sender, args) => updater.Invoke();
logSources.OnLogSourceStatsChanged += (object sender, LogSourceStatsEventArgs e) =>
{
if ((e.Flags & LogProviderStatsFlag.ContentsEtag) != 0)
{
updater.Invoke();
}
};
this.visiblePostprocessorsOutputs = Selectors.Create(
() => postprocessorsOutputs,
() => logSources.Items,
(outputs, sources) => {
var sourcesMap = sources.ToLookup(s => s);
return(ImmutableArray.CreateRange(outputs.Where(output => sourcesMap.Contains(output.LogSource))));
}
);
Refresh();
}
public static async Task SerializePostprocessorOutput(
Task <ILogPartToken> logPartToken,
ILogPartTokenFactories logPartTokenFactories,
IEnumerableAsync <M.Event[]> events,
Task <ISameNodeDetectionToken> sameNodeDetectionTokenTask,
ISameNodeDetectionTokenFactories nodeDetectionTokenFactories,
Func <object, TextLogEventTrigger> triggersConverter,
string contentsEtagAttr,
Func <Task <Stream> > openOutputStream,
ITempFilesManager tempFiles,
CancellationToken cancellation
)
{
events = events ?? new List <M.Event[]>().ToAsync();
logPartToken = logPartToken ?? Task.FromResult <ILogPartToken>(null);
sameNodeDetectionTokenTask = sameNodeDetectionTokenTask ?? Task.FromResult <ISameNodeDetectionToken>(null);
var eventsTmpFile = tempFiles.GenerateNewName();
Func <Task <Stream> > openTempFile(string fileName) => () => Task.FromResult <Stream>(new FileStream(fileName, FileMode.OpenOrCreate));
var serializeMessagingEvents = events.SerializePostprocessorOutput <M.Event, M.EventsSerializer, M.IEventsVisitor>(
triggerSerializer => new M.EventsSerializer(triggerSerializer),
null, logPartTokenFactories, triggersConverter, null, messagingEventsElementName, openTempFile(eventsTmpFile), tempFiles, cancellation
);
await Task.WhenAll(serializeMessagingEvents, logPartToken, sameNodeDetectionTokenTask);
using (var outputWriter = XmlWriter.Create(await openOutputStream(), new XmlWriterSettings()
{
Indent = true, Async = true, CloseOutput = true
}))
using (var messagingEventsReader = XmlReader.Create(eventsTmpFile))
{
outputWriter.WriteStartElement("root");
new PostprocessorOutputETag(contentsEtagAttr).Write(outputWriter);
logPartTokenFactories.SafeWriteTo(await logPartToken, outputWriter);
nodeDetectionTokenFactories.SafeWriteTo(await sameNodeDetectionTokenTask, outputWriter);
messagingEventsReader.ReadToFollowing(messagingEventsElementName);
await outputWriter.WriteNodeAsync(messagingEventsReader, false);
outputWriter.WriteEndElement(); // root
}
File.Delete(eventsTmpFile);
}
public PostprocessorOutput(
LogSourcePostprocessorDeserializationParams p,
ILogPartTokenFactories rotatedLogPartFactories,
ISameNodeDetectionTokenFactories nodeDetectionTokenFactories)
{
this.logSource = p.LogSource;
var reader = p.Reader;
events = new List <M.Event>();
rotatedLogPartToken = new NullLogPartToken();
sameNodeDetectionToken = new NullSameNodeDetectionToken();
if (!reader.ReadToFollowing("root"))
{
throw new FormatException();
}
etag.Read(reader);
foreach (var elt in p.Reader.ReadChildrenElements())
{
if (rotatedLogPartFactories.TryReadLogPartToken(elt, out var tmp))
{
this.rotatedLogPartToken = tmp;
}
else if (nodeDetectionTokenFactories.TryReadLogPartToken(elt, out var tmp2))
{
sameNodeDetectionToken = tmp2;
}
else if (elt.Name == messagingEventsElementName)
{
var eventsDeserializer = new M.EventsDeserializer(TextLogEventTrigger.DeserializerFunction);
foreach (var me in elt.Elements())
{
if (eventsDeserializer.TryDeserialize(me, out var evt))
{
events.Add(evt);
}
}
}
p.Cancellation.ThrowIfCancellationRequested();
}
}
public void BeforeEach()
{
logSources = Substitute.For <ILogSourcesManager>();
telemetry = Substitute.For <Telemetry.ITelemetryCollector>();
mockedSyncContext = new ManualSynchronizationContext();
heartbeat = Substitute.For <IHeartBeatTimer>();
progressAggregator = Substitute.For <Progress.IProgressAggregator>();
settingsAccessor = Substitute.For <Settings.IGlobalSettingsAccessor>();
logSource1 = Substitute.For <ILogSource>();
logProviderFac1 = Substitute.For <ILogProviderFactory>();
logSource1.Provider.Factory.Returns(logProviderFac1);
logSource1.Provider.ConnectionParams.Returns(new ConnectionParams($"{ConnectionParamsKeys.PathConnectionParam}=/log.txt"));
logSource1.Provider.Stats.Returns(new LogProviderStats()
{
ContentsEtag = null
});
logSourcePP1 = Substitute.For <ILogSourcePostprocessor>();
logSourcePP1.Kind.Returns(PostprocessorKind.SequenceDiagram);
pp1outputXmlSection = Substitute.For <Persistence.ISaxXMLStorageSection>();
logSource1.LogSourceSpecificStorageEntry.OpenSaxXMLSection("postproc-sequencediagram.xml", Persistence.StorageSectionOpenFlag.ReadOnly).Returns(pp1outputXmlSection);
pp1outputXmlSection.Reader.Returns(Substitute.For <XmlReader>());
pp1PostprocessorOutput = Substitute.For <IPostprocessorOutputETag>();
outputDataDeserializer = Substitute.For <IOutputDataDeserializer>();
outputDataDeserializer.Deserialize(PostprocessorKind.SequenceDiagram, Arg.Any <LogSourcePostprocessorDeserializationParams>()).Returns(pp1PostprocessorOutput);
pp1RunSummary = Substitute.For <IPostprocessorRunSummary>();
logSourcePP1.Run(null).ReturnsForAnyArgs(Task.FromResult(pp1RunSummary));
pp1RunSummary.GetLogSpecificSummary(null).ReturnsForAnyArgs((IPostprocessorRunSummary)null);
logPartTokenFactories = Substitute.For <ILogPartTokenFactories>();
sameNodeDetectionTokenFactories = Substitute.For <ISameNodeDetectionTokenFactories>();
changeNotification = Substitute.For <IChangeNotification>();
fileSystem = Substitute.For <IFileSystem>();
manager = new LogJoint.Postprocessing.PostprocessorsManager(
logSources, telemetry, mockedSyncContext, mockedSyncContext, heartbeat, progressAggregator, settingsAccessor, outputDataDeserializer, new TraceSourceFactory(),
logPartTokenFactories, sameNodeDetectionTokenFactories, changeNotification, fileSystem);
manager.Register(new LogSourceMetadata(logProviderFac1, logSourcePP1));
}
public static Task SerializePostprocessorOutput(
IEnumerableAsync <Event[]> events,
Task <ILogPartToken> rotatedLogPartToken,
ILogPartTokenFactories logPartTokenFactories,
Func <object, TextLogEventTrigger> triggersConverter,
string contentsEtagAttr,
Func <Task <Stream> > openOutputStream,
ITempFilesManager tempFiles,
CancellationToken cancellation
)
{
return(events.SerializePostprocessorOutput <Event, EventsSerializer, IEventsVisitor>(
triggerSerializer => new EventsSerializer(triggerSerializer),
rotatedLogPartToken,
logPartTokenFactories,
triggersConverter,
contentsEtagAttr,
"root",
openOutputStream,
tempFiles,
cancellation
));
}
public static async Task SerializePostprocessorOutput <Evt, Serializer, EvtVisitor>(
this IEnumerableAsync <Evt[]> events,
Func <Action <object, XElement>, Serializer> serializerFactory,
Task <ILogPartToken> rotatedLogPartToken,
ILogPartTokenFactories rotatedLogPartFactories,
Func <object, TextLogEventTrigger> triggersConverter,
string contentsEtagAttr,
string rootElementName,
string outputFileName,
ITempFilesManager tempFiles,
CancellationToken cancellation
) where Evt : IVisitable <EvtVisitor> where Serializer : class, IEventsSerializer, EvtVisitor
{
rotatedLogPartToken = rotatedLogPartToken ?? Task.FromResult <ILogPartToken>(null);
var sortKeyAttr = XName.Get("__key");
var chunks = new List <string>();
Serializer serializer = null;
Action resetSerializer = () =>
{
if (serializer?.Output?.Count > 0)
{
string chunkFileName = tempFiles.GenerateNewName();
chunks.Add(chunkFileName);
using (var writer = XmlWriter.Create(chunkFileName, new XmlWriterSettings()
{
OmitXmlDeclaration = true,
ConformanceLevel = ConformanceLevel.Fragment
}))
{
foreach (var e in serializer.Output.OrderBy(e => e.Attribute(sortKeyAttr).Value))
{
e.WriteTo(writer);
}
}
}
serializer = serializerFactory((trigger, elt) =>
{
triggersConverter(trigger).Save(elt);
elt.SetAttributeValue(sortKeyAttr, ((IOrderedTrigger)trigger).Index.ToString("x8"));
});
};
resetSerializer();
await events.ForEach(batch =>
{
foreach (var e in batch)
{
e.Visit(serializer);
if (serializer.Output.Count >= 8 * 1024)
{
resetSerializer();
}
}
return(Task.FromResult(!cancellation.IsCancellationRequested));
});
resetSerializer();
if (cancellation.IsCancellationRequested)
{
return;
}
using (var outputWriter = XmlWriter.Create(outputFileName, new XmlWriterSettings()
{
Indent = true
}))
{
outputWriter.WriteStartElement(rootElementName);
new PostprocessorOutputETag(contentsEtagAttr).Write(outputWriter);
rotatedLogPartFactories.SafeWriteTo(await rotatedLogPartToken, outputWriter);
var readersSettings = new XmlReaderSettings()
{
ConformanceLevel = ConformanceLevel.Fragment
};
var readers = chunks.Select(chunkFileName => XmlReader.Create(chunkFileName, readersSettings)).ToList();
try
{
var q = new VCSKicksCollection.PriorityQueue <KeyValuePair <XmlReader, XElement> >(Comparer <KeyValuePair <XmlReader, XElement> > .Create((item1, item2) =>
{
return(string.CompareOrdinal(item1.Value.Attribute(sortKeyAttr).Value, item2.Value.Attribute(sortKeyAttr).Value));
}));
Action <XmlReader> enqueueReader = reader =>
{
if (!reader.EOF)
{
if (reader.MoveToContent() != XmlNodeType.Element)
{
throw new InvalidOperationException("bad chunk");
}
q.Enqueue(new KeyValuePair <XmlReader, XElement>(reader, (XElement)XNode.ReadFrom(reader)));
}
};
readers.ForEach(enqueueReader);
while (q.Count > 0)
{
var item = q.Dequeue();
item.Value.Attribute(sortKeyAttr).Remove();
item.Value.WriteTo(outputWriter);
enqueueReader(item.Key);
}
}
finally
{
readers.ForEach(r => r.Dispose());
chunks.ForEach(chunkFileName => tempFiles.DeleteIfTemporary(chunkFileName));
}
outputWriter.WriteEndElement(); // end of root node
}
}
public Model(ITempFilesManager tempFiles, ILogPartTokenFactories logPartTokenFactories)
{
this.tempFiles = tempFiles;
this.logPartTokenFactories = logPartTokenFactories;
}
public static async Task SerializePostprocessorOutput(
IEnumerableAsync <M.Event[]> events,
IEnumerableAsync <TLBlock.Event[]> timelineComments,
IEnumerableAsync <SIBlock.Event[]> stateInspectorComments,
Task <ILogPartToken> logPartToken,
ILogPartTokenFactories logPartTokenFactories,
Func <object, TextLogEventTrigger> triggersConverter,
string contentsEtagAttr,
string outputFileName,
ITempFilesManager tempFiles,
CancellationToken cancellation
)
{
events = events ?? new List <M.Event[]>().ToAsync();
timelineComments = timelineComments ?? new List <TLBlock.Event[]>().ToAsync();
stateInspectorComments = stateInspectorComments ?? new List <SIBlock.Event[]>().ToAsync();
logPartToken = logPartToken ?? Task.FromResult <ILogPartToken>(null);
var eventsTmpFile = tempFiles.GenerateNewName();
var timelineCommentsTmpFile = tempFiles.GenerateNewName();
var stateInsectorCommentsTmpFile = tempFiles.GenerateNewName();
var serializeMessagingEvents = events.SerializePostprocessorOutput <M.Event, M.EventsSerializer, M.IEventsVisitor>(
triggerSerializer => new M.EventsSerializer(triggerSerializer),
null, logPartTokenFactories, triggersConverter, null, messagingEventsElementName, eventsTmpFile, tempFiles, cancellation
);
var serializeTimelineComments = timelineComments.SerializePostprocessorOutput <TLBlock.Event, TLBlock.EventsSerializer, TLBlock.IEventsVisitor>(
triggerSerializer => new TLBlock.EventsSerializer(triggerSerializer),
null, logPartTokenFactories, triggersConverter, null, timelineCommentsElementName, timelineCommentsTmpFile, tempFiles, cancellation
);
var serializeStateInspectorComments = stateInspectorComments.SerializePostprocessorOutput <SIBlock.Event, SIBlock.EventsSerializer, SIBlock.IEventsVisitor>(
triggerSerializer => new SIBlock.EventsSerializer(triggerSerializer),
null, logPartTokenFactories, triggersConverter, null, stateCommentsElementName, stateInsectorCommentsTmpFile, tempFiles, cancellation
);
await Task.WhenAll(serializeMessagingEvents, serializeTimelineComments, serializeStateInspectorComments, logPartToken);
using (var outputWriter = XmlWriter.Create(outputFileName, new XmlWriterSettings()
{
Indent = true, Async = true
}))
using (var messagingEventsReader = XmlReader.Create(eventsTmpFile))
using (var timelineCommentsReader = XmlReader.Create(timelineCommentsTmpFile))
using (var stateInspectorCommentsReader = XmlReader.Create(stateInsectorCommentsTmpFile))
{
outputWriter.WriteStartElement("root");
new PostprocessorOutputETag(contentsEtagAttr).Write(outputWriter);
logPartTokenFactories.SafeWriteTo(await logPartToken, outputWriter);
messagingEventsReader.ReadToFollowing(messagingEventsElementName);
await outputWriter.WriteNodeAsync(messagingEventsReader, false);
timelineCommentsReader.ReadToFollowing(timelineCommentsElementName);
await outputWriter.WriteNodeAsync(timelineCommentsReader, false);
stateInspectorCommentsReader.ReadToFollowing(stateCommentsElementName);
await outputWriter.WriteNodeAsync(stateInspectorCommentsReader, false);
outputWriter.WriteEndElement(); // root
}
File.Delete(eventsTmpFile);
File.Delete(timelineCommentsTmpFile);
File.Delete(stateInsectorCommentsTmpFile);
}
public TimelinePostprocessorOutput(LogSourcePostprocessorDeserializationParams p, ILogPartTokenFactories logPartTokenFactories) :
this(p, TimelineEntitiesComparer.Instance, logPartTokenFactories)
{
}
