async Task RunForChromeDebug(LogSourcePostprocessorInput input) { var reader = new CDL.Reader(postprocessing.TextLogParser, input.CancellationToken).Read(input.OpenLogFile, s => s.Dispose(), input.ProgressHandler); var multiplexedInput = reader.Multiplex(); IPrefixMatcher prefixMatcher = postprocessing.CreatePrefixMatcher(); var matchedMessages = multiplexedInput.MatchTextPrefixes(prefixMatcher).Multiplex(); var webRtcStateInspector = new CDL.WebRtcStateInspector(prefixMatcher); var processIdDetector = new CDL.ProcessIdDetector(); var nodeDetectionTokenTask = (new CDL.NodeDetectionTokenSource(processIdDetector, webRtcStateInspector)).GetToken(matchedMessages); var matcher = postprocessing.CreatePrefixMatcher(); var extensionSources = pluginModel.ChromeDebugLogMessagingEventSources.Select(src => src( matcher, multiplexedInput, input.TemplatesTracker)).ToArray(); var events = EnumerableAsync.Merge(extensionSources.Select(s => s.Events).ToArray()); var serialize = postprocessing.Correlation.CreatePostprocessorOutputBuilder() .SetSameNodeDetectionToken(nodeDetectionTokenTask) .SetMessagingEvents(events) .SetTriggersConverter(evtTrigger => TextLogEventTrigger.FromUnknownTrigger(evtTrigger)) .Build(input); var tasks = new List <Task>(); tasks.Add(serialize); tasks.AddRange(extensionSources.SelectMany(s => s.MultiplexingEnumerables.Select(e => e.Open()))); tasks.Add(matchedMessages.Open()); tasks.Add(multiplexedInput.Open()); await Task.WhenAll(tasks); }
async static Task RunForChromeDebug( IEnumerableAsync <CDL.Message[]> input, string outputFileName, CancellationToken cancellation, ICodepathTracker templatesTracker, XAttribute contentsEtagAttr ) { var inputMultiplexed = input.Multiplex(); IPrefixMatcher matcher = new PrefixMatcher(); var logMessages = CDL.Helpers.MatchPrefixes(inputMultiplexed, matcher).Multiplex(); CDL.IWebRtcStateInspector webRtcStateInspector = new CDL.WebRtcStateInspector(matcher); var webRtcEvts = webRtcStateInspector.GetEvents(logMessages); Sym.IMeetingsStateInspector symMeetingsStateInspector = new Sym.MeetingsStateInspector(matcher); Sym.IMediaStateInspector symMediaStateInspector = new Sym.MediaStateInspector(matcher); var symMessages = Sym.Helpers.MatchPrefixes((new Sym.Reader()).FromChromeDebugLog(inputMultiplexed), matcher).Multiplex(); var symMeetingEvents = symMeetingsStateInspector.GetEvents(symMessages); var symMediaEvents = symMediaStateInspector.GetEvents(symMessages); matcher.Freeze(); var events = EnumerableAsync.Merge( webRtcEvts.Select(ConvertTriggers <CDL.Message>), symMeetingEvents.Select(ConvertTriggers <Sym.Message>), symMediaEvents.Select(ConvertTriggers <Sym.Message>) ) .ToFlatList(); await Task.WhenAll(events, symMessages.Open(), logMessages.Open(), inputMultiplexed.Open()); if (cancellation.IsCancellationRequested) { return; } if (templatesTracker != null) { (await events).ForEach(e => templatesTracker.RegisterUsage(e.TemplateId)); } StateInspectorOutput.SerializePostprocessorOutput(await events, null, contentsEtagAttr).Save(outputFileName); }
async Task RunForChromeDebug( IEnumerableAsync <CDL.Message[]> inputMessages, LogSourcePostprocessorInput postprocessorInput ) { var inputMultiplexed = inputMessages.Multiplex(); IPrefixMatcher matcher = postprocessing.CreatePrefixMatcher(); var logMessages = inputMultiplexed.MatchTextPrefixes(matcher).Multiplex(); CDL.IWebRtcStateInspector webRtcStateInspector = new CDL.WebRtcStateInspector(matcher); var webRtcEvts = webRtcStateInspector.GetEvents(logMessages); var extensionSources = pluginModel .ChromeDebugStateEventSources.Select( source => source(matcher, inputMultiplexed, postprocessorInput.TemplatesTracker) ) .ToArray(); var eventSources = new List <IEnumerableAsync <Event[]> > { webRtcEvts }; eventSources.AddRange(extensionSources.Select(s => s.Events)); matcher.Freeze(); var events = postprocessorInput.TemplatesTracker.TrackTemplates(EnumerableAsync.Merge(eventSources.ToArray())); var serialize = postprocessing.StateInspector.SavePostprocessorOutput( events, null, evtTrigger => TextLogEventTrigger.FromUnknownTrigger(evtTrigger), postprocessorInput ); var tasks = new List <Task>(); tasks.Add(serialize); tasks.AddRange(extensionSources.SelectMany(s => s.MultiplexingEnumerables.Select(e => e.Open()))); tasks.Add(logMessages.Open()); tasks.Add(inputMultiplexed.Open()); await Task.WhenAll(tasks); }
async Task <IPostprocessorRunSummary> Run(LogSourcePostprocessorInput[] inputFiles, IPostprocessorsRegistry postprocessorsRegistry) { var usedRoleInstanceNames = new HashSet <string>(); Func <LogSourcePostprocessorInput, string> getUniqueRoleInstanceName = inputFile => { for (int tryCount = 0; ; ++tryCount) { var ret = string.Format( tryCount == 0 ? "{0}" : "{0} ({1})", inputFile.LogSource.GetShortDisplayNameWithAnnotation(), tryCount ); if (usedRoleInstanceNames.Add(ret)) { return(ret); } } }; var noMessagingEvents = Task.FromResult(new List <M.Event>()); var chromeDebugLogs = Enumerable.Empty <NodeInfo>() .Concat( inputFiles .Where(f => f.LogSource.Provider.Factory == postprocessorsRegistry.ChromeDebugLog.LogProviderFactory) .Select(inputFile => { var reader = (new CDL.Reader(inputFile.CancellationToken)).Read(inputFile.LogFileName, inputFile.GetLogFileNameHint(), inputFile.ProgressHandler); IPrefixMatcher prefixMatcher = new PrefixMatcher(); var nodeId = new NodeId("chrome-debug", getUniqueRoleInstanceName(inputFile)); var matchedMessages = CDL.Helpers.MatchPrefixes(reader, prefixMatcher).Multiplex(); var webRtcStateInspector = new CDL.WebRtcStateInspector(prefixMatcher); var processIdDetector = new CDL.ProcessIdDetector(); var nodeDetectionTokenTask = (new CDL.NodeDetectionTokenSource(processIdDetector, webRtcStateInspector)).GetToken(matchedMessages); return(new NodeInfo( new[] { inputFile.LogSource }, nodeId, matchedMessages, null, noMessagingEvents, nodeDetectionTokenTask )); }) ) .ToList(); var webRtcInternalsDumps = Enumerable.Empty <NodeInfo>() .Concat( inputFiles .Where(f => f.LogSource.Provider.Factory == postprocessorsRegistry.WebRtcInternalsDump.LogProviderFactory) .Select(inputFile => { var reader = (new WRD.Reader(inputFile.CancellationToken)).Read(inputFile.LogFileName, inputFile.GetLogFileNameHint(), inputFile.ProgressHandler); IPrefixMatcher prefixMatcher = new PrefixMatcher(); var nodeId = new NodeId("webrtc-int", getUniqueRoleInstanceName(inputFile)); var matchedMessages = WRD.Helpers.MatchPrefixes(reader, prefixMatcher).Multiplex(); var webRtcStateInspector = new WRD.WebRtcStateInspector(prefixMatcher); var nodeDetectionTokenTask = (new WRD.NodeDetectionTokenSource(webRtcStateInspector)).GetToken(matchedMessages); return(new NodeInfo( new[] { inputFile.LogSource }, nodeId, matchedMessages, null, noMessagingEvents, nodeDetectionTokenTask )); }) ) .ToList(); var chromeDriverLogs = Enumerable.Empty <NodeInfo>() .Concat( inputFiles .Where(f => f.LogSource.Provider.Factory == postprocessorsRegistry.ChromeDriver.LogProviderFactory) .Select(inputFile => { var reader = (new CD.Reader(inputFile.CancellationToken)).Read(inputFile.LogFileName, inputFile.GetLogFileNameHint(), inputFile.ProgressHandler); IPrefixMatcher prefixMatcher = new PrefixMatcher(); var nodeId = new NodeId("webrtc-int", getUniqueRoleInstanceName(inputFile)); var matchedMessages = CD.Helpers.MatchPrefixes(reader, prefixMatcher).Multiplex(); var nodeDetectionTokenTask = (new CD.NodeDetectionTokenSource(new CD.ProcessIdDetector(prefixMatcher), prefixMatcher)).GetToken(matchedMessages); return(new NodeInfo( new[] { inputFile.LogSource }, nodeId, matchedMessages, null, noMessagingEvents, nodeDetectionTokenTask )); }) ) .ToList(); var tasks = new List <Task>(); tasks.AddRange(chromeDebugLogs.Select(l => l.SameNodeDetectionTokenTask)); tasks.AddRange(chromeDebugLogs.Select(l => l.MultiplexingEnumerable.Open())); tasks.AddRange(webRtcInternalsDumps.Select(l => l.SameNodeDetectionTokenTask)); tasks.AddRange(webRtcInternalsDumps.Select(l => l.MultiplexingEnumerable.Open())); tasks.AddRange(chromeDriverLogs.Select(l => l.SameNodeDetectionTokenTask)); tasks.AddRange(chromeDriverLogs.Select(l => l.MultiplexingEnumerable.Open())); await Task.WhenAll(tasks); var allLogs = Enumerable.Empty <NodeInfo>() .Concat(chromeDebugLogs) .Concat(webRtcInternalsDumps) .Concat(chromeDriverLogs) .ToArray(); var fixedConstraints = allLogs .GroupBy(l => l.SameNodeDetectionTokenTask.Result, new SameNodeEqualityComparer()) .SelectMany(group => LinqUtils.ZipWithNext(group).Select(pair => new NodesConstraint() { Node1 = pair.Key.NodeId, Node2 = pair.Value.NodeId, Value = pair.Value.SameNodeDetectionTokenTask.Result.DetectSameNode(pair.Key.SameNodeDetectionTokenTask.Result).TimeDiff })) .ToList(); var allowInstacesMergingForRoles = new HashSet <string>(); ICorrelator correlator = new LogJoint.Analytics.Correlation.Correlator( new M.Analisys.InternodeMessagesDetector(), SolverFactory.Create() ); var correlatorSolution = await correlator.Correlate( allLogs.ToDictionary(i => i.NodeId, i => (IEnumerable <M.Event>)i.MessagesTask.Result), fixedConstraints, allowInstacesMergingForRoles ); var nodeIdToLogSources = (from l in allLogs from ls in l.LogSources select new { L = l.NodeId, Ls = ls }) .ToLookup(i => i.L, i => i.Ls); var grouppedLogsReport = new StringBuilder(); foreach (var group in allLogs.Where(g => g.LogSources.Length > 1)) { if (grouppedLogsReport.Length == 0) { grouppedLogsReport.AppendLine(); grouppedLogsReport.AppendLine("Groupped logs info:"); } grouppedLogsReport.AppendLine(string.Format( " - {0} were groupped to represent node {1}", string.Join(", ", group.LogSources.Select(ls => '\"' + ls.GetShortDisplayNameWithAnnotation() + '\"')), group.NodeId)); } var timeOffsets = (from ns in correlatorSolution.NodeSolutions from ls in nodeIdToLogSources[ns.Key] select new { Sln = ns.Value, Ls = ls }) .ToDictionary(i => i.Ls, i => i.Sln); await modelThreadSync.Invoke(() => { foreach (var ls in ljModel.SourcesManager.Items) { NodeSolution sln; if (timeOffsets.TryGetValue(ls, out sln)) { ITimeOffsetsBuilder builder = new TimeOffsets.Builder(); builder.SetBaseOffset(sln.BaseDelta); if (sln.TimeDeltas != null) { foreach (var d in sln.TimeDeltas) { builder.AddOffset(d.At, d.Delta); } } ls.TimeOffsets = builder.ToTimeOffsets(); } } }); var correlatedLogsConnectionIds = postprocessorsManager.GetCorrelatableLogsConnectionIds(inputFiles.Select(i => i.LogSource)); foreach (var inputFile in inputFiles) { NodeSolution sln; if (!timeOffsets.TryGetValue(inputFile.LogSource, out sln)) { continue; } (new CorrelatorPostprocessorOutput(sln, correlatedLogsConnectionIds)).Save(inputFile.OutputFileName); } return(new CorrelatorPostprocessorRunSummary(correlatorSolution.Success, correlatorSolution.CorrelationLog + grouppedLogsReport.ToString())); }