public async Task <ISameNodeDetectionToken> GetToken(IEnumerableAsync <MessagePrefixesPair <Message>[]> input)
{
var processIdTask = processIdDetector.DetectProcessId(input);
var candidateTypeInfo = webRtcStateInspector.CandidateTypeInfo;
var candidateEventsTask = webRtcStateInspector.GetEvents(input).SelectMany(
evts => evts.Where(e => e.ObjectType == candidateTypeInfo)).ToList();
var logsTask = GetLogs(input);
await Task.WhenAll(processIdTask, candidateEventsTask, logsTask);
if (processIdTask.Result.Length == 0)
{
return(null);
}
var iceCandidates = candidateEventsTask.Result.ToDictionarySafe(e => e.ObjectId, e => e, (e, e2) => e);
if (iceCandidates.Count == 0 && logsTask.Result.Count == 0)
{
return(null);
}
return(new NodeDetectionToken(
processIdTask.Result,
iceCandidates.Select(c => new NodeDetectionToken.ICECandidateInfo(c.Key, ((ITriggerTime)c.Value.Trigger).Timestamp)),
logsTask.Result
));
}
async Task <List <NodeDetectionToken.ConsoleLogEntry> > GetLogs(IEnumerableAsync <MessagePrefixesPair <Message>[]> input)
{
var retVal = new Dictionary <string, NodeDetectionToken.ConsoleLogEntry?>();
await input.ForEach(messages =>
{
foreach (var msg in messages)
{
if (msg.Message.File == "CONSOLE")
{
var m = consoleLogRegex.Match(msg.Message.Text);
if (m.Success)
{
var entry = new NodeDetectionToken.ConsoleLogEntry(m.Groups[1].Value, msg.Message.Timestamp);
if (retVal.ContainsKey(entry.LogText))
{
retVal[entry.LogText] = null;
}
else
{
retVal[entry.LogText] = entry;
}
}
}
}
return(Task.FromResult(true));
});
return(retVal.Values.Where(x => x.HasValue).Select(x => x.Value).ToList());
}
async Task RunForChromeDebug(
IEnumerableAsync <CDL.Message[]> input,
LogSourcePostprocessorInput postprocessorInput
)
{
var multiplexedInput = input.Multiplex();
IPrefixMatcher matcher = postprocessing.CreatePrefixMatcher();
var extensionSources = pluginModel.ChromeDebugLogTimeLineEventSources.Select(src => src(
matcher, multiplexedInput, postprocessorInput.TemplatesTracker)).ToArray();
var events = postprocessorInput.TemplatesTracker.TrackTemplates(EnumerableAsync.Merge(extensionSources.Select(s => s.Events).ToArray()));
matcher.Freeze();
var serialize = postprocessing.Timeline.CreatePostprocessorOutputBuilder()
.SetEvents(events)
.SetTriggersConverter(evtTrigger => TextLogEventTrigger.FromUnknownTrigger(evtTrigger))
.Build(postprocessorInput);
var tasks = new List <Task>();
tasks.Add(serialize);
tasks.AddRange(extensionSources.SelectMany(s => s.MultiplexingEnumerables.Select(e => e.Open())));
tasks.Add(multiplexedInput.Open());
await Task.WhenAll(tasks);
}
public static async Task <IRefs <T> > AsRefsAsync <T>(this IEnumerableAsync <Guid> ids)
where T : IReferenceable
{
var idsArray = await ids.ToArrayAsync();
return(new Refs <T>(idsArray));
}
async Task RunForWiresharkDpmlMessages(
IEnumerableAsync <Pdml.Message[]> input,
LogSourcePostprocessorInput postprocessorInput
)
{
var logMessages = input.Multiplex();
Pdml.ITimelineEvents networkEvents = new Pdml.TimelineEvents();
var endOfTimelineEventSource = postprocessing.Timeline.CreateEndOfTimelineEventSource <Pdml.Message>();
var networkEvts = networkEvents.GetEvents(logMessages);
var eofEvts = endOfTimelineEventSource.GetEvents(logMessages);
var events = EnumerableAsync.Merge(
networkEvts,
eofEvts
);
var serialize = postprocessing.Timeline.SavePostprocessorOutput(
events,
null,
evtTrigger => TextLogEventTrigger.Make((Pdml.Message)evtTrigger),
postprocessorInput
);
await Task.WhenAll(serialize, logMessages.Open());
}
public static IEnumerable <int> ToEnumerableX(this IEnumerableAsync <TestDelegateAsync> items,
TestDelegateResult <int> convert)
{
var iterator = ((EnumerableAsyncTest)items).GetEnumerable(convert);
return(iterator);
}
async Task <uint[]> IProcessIdDetector.DetectProcessId(IEnumerableAsync <MessagePrefixesPair[]> input)
{
var retVal = new HashSet <uint>();
await input.ForEach(messages =>
{
foreach (var msg in messages)
{
if (msg.Prefixes.Contains(dataCollectedPrefix))
{
var arr = DevTools.Events.LogMessage.Parse(msg.Message.Text)?.ParsePayload <DevTools.Events.Tracing.DataCollected>()?.value;
if (arr != null)
{
foreach (var i in arr)
{
if (i.pid != null)
{
retVal.Add(i.pid.Value);
}
}
}
}
}
return(Task.FromResult(true));
});
return(retVal.ToArray());
}
public static IEnumerableAsync <T> SelectMany <M, T>(this IEnumerableAsync <M> input,
Action <M, Queue <T> > selector, Action <Queue <T> > finalizer = null)
{
var buffer = new Queue <T> ();
return(EnumerableAsync.Produce <T>(async yieldAsync =>
{
await input.ForEach(async m =>
{
selector(m, buffer);
while (buffer.Count > 0)
{
await yieldAsync.YieldAsync(buffer.Dequeue());
}
return true;
});
if (finalizer != null)
{
finalizer(buffer);
while (buffer.Count > 0)
{
await yieldAsync.YieldAsync(buffer.Dequeue());
}
}
}));
}
async Task RunForSpringServiceLog(
IEnumerableAsync <SVC.Message[]> input,
LogSourcePostprocessorInput postprocessorInput
)
{
var inputMultiplexed = input.Multiplex();
var messagingEventsSource = postprocessing.Timeline.CreateMessagingEventsSource();
var messagingEvents = messagingEventsSource.GetEvents(
((SVC.IMessagingEvents) new SVC.MessagingEvents()).GetEvents(inputMultiplexed));
var eofEvents = postprocessing.Timeline.CreateEndOfTimelineEventSource <SVC.Message>()
.GetEvents(inputMultiplexed);
var events = EnumerableAsync.Merge(
messagingEvents,
eofEvents
);
var serialize = postprocessing.Timeline.CreatePostprocessorOutputBuilder()
.SetEvents(events)
.SetTriggersConverter(evtTrigger => TextLogEventTrigger.Make((SVC.Message)evtTrigger))
.Build(postprocessorInput);
await Task.WhenAll(serialize, inputMultiplexed.Open());
}
async Task RunForWebRtcNativeLogMessages(
IEnumerableAsync <CDL.Message[]> input,
string outputFileName,
CancellationToken cancellation,
ICodepathTracker templatesTracker,
XAttribute contentsEtagAttr
)
{
timeSeriesTypesAccess.CheckForCustomConfigUpdate();
ICombinedParser parser = new TimeSeriesCombinedParser(timeSeriesTypesAccess.GetMetadataTypes());
await parser.FeedLogMessages(input);
foreach (var ts in parser.GetParsedTimeSeries())
{
ts.DataPoints = Analytics.TimeSeries.Filters.RemoveRepeatedValues.Filter(ts.DataPoints).ToList();
}
TimeSeriesPostprocessorOutput.SerializePostprocessorOutput(
parser.GetParsedTimeSeries(),
parser.GetParsedEvents(),
outputFileName,
timeSeriesTypesAccess);
}
async Task FeedLogMessages <M>(IEnumerableAsync <M[]> messages,
Func <M, string> getPrefix, Func <M, string> getText) where M : ITriggerStreamPosition, ITriggerTime
{
PrepareParsing();
var matchedLogMessages = messages.Select(msgs => msgs.Select(
m => new KeyValuePair <M, IMatchedPrefixesCollection>(
m, _prefixMatcher.Match(getPrefix(m)))).ToArray());
await matchedLogMessages.ForEach(batch =>
{
foreach (var m in batch)
{
_currentPosition = m.Key.StreamPosition;
_currentTimestamp = m.Key.Timestamp;
foreach (var prefix in m.Value)
{
foreach (var parser in _parsers[prefix])
{
var c = StartMeasure(parser.GetMetadataSource());
parser.Parse(getText(m.Key), this, null);
EndMeasure(c);
}
}
}
return(Task.FromResult(true));
});
}
async Task RunForSymLog(
IEnumerableAsync <Sym.Message[]> input,
LogSourcePostprocessorInput postprocessorInput
)
{
IPrefixMatcher matcher = postprocessing.CreatePrefixMatcher();
var inputMultiplexed = input.Multiplex();
var symEvents = RunForSymMessages(matcher, inputMultiplexed, postprocessorInput.TemplatesTracker, out var symLog);
var endOfTimelineEventSource = postprocessing.Timeline.CreateEndOfTimelineEventSource <Sym.Message>();
var messagingEventsSource = postprocessing.Timeline.CreateMessagingEventsSource();
var messagingEvents = messagingEventsSource.GetEvents(
((Sym.IMessagingEvents) new Sym.MessagingEvents()).GetEvents(inputMultiplexed));
var eofEvts = endOfTimelineEventSource.GetEvents(inputMultiplexed);
matcher.Freeze();
var events = EnumerableAsync.Merge(
symEvents,
messagingEvents,
eofEvts
);
var serialize = postprocessing.Timeline.CreatePostprocessorOutputBuilder()
.SetEvents(events)
.SetTriggersConverter(evtTrigger => TextLogEventTrigger.Make((Sym.Message)evtTrigger))
.Build(postprocessorInput);
await Task.WhenAll(serialize, symLog.Open(), inputMultiplexed.Open());
}
async Task <List <NodeDetectionToken.ConsoleLogEntry> > GetLogs(IEnumerableAsync <MessagePrefixesPair <Message>[]> input)
{
var retVal = new Dictionary <string, NodeDetectionToken.ConsoleLogEntry?>();
await input.ForEach(messages =>
{
foreach (var msg in messages)
{
if (msg.Prefixes.Contains(consoleApiPrefix))
{
var arg = DevTools.Events.LogMessage.Parse(msg.Message.Text)?.ParsePayload <DevTools.Events.Runtime.LogAPICalled>()?.args?[0];
if (arg != null && arg.type == "string")
{
var entry = new NodeDetectionToken.ConsoleLogEntry(arg.value.ToString(), msg.Message.Timestamp);
if (retVal.ContainsKey(entry.LogText))
{
retVal[entry.LogText] = null;
}
else
{
retVal[entry.LogText] = entry;
}
}
}
}
return(Task.FromResult(true));
});
return(retVal.Values.Where(x => x.HasValue).Select(x => x.Value).ToList());
}
public static async Task ForEachAsync <TSource>(this IEnumerableAsync <TSource> source, Func <TSource, Task> iterationAction, CancellationToken cancellationToken)
{
using (IEnumeratorAsync <TSource> en = source.GetEnumerator())
while (await en.MoveNextAsync(cancellationToken))
{
await iterationAction(en.Current);
}
}
public IEnumerableAsync <Timeline.Event[]> GetEvents(IEnumerableAsync <StateInspector.Event[]> input)
{
return(input.Select <StateInspector.Event, Timeline.Event>((evt, buffer) =>
{
resultEvents = buffer;
evt.Visit(this);
}, YieldFinalEvents));
}
IEnumerableAsync <Event[]> IMeetingsStateInspector.GetEvents(IEnumerableAsync <MessagePrefixesPair <Message>[]> input)
{
return(input
.Select <MessagePrefixesPair <Message>, Event>(GetEvents, GetFinalEvents, e => e.SetTags(tags))
.EnsureParented((creationEvt, buffer) =>
creationEvt.ObjectType == rootTypeInfo ? null :
EnsureRootReported((Message)creationEvt.Trigger, buffer)));
}
public static IEnumerable <TResult> ToEnumerable <TDelegateItems, T1, T2, T3, T4, TResult>(
this IEnumerableAsync <TDelegateItems> items,
Func <T1, T2, T3, T4, TResult> convert)
{
var iterator = items.GetEnumerable <TResult, Func <T1, T2, T3, T4, TResult> >(convert);
return(iterator);
}
public static IEnumerable <TResult> ToEnumerable <TDelegateItems, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, TResult>(
this IEnumerableAsync <TDelegateItems> items,
Func <T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, TResult> convert)
{
var iterator = items.GetEnumerable <TResult, Func <T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, TResult> >(convert);
return(iterator);
}
public static Task <TResult> ToDictionaryAsync <TKey, TValue, TResult>(this IEnumerableAsync <KeyValuePair <TKey, TValue> > kvpItems,
Func <Dictionary <TKey, TValue>, KeyValuePair <TKey, TValue>[], TResult> dictionaryAndDuplicates)
{
return(kvpItems.ToDictionaryAsync(
kvp => kvp.Key,
kvp => kvp.Value,
dictionaryAndDuplicates));
}
//public delegate void ToDictionaryDelegate<TKey, TValue>(TKey key, TValue value);
//public static IDictionary<TKey, TValue> ToEnumerable<TDelegateItems, T1, T2, T3, TKey, TValue>(
// this IEnumerableAsync<TDelegateItems> items,
// Action<T1, T2, T3, ToDictionaryDelegate<TKey, TValue>> convert)
//{
// var iterator = items.GetEnumerable<KeyValuePair<TKey, TValue>, Func<T1, T2, T3, KeyValuePair<TKey, TValue>>>(
// (t1, t2, t3) =>
// {
// var key = default(TKey);
// var value = default(TValue);
// convert(t1, t2, t3, (k, v) => { key = k; value = v; });
// return new KeyValuePair<TKey, TValue>(key, value);
// });
// return iterator.ToDictionary(;
//}
//public static IDictionary<TKey, TValue> ToDictionary<TDelegateItems, T1, TKey, TValue>(
// this IEnumerableAsync<TDelegateItems> items,
// Action<T1, ToDictionaryDelegate<TKey, TValue>> convert)
//{
// var iterator = items.GetEnumerable<TResult, Func<T1, TResult>>(convert);
// return iterator;
//}
#endregion
#region ToEnumerable
public static IEnumerable <TResult> ToEnumerable <TDelegateItems, TDelegateConvert, TResult>(
this IEnumerableAsync <TDelegateItems> items,
TDelegateConvert convert)
{
var iterator = items.GetEnumerable <TResult, TDelegateConvert>(convert);
return(iterator);
}
public static IEnumerableAsync <MessagePrefixesPair[]> MatchPrefixes(this IEnumerableAsync <Message[]> input, IPrefixMatcher prefixMatcher)
{
return(input.Select(
msgs => msgs.Select(
m => new MessagePrefixesPair(m, prefixMatcher.Match(m.Text))
).ToArray()
));
}
public EventsSource(
IEnumerableAsync <EventType[]> events,
params IMultiplexingEnumerableOpen[] multiplexingEnumerables
)
{
Events = events;
MultiplexingEnumerables = new List <IMultiplexingEnumerableOpen>(multiplexingEnumerables);
}
public static async Task <int> CountAsync <TSource>(this IEnumerableAsync <TSource> source, CancellationToken cancellationToken)
{
int count = 0;
await ForEachAsync(source, x => count++, cancellationToken);
return(count);
}
public IEnumerableAsync <Message[]> FromChromeDriverLog(IEnumerableAsync <CD.Message[]> messages)
{
using (var ctx = new Context())
{
return(messages.SelectMany(batch => new [] {
batch.Select(ctx.FromCDMessage).Where(m => m != null).ToArray()
}));
}
}
public static async Task <TResult> ToArrayAsync <T, TResult>(this IEnumerableAsync <T> enumerableAsync,
Func <T[], TResult> onComplete)
{
var enumerable = await enumerableAsync.Async();
var items = enumerable.ToArray();
return(onComplete(items));
}
public EnumerableAsyncHttpResponse(IApplication application,
IHttpRequest request, ParameterInfo parameterInfo, HttpStatusCode statusCode,
IEnumerableAsync <T> objectsAsync)
: base(request, statusCode)
{
this.application = application;
this.objectsAsync = objectsAsync;
this.parameterInfo = parameterInfo;
}
public static async Task ForEachAsync <T>(this IEnumerableAsync <T> list, Func <T, Task> func)
{
var enumerator = await list.GetEnumeratorAsync();
while (await enumerator.MoveNextAsync())
{
await func(enumerator.Current);
}
}
public static IEnumerableAsync <TSelect> SelectOptional <TItem, TSelect>(this IEnumerableAsync <TItem> enumerable,
Func <TItem, Func <TSelect, ISelectionResult <TSelect> >, Func <ISelectionResult <TSelect> >, ISelectionResult <TSelect> > selection)
{
return(enumerable.SelectOptional <TItem, TSelect, int, IEnumerableAsync <TSelect> >(1,
(item, carryUpdated, next, skip) => selection(item,
(selected) => next(selected, 1),
() => skip(1)),
(aggregation, carryUpdated) => aggregation));
}
public static IEnumerableAsync <TItem> Throttle <TItem>(this IEnumerableAsync <Task <TItem> > enumerable,
int desiredRunCount = 1,
ILogger log = default(ILogger))
{
var logScope = log.CreateScope($"Throttle");
var taskList = new List <Task <TItem> >();
var locker = new AutoResetEvent(true);
var enumerator = enumerable.GetEnumerator();
var moving = true;
return(Yield <TItem>(
async(yieldReturn, yieldBreak) =>
{
while (true)
{
locker.WaitOne();
if (taskList.Count >= desiredRunCount)
{
Task <TItem>[] tasks = taskList.ToArray();
var finishedTask = await Task.WhenAny(tasks);
taskList.Remove(finishedTask);
locker.Set();
var item = await finishedTask;
return yieldReturn(item);
}
if (!moving)
{
if (!taskList.Any())
{
locker.Set();
return yieldBreak;
}
Task <TItem>[] tasks = taskList.ToArray();
var finishedTask = await Task.WhenAny(tasks);
taskList.Remove(finishedTask);
locker.Set();
var item = await finishedTask;
return yieldReturn(item);
}
locker.Set();
if (!await enumerator.MoveNextAsync())
{
moving = false;
continue;
}
var nextTask = enumerator.Current;
locker.WaitOne();
taskList.Add(nextTask);
locker.Set();
}
}));
}
public static IEnumerableAsync <T> JoinTask <T>(this IEnumerableAsync <T> enumerableAsync,
Func <Task> task,
EastFive.Analytics.ILogger logger = default(ILogger))
{
enumerableAsync
.JoinTask(
Task.Run(task),
logger);
return(enumerableAsync);
}