public static IEnumerableAsync <T> SelectMany <M, T>(this IEnumerableAsync <M> input,
Action <M, Queue <T> > selector, Action <Queue <T> > finalizer = null)
{
var buffer = new Queue <T> ();
return(EnumerableAsync.Produce <T>(async yieldAsync =>
{
await input.ForEach(async m =>
{
selector(m, buffer);
while (buffer.Count > 0)
{
await yieldAsync.YieldAsync(buffer.Dequeue());
}
return true;
});
if (finalizer != null)
{
finalizer(buffer);
while (buffer.Count > 0)
{
await yieldAsync.YieldAsync(buffer.Dequeue());
}
}
}));
}
public async Task Write(Func <Stream> getStream, Action <Stream> releaseStream, IEnumerableAsync <Message[]> messages)
{
var stream = getStream();
try
{
using (var streamWriter = new StreamWriter(stream, Encoding.ASCII, 32 * 1024, true))
await messages.ForEach(async batch =>
{
foreach (var m in batch)
{
await streamWriter.WriteAsync(string.Format(
"[{0}:{1}:{2:MMdd\\/HHmmss.fff}:{3}:{4}({5})] {6}\n",
m.ProcessId,
m.ThreadId,
m.Timestamp,
m.Severity,
m.File,
m.LineNum,
m.Text
));
}
return(true);
});
}
finally
{
releaseStream(stream);
}
}
async Task <List <NodeDetectionToken.ConsoleLogEntry> > GetLogs(IEnumerableAsync <MessagePrefixesPair[]> input)
{
var retVal = new Dictionary <string, NodeDetectionToken.ConsoleLogEntry?>();
await input.ForEach(messages =>
{
foreach (var msg in messages)
{
if (msg.Message.File == "CONSOLE")
{
var m = consoleLogRegex.Match(msg.Message.Text);
if (m.Success)
{
var entry = new NodeDetectionToken.ConsoleLogEntry(m.Groups[1].Value, msg.Message.Timestamp);
if (retVal.ContainsKey(entry.LogText))
{
retVal[entry.LogText] = null;
}
else
{
retVal[entry.LogText] = entry;
}
}
}
}
return(Task.FromResult(true));
});
return(retVal.Values.Where(x => x.HasValue).Select(x => x.Value).ToList());
}
public async Task Write(Func <Stream> getStream, Action <Stream> releaseStream, IEnumerableAsync <Message[]> messages)
{
var stream = getStream();
try
{
using (var streamWriter = new StreamWriter(stream, Encoding.ASCII, 32 * 1024, true))
await messages.ForEach(async batch =>
{
foreach (var m in batch)
{
await streamWriter.WriteAsync(string.Format(
@"{0:yyyy-MM-ddTHH\:mm\:ss.fff} {1}#{2} {3} {4} {5}" + "\n",
m.Timestamp,
m.ObjectType,
m.ObjectId,
m.Severity,
m.MessageType,
m.Text
));
}
return(true);
});
}
finally
{
releaseStream(stream);
}
}
public async Task Write(Func <Stream> getStream, Action <Stream> releaseStream, IEnumerableAsync <Message[]> messages)
{
var stream = getStream();
try
{
using (var streamWriter = new StreamWriter(stream, Encoding.ASCII, 32 * 1024, true))
await messages.ForEach(async batch =>
{
foreach (var m in batch)
{
await streamWriter.WriteAsync(string.Format(
"{0:yyyy-MM-ddTHH:mm:ss.ffffff}|{1}|{2}|{3}|{4}|{5}\n",
m.Timestamp,
m.RootObjectType.ToString(),
m.RootObjectId.ToString(),
m.ObjectId.ToString(),
m.PropName.ToString(),
m.PropValue.ToString()
));
}
return(true);
});
}
finally
{
releaseStream(stream);
}
}
async Task <uint[]> IProcessIdDetector.DetectProcessId(IEnumerableAsync <MessagePrefixesPair <Message>[]> input)
{
var retVal = new HashSet <uint>();
await input.ForEach(messages =>
{
foreach (var msg in messages)
{
if (msg.Prefixes.Contains(dataCollectedPrefix1) || msg.Prefixes.Contains(dataCollectedPrefix2))
{
var arr = DevTools.Events.LogMessage.Parse(msg.Message.Text)?.ParsePayload <DevTools.Events.Tracing.DataCollected>()?.value;
if (arr != null)
{
foreach (var i in arr)
{
if (i.pid != null)
{
retVal.Add(i.pid.Value);
}
}
}
}
}
return(Task.FromResult(true));
});
return(retVal.ToArray());
}
async Task <List <NodeDetectionToken.ConsoleLogEntry> > GetLogs(IEnumerableAsync <MessagePrefixesPair <Message>[]> input)
{
var retVal = new Dictionary <string, NodeDetectionToken.ConsoleLogEntry?>();
await input.ForEach(messages =>
{
foreach (var msg in messages)
{
if (msg.Prefixes.Contains(consoleApiPrefix))
{
var arg = DevTools.Events.LogMessage.Parse(msg.Message.Text)?.ParsePayload <DevTools.Events.Runtime.LogAPICalled>()?.args?[0];
if (arg != null && arg.type == "string")
{
var entry = new NodeDetectionToken.ConsoleLogEntry(arg.value.ToString(), msg.Message.Timestamp);
if (retVal.ContainsKey(entry.LogText))
{
retVal[entry.LogText] = null;
}
else
{
retVal[entry.LogText] = entry;
}
}
}
}
return(Task.FromResult(true));
});
return(retVal.Values.Where(x => x.HasValue).Select(x => x.Value).ToList());
}
public async Task Write(Func <Stream> getStream, Action <Stream> releaseStream, IEnumerableAsync <Message[]> messages)
{
var stream = getStream();
try
{
using (var streamWriter = new StreamWriter(stream, Encoding.ASCII, 32 * 1024, true))
await messages.ForEach(async batch =>
{
foreach (var m in batch)
{
var ts = TimeUtils.ToUnixTimestampMillis(m.Timestamp);
/*await streamWriter.WriteAsync(string.Format(
* "[{0}{4}{1:D3}][{2}]: {3}\n",
* ts/1000,
* ts%1000,
* m.Severity,
* m.Text,
* m.MillisSeparator
* ));*/
}
return(true);
});
}
finally
{
releaseStream(stream);
}
}
public async static Task <List <T> > ToFlatList <T>(this IEnumerableAsync <T[]> input)
{
var ret = new List <T>();
await input.ForEach(x =>
{
ret.AddRange(x);
return(Task.FromResult(true));
});
return(ret);
}
static IEnumerableAsync <FileRange.Range> IterateMatchRanges(
IEnumerableAsync <Checkpoint> checkpoints, long threshhold, ProgressAndCancellation progressAndCancellation)
{
return(EnumerableAsync.Produce <FileRange.Range>(async yieldAsync =>
{
FileRange.Range?lastMatch = null;
await checkpoints.ForEach(async checkpoint =>
{
if (lastMatch == null)
{
if (checkpoint.IsMatch)
{
lastMatch = new FileRange.Range(checkpoint.Position, checkpoint.EndPosition);
}
else
{
progressAndCancellation.continuationToken.NextPosition = checkpoint.EndPosition;
progressAndCancellation.HandleTextIterationProgress(checkpoint.EndPosition);
}
}
else
{
FileRange.Range lastMatchVal = lastMatch.Value;
if (checkpoint.Position - lastMatchVal.End < threshhold)
{
if (checkpoint.IsMatch)
{
lastMatch = new FileRange.Range(lastMatchVal.Begin, checkpoint.EndPosition);
}
}
else
{
await yieldAsync.YieldAsync(lastMatchVal);
progressAndCancellation.continuationToken.NextPosition = checkpoint.EndPosition;
progressAndCancellation.HandleTextIterationProgress(checkpoint.EndPosition);
if (checkpoint.IsMatch)
{
lastMatch = new FileRange.Range(checkpoint.Position, checkpoint.EndPosition);
}
else
{
lastMatch = null;
}
}
}
return true;
});
if (lastMatch != null)
{
await yieldAsync.YieldAsync(lastMatch.Value);
}
}));
}
public static IEnumerableAsync <T> Select <T, M>(this IEnumerableAsync <M[]> input, Func <M, T> selector)
{
return(EnumerableAsync.Produce <T>(yieldAsync =>
input.ForEach(async messages =>
{
foreach (var evt in messages.Select(selector))
{
await yieldAsync.YieldAsync(evt);
}
return true;
})
));
}
public static IEnumerableAsync <T> Concat <T>(this IEnumerableAsync <T> sequence1, IEnumerableAsync <T> sequence2)
{
return(Produce <T>(async yieldAsync =>
{
if (!await sequence1.ForEach(yieldAsync.YieldAsync))
{
return;
}
if (!await sequence2.ForEach(yieldAsync.YieldAsync))
{
return;
}
}));
}
public static async Task <bool> Any <T>(this IEnumerableAsync <T> input, Func <T, bool> predecate)
{
bool ret = false;
await input.ForEach(x =>
{
if (predecate(x))
{
ret = true;
}
return(Task.FromResult(!ret));
});
return(ret);
}
public static IEnumerableAsync <Out[]> Select <In, Out>(
this IEnumerableAsync <In[]> input,
Action <In, Queue <Out> > selector,
Action <Queue <Out> > finalSelector = null,
Action <Out> resultPostprocessor = null)
{
var buf = new Queue <Out>();
var emptyOutBatch = new Out[0];
return(Produce <Out[]>(async yieldAsync =>
{
await input.ForEach(async inBatch =>
{
foreach (var i in inBatch)
{
selector(i, buf);
}
Out[] outBatch;
if (buf.Count == 0)
{
outBatch = emptyOutBatch;
}
else
{
outBatch = buf.ToArray();
buf.Clear();
if (resultPostprocessor != null)
{
foreach (var i in outBatch)
{
resultPostprocessor(i);
}
}
}
return await yieldAsync.YieldAsync(outBatch);
});
if (finalSelector != null)
{
finalSelector(buf);
if (resultPostprocessor != null)
{
foreach (var i in buf)
{
resultPostprocessor(i);
}
}
await yieldAsync.YieldAsync(buf.ToArray());
}
}));
}
public static IEnumerableAsync <T> Where <T>(this IEnumerableAsync <T> input, Func <T, bool> predecate)
{
return(Produce <T>(yieldAsync => input.ForEach(async i =>
{
if (predecate(i))
{
if (!await yieldAsync.YieldAsync(i))
{
return false;
}
}
return true;
})));
}
public static IEnumerableAsync <Out> SelectMany <In, Out>(this IEnumerableAsync <In> input, Func <In, IEnumerable <Out> > selector)
{
return(Produce <Out>(yieldAsync => input.ForEach(async i =>
{
foreach (var j in selector(i))
{
if (!await yieldAsync.YieldAsync(j))
{
return false;
}
}
return true;
})));
}
async Task <uint[]> IProcessIdDetector.DetectProcessId(IEnumerableAsync <MessagePrefixesPair <Message>[]> input)
{
var retVal = new HashSet <uint>();
await input.ForEach(messages =>
{
uint pid;
foreach (var msg in messages)
{
if (uint.TryParse(msg.Message.ProcessId.Value, out pid))
{
retVal.Add(pid);
}
}
return(Task.FromResult(true));
});
return(retVal.ToArray());
}
public static IEnumerableAsync <T> SelectMany <M, T>(
this IEnumerableAsync <M[]> input,
Action <M, Queue <T> > selector,
Action <Queue <T> > finalSelector = null,
Action <T> resultPostprocessor = null)
{
var buffer = new Queue <T> ();
Func <T, T> postprocessor;
if (resultPostprocessor != null)
{
postprocessor = x => { resultPostprocessor(x); return(x); }
}
;
else
{
postprocessor = x => x;
}
return(EnumerableAsync.Produce <T>(async yieldAsync =>
{
await input.ForEach(async messages =>
{
foreach (var m in messages)
{
selector(m, buffer);
while (buffer.Count > 0)
{
await yieldAsync.YieldAsync(postprocessor(buffer.Dequeue()));
}
}
return true;
});
if (finalSelector != null)
{
finalSelector(buffer);
while (buffer.Count > 0)
{
await yieldAsync.YieldAsync(postprocessor(buffer.Dequeue()));
}
}
}));
}
public static async Task <T> FirstOrDefault <T>(this IEnumerableAsync <T> input, Func <T, bool> predecate)
{
T result = default;
bool resultSet = false;
await input.ForEach(i =>
{
if (!predecate(i))
{
return(Task.FromResult(true));
}
if (resultSet)
{
return(Task.FromResult(false));
}
result = i;
resultSet = true;
return(Task.FromResult(false));
});
return(result);
}
public static IEnumerableAsync <T> Select <T, M>(this IEnumerableAsync <M[]> input,
Func <M, IEnumerable <T> > selector, Func <IEnumerable <T> > finalizer)
{
return(EnumerableAsync.Produce <T>(async yieldAsync =>
{
await input.ForEach(async messages =>
{
foreach (var evt in messages.SelectMany(selector))
{
await yieldAsync.YieldAsync(evt);
}
return true;
});
if (finalizer != null)
{
foreach (var evt in finalizer())
{
await yieldAsync.YieldAsync(evt);
}
}
}));
}
public static async Task SerializePostprocessorOutput <Evt, Serializer, EvtVisitor>(
this IEnumerableAsync <Evt[]> events,
Func <Action <object, XElement>, Serializer> serializerFactory,
Task <ILogPartToken> rotatedLogPartToken,
ILogPartTokenFactories rotatedLogPartFactories,
Func <object, TextLogEventTrigger> triggersConverter,
string contentsEtagAttr,
string rootElementName,
string outputFileName,
ITempFilesManager tempFiles,
CancellationToken cancellation
) where Evt : IVisitable <EvtVisitor> where Serializer : class, IEventsSerializer, EvtVisitor
{
rotatedLogPartToken = rotatedLogPartToken ?? Task.FromResult <ILogPartToken>(null);
var sortKeyAttr = XName.Get("__key");
var chunks = new List <string>();
Serializer serializer = null;
Action resetSerializer = () =>
{
if (serializer?.Output?.Count > 0)
{
string chunkFileName = tempFiles.GenerateNewName();
chunks.Add(chunkFileName);
using (var writer = XmlWriter.Create(chunkFileName, new XmlWriterSettings()
{
OmitXmlDeclaration = true,
ConformanceLevel = ConformanceLevel.Fragment
}))
{
foreach (var e in serializer.Output.OrderBy(e => e.Attribute(sortKeyAttr).Value))
{
e.WriteTo(writer);
}
}
}
serializer = serializerFactory((trigger, elt) =>
{
triggersConverter(trigger).Save(elt);
elt.SetAttributeValue(sortKeyAttr, ((IOrderedTrigger)trigger).Index.ToString("x8"));
});
};
resetSerializer();
await events.ForEach(batch =>
{
foreach (var e in batch)
{
e.Visit(serializer);
if (serializer.Output.Count >= 8 * 1024)
{
resetSerializer();
}
}
return(Task.FromResult(!cancellation.IsCancellationRequested));
});
resetSerializer();
if (cancellation.IsCancellationRequested)
{
return;
}
using (var outputWriter = XmlWriter.Create(outputFileName, new XmlWriterSettings()
{
Indent = true
}))
{
outputWriter.WriteStartElement(rootElementName);
new PostprocessorOutputETag(contentsEtagAttr).Write(outputWriter);
rotatedLogPartFactories.SafeWriteTo(await rotatedLogPartToken, outputWriter);
var readersSettings = new XmlReaderSettings()
{
ConformanceLevel = ConformanceLevel.Fragment
};
var readers = chunks.Select(chunkFileName => XmlReader.Create(chunkFileName, readersSettings)).ToList();
try
{
var q = new VCSKicksCollection.PriorityQueue <KeyValuePair <XmlReader, XElement> >(Comparer <KeyValuePair <XmlReader, XElement> > .Create((item1, item2) =>
{
return(string.CompareOrdinal(item1.Value.Attribute(sortKeyAttr).Value, item2.Value.Attribute(sortKeyAttr).Value));
}));
Action <XmlReader> enqueueReader = reader =>
{
if (!reader.EOF)
{
if (reader.MoveToContent() != XmlNodeType.Element)
{
throw new InvalidOperationException("bad chunk");
}
q.Enqueue(new KeyValuePair <XmlReader, XElement>(reader, (XElement)XNode.ReadFrom(reader)));
}
};
readers.ForEach(enqueueReader);
while (q.Count > 0)
{
var item = q.Dequeue();
item.Value.Attribute(sortKeyAttr).Remove();
item.Value.WriteTo(outputWriter);
enqueueReader(item.Key);
}
}
finally
{
readers.ForEach(r => r.Dispose());
chunks.ForEach(chunkFileName => tempFiles.DeleteIfTemporary(chunkFileName));
}
outputWriter.WriteEndElement(); // end of root node
}
}
public static IEnumerableAsync <Out> Select <In, Out>(this IEnumerableAsync <In> input, Func <In, Task <Out> > selector)
{
return(Produce <Out>(yieldAsync => input.ForEach(async i => await yieldAsync.YieldAsync(await selector(i)))));
}
public static IEnumerableAsync <Out> Select <In, Out>(this IEnumerableAsync <In> input, Func <In, Out> selector)
{
return(Produce <Out>(yieldAsync => input.ForEach(i => yieldAsync.YieldAsync(selector(i)))));
}