public async Task ProcessLogsStream(string id, Stream stream, ModuleLogOptions logOptions, Func <ArraySegment <byte>, Task> callback)
{
GraphBuilder graphBuilder = GraphBuilder.CreateParsingGraphBuilder(stream, b => this.logMessageParser.Parse(b, id));
logOptions.Filter.LogLevel.ForEach(l => graphBuilder.AddFilter(m => m.LogLevel == l));
logOptions.Filter.Regex.ForEach(r => graphBuilder.AddFilter(m => r.IsMatch(m.Text)));
async Task <bool> ConsumerCallback(ArraySegment <byte> a)
{
await callback(a);
return(true);
}
ArraySegment <byte> BasicMapper(ModuleLogMessageData l)
=> logOptions.ContentType == LogsContentType.Text
? new ArraySegment <byte>(l.FullFrame.ToArray())
: new ArraySegment <byte>(l.ToBytes());
var mappers = new List <Func <ArraySegment <byte>, ArraySegment <byte> > >();
if (logOptions.ContentEncoding == LogsContentEncoding.Gzip)
{
mappers.Add(m => new ArraySegment <byte>(Compression.CompressToGzip(m.Array)));
}
IRunnableGraph <Task> graph = graphBuilder.GetStreamingGraph(
ConsumerCallback,
BasicMapper,
mappers);
await graph.Run(this.materializer);
}
async Task <byte[]> GetProcessedLogs(Stream logsStream, ModuleLogOptions logOptions)
{
byte[] logBytes = await this.ProcessByContentType(logsStream, logOptions);
logBytes = ProcessByContentEncoding(logBytes, logOptions.ContentEncoding);
return(logBytes);
}
// The id parameter is a regex. Logs for all modules that match this regex are processed.
public Task GetLogsStream(string id, ModuleLogOptions logOptions, Func <ArraySegment <byte>, Task> callback, CancellationToken cancellationToken)
{
Preconditions.CheckNotNull(logOptions, nameof(logOptions));
Preconditions.CheckNotNull(callback, nameof(callback));
Events.StreamingLogs(id, logOptions);
return(this.GetLogsStreamInternal(id, logOptions, callback, cancellationToken));
}
public async Task <byte[]> GetLogs(ModuleLogOptions logOptions, CancellationToken cancellationToken)
{
Stream logsStream = await this.runtimeInfoProvider.GetModuleLogs(logOptions.Id, false, Option.None <int>(), cancellationToken);
byte[] logBytes = await this.GetProcessedLogs(logsStream, logOptions);
return(logBytes);
}
public async Task <byte[]> GetLogs(string id, ModuleLogOptions logOptions, CancellationToken cancellationToken)
{
Preconditions.CheckNotNull(logOptions, nameof(logOptions));
Stream logsStream = await this.runtimeInfoProvider.GetModuleLogs(id, false, logOptions.Filter.Tail, logOptions.Filter.Since, cancellationToken);
Events.ReceivedStream(id);
byte[] logBytes = await this.GetProcessedLogs(id, logsStream, logOptions);
return(logBytes);
}
// The id parameter is a regex. Logs for all modules that match this regex are processed.
public async Task GetLogsStream(string id, ModuleLogOptions logOptions, Func <ArraySegment <byte>, Task> callback, CancellationToken cancellationToken)
{
Preconditions.CheckNotNull(logOptions, nameof(logOptions));
Preconditions.CheckNotNull(callback, nameof(callback));
IEnumerable <string> modules = (await this.runtimeInfoProvider.GetModules(cancellationToken))
.Select(m => m.Name);
ISet <string> matchingIds = GetMatchingIds(id, modules);
Events.StreamingLogs(matchingIds, logOptions);
IEnumerable <Task> streamingTasks = matchingIds.Select(i => this.GetLogsStreamInternal(i, logOptions, callback, cancellationToken));
await Task.WhenAll(streamingTasks);
}
async Task <byte[]> ProcessByContentType(Stream logsStream, ModuleLogOptions logOptions)
{
switch (logOptions.ContentType)
{
case LogsContentType.Json:
IEnumerable <ModuleLogMessage> logMessages = await this.logsProcessor.GetMessages(logsStream, logOptions.Id);
return(logMessages.ToBytes());
default:
IEnumerable <string> logTexts = await this.logsProcessor.GetText(logsStream);
string logTextString = logTexts.Join(string.Empty);
return(logTextString.ToBytes());
}
}
public async Task <byte[]> GetLogs(string id, ModuleLogOptions logOptions, CancellationToken cancellationToken)
{
Preconditions.CheckNotNull(logOptions, nameof(logOptions));
if (logOptions.ContentType == LogsContentType.Json)
{
logOptions.Filter.IncludeTimestamp = Option.Some(true);
}
Stream logsStream = await this.runtimeInfoProvider.GetModuleLogs(id, false, logOptions.Filter.Tail, logOptions.Filter.Since, logOptions.Filter.Until, logOptions.Filter.IncludeTimestamp, cancellationToken);
Events.ReceivedStream(id);
byte[] logBytes = await this.GetProcessedLogs(id, logsStream, logOptions);
return(logBytes);
}
public async Task ProcessLogsStream(string id, Stream stream, ModuleLogOptions logOptions, Func <ArraySegment <byte>, Task> callback)
{
GraphBuilder graphBuilder = GraphBuilder.CreateParsingGraphBuilder(stream, b => this.logMessageParser.Parse(b, id));
logOptions.Filter.LogLevel.ForEach(l => graphBuilder.AddFilter(m => m.LogLevel == l));
logOptions.Filter.Regex.ForEach(r => graphBuilder.AddFilter(m => r.IsMatch(m.Text)));
async Task <bool> ConsumerCallback(ArraySegment <byte> a)
{
await callback(a);
return(true);
}
ArraySegment <byte> BasicMapper(ModuleLogMessageData l)
=> logOptions.ContentType == LogsContentType.Text
? new ArraySegment <byte>(l.FullText.ToBytes())
: new ArraySegment <byte>(l.ToBytes());
var sourceMappers = new List <Func <Source <ArraySegment <byte>, AkkaNet.NotUsed>, Source <ArraySegment <byte>, AkkaNet.NotUsed> > >();
if (logOptions.ContentEncoding == LogsContentEncoding.Gzip)
{
sourceMappers.Add(
s => logOptions.OutputGroupingConfig.Map(o => GroupingGzipMapper(s, o))
.GetOrElse(() => NonGroupingGzipMapper(s)));
}
if (logOptions.OutputFraming == LogOutputFraming.SimpleLength)
{
sourceMappers.Add(SimpleLengthFramingMapper);
}
IRunnableGraph <Task> graph = graphBuilder.GetStreamingGraph(
ConsumerCallback,
BasicMapper,
sourceMappers);
await graph.Run(this.materializer);
}
