private async Task SendBatchAsync(Metric[] metrics, ProcessorMetrics processorMetrics)
{
await retriableCallStrategy.CallAsync(() => graphiteClient.SendAsync(metrics), ex =>
{
processorMetrics.SendingErrorCounter.Add();
return(true);
}, log);
}
public void Process(List <AirlockEvent <SampleEvent> > events, ProcessorMetrics processorMetrics)
{
log.Info($"New events batch has arrived of size {events.Count}");
foreach (var @event in events)
{
log.Info($"{@event.RoutingKey}|{@event.Timestamp:O} - {@event.Payload.Message}");
}
}
public void Process(List <AirlockEvent <byte[]> > events, ProcessorMetrics processorMetrics)
{
foreach (var @event in events)
{
ValidateRoutingKey(@event.RoutingKey);
var metricEvent = TryGetMetricEvent(@event);
if (metricEvent != null)
{
eventsTimestampProvider.AddTimestamp(metricEvent.Timestamp);
aggregator.ProcessMetricEvent(metricEvent);
}
processorMetrics.EventProcessedCounter.Add();
}
}
private void ProcessEvent(AirlockEvent <Span> @event, ProcessorMetrics processorMetrics)
{
try
{
contrailsClient.AddSpan(@event.Payload).GetAwaiter().GetResult();
processorMetrics.EventProcessedCounter.Add();
}
catch (Exception e)
{
processorMetrics.EventFailedCounter.Add();
processorMetrics.SendingErrorCounter.Add();
log.Error(e);
}
}
public sealed override void Process(List <AirlockEvent <LogEventData> > events, ProcessorMetrics processorMetrics)
{
var bulks = events
.Select(
@event =>
{
RoutingKey.Parse(@event.RoutingKey, out var project, out var environment, out var service,
out var _);
var indexName = $"{project}-{environment}";
var indexRecordMeta = BuildIndexRecordMeta(indexName);
var indexRecord = BuildIndexRecord(@event, service);
return(new { indexName, indexRecordMeta, indexRecord });
})
.GroupBy(x => x.indexName)
.SelectMany(g => g.Batch(10000).Select(records =>
{
var postDataItems = new List <object>();
foreach (var record in records)
{
postDataItems.Add(record.indexRecordMeta);
postDataItems.Add(record.indexRecord);
}
var postData = new PostData <object>(postDataItems);
return(new { postData, recordsCount = postDataItems.Count / 2 });
}))
.ToList();
Parallel.ForEach(
bulks,
bulk =>
{
try
{
BulkIndex(bulk.postData, processorMetrics.SendingErrorCounter);
processorMetrics.EventProcessedCounter.Add(bulk.recordsCount);
}
catch (Exception)
{
processorMetrics.EventFailedCounter.Add(bulk.recordsCount);
throw;
}
});
}
public sealed override void Process(List <AirlockEvent <MetricEvent> > events, ProcessorMetrics processorMetrics)
{
var metrics = events.SelectMany(x => metricConverter.Convert(x.RoutingKey, x.Payload)).ToArray();
try
{
SendBatchAsync(metrics, processorMetrics).GetAwaiter().GetResult();
processorMetrics.EventProcessedCounter.Add(events.Count);
}
catch (Exception)
{
processorMetrics.EventFailedCounter.Add(events.Count);
throw;
}
}
public sealed override void Process(List <AirlockEvent <Span> > events, ProcessorMetrics processorMetrics)
{
Parallel.ForEach(events, new ParallelOptions {
MaxDegreeOfParallelism = maxCassandraTasks
}, @event => ProcessEvent(@event, processorMetrics));
}
public sealed override void Process(List <AirlockEvent <LogEventData> > events, ProcessorMetrics processorMetrics)
{
var errorEvents = events.Where(x => x.Payload.Level == LogLevel.Error || x.Payload.Level == LogLevel.Fatal);
var throttledErrorEvents = ThrottleEvents(errorEvents.OrderBy(x => x.Timestamp), processorMetrics.EventIgnoredCounter);
var packets = throttledErrorEvents.Select(
x =>
{
var logEvent = x.Payload;
var jsonPacket = new JsonPacket(sentryProjectId)
{
Level = logEvent.Level == LogLevel.Error ? ErrorLevel.Error : ErrorLevel.Fatal,
Tags = logEvent.Properties,
TimeStamp = logEvent.Timestamp.UtcDateTime,
Exceptions = logEvent.Exceptions?.Select(ex => ex.ToSentry()).ToList(),
Message = logEvent.Message,
MessageObject = logEvent.Message
};
JsonPacketPatcher.PatchPacket(jsonPacket);
return(jsonPacket);
});
Parallel.ForEach(
packets,
new ParallelOptions {
MaxDegreeOfParallelism = processorSettings.MaxTasks
},
packet =>
{
try
{
packetSender.SendPacket(packet, processorMetrics.SendingErrorCounter);
try
{
processorMetrics.EventProcessedCounter.Add();
}
catch (Exception e)
{
log.Error(e);
}
}
catch (Exception e)
{
processorMetrics.EventFailedCounter.Add();
log.Error(e);
}
});
}