private async Task SendBatchAsync(Metric[] metrics, ProcessorMetrics processorMetrics) { await retriableCallStrategy.CallAsync(() => graphiteClient.SendAsync(metrics), ex => { processorMetrics.SendingErrorCounter.Add(); return(true); }, log); }
public void Process(List <AirlockEvent <SampleEvent> > events, ProcessorMetrics processorMetrics) { log.Info($"New events batch has arrived of size {events.Count}"); foreach (var @event in events) { log.Info($"{@event.RoutingKey}|{@event.Timestamp:O} - {@event.Payload.Message}"); } }
public void Process(List <AirlockEvent <byte[]> > events, ProcessorMetrics processorMetrics) { foreach (var @event in events) { ValidateRoutingKey(@event.RoutingKey); var metricEvent = TryGetMetricEvent(@event); if (metricEvent != null) { eventsTimestampProvider.AddTimestamp(metricEvent.Timestamp); aggregator.ProcessMetricEvent(metricEvent); } processorMetrics.EventProcessedCounter.Add(); } }
private void ProcessEvent(AirlockEvent <Span> @event, ProcessorMetrics processorMetrics) { try { contrailsClient.AddSpan(@event.Payload).GetAwaiter().GetResult(); processorMetrics.EventProcessedCounter.Add(); } catch (Exception e) { processorMetrics.EventFailedCounter.Add(); processorMetrics.SendingErrorCounter.Add(); log.Error(e); } }
public sealed override void Process(List <AirlockEvent <LogEventData> > events, ProcessorMetrics processorMetrics) { var bulks = events .Select( @event => { RoutingKey.Parse(@event.RoutingKey, out var project, out var environment, out var service, out var _); var indexName = $"{project}-{environment}"; var indexRecordMeta = BuildIndexRecordMeta(indexName); var indexRecord = BuildIndexRecord(@event, service); return(new { indexName, indexRecordMeta, indexRecord }); }) .GroupBy(x => x.indexName) .SelectMany(g => g.Batch(10000).Select(records => { var postDataItems = new List <object>(); foreach (var record in records) { postDataItems.Add(record.indexRecordMeta); postDataItems.Add(record.indexRecord); } var postData = new PostData <object>(postDataItems); return(new { postData, recordsCount = postDataItems.Count / 2 }); })) .ToList(); Parallel.ForEach( bulks, bulk => { try { BulkIndex(bulk.postData, processorMetrics.SendingErrorCounter); processorMetrics.EventProcessedCounter.Add(bulk.recordsCount); } catch (Exception) { processorMetrics.EventFailedCounter.Add(bulk.recordsCount); throw; } }); }
public sealed override void Process(List <AirlockEvent <MetricEvent> > events, ProcessorMetrics processorMetrics) { var metrics = events.SelectMany(x => metricConverter.Convert(x.RoutingKey, x.Payload)).ToArray(); try { SendBatchAsync(metrics, processorMetrics).GetAwaiter().GetResult(); processorMetrics.EventProcessedCounter.Add(events.Count); } catch (Exception) { processorMetrics.EventFailedCounter.Add(events.Count); throw; } }
public sealed override void Process(List <AirlockEvent <Span> > events, ProcessorMetrics processorMetrics) { Parallel.ForEach(events, new ParallelOptions { MaxDegreeOfParallelism = maxCassandraTasks }, @event => ProcessEvent(@event, processorMetrics)); }
public sealed override void Process(List <AirlockEvent <LogEventData> > events, ProcessorMetrics processorMetrics) { var errorEvents = events.Where(x => x.Payload.Level == LogLevel.Error || x.Payload.Level == LogLevel.Fatal); var throttledErrorEvents = ThrottleEvents(errorEvents.OrderBy(x => x.Timestamp), processorMetrics.EventIgnoredCounter); var packets = throttledErrorEvents.Select( x => { var logEvent = x.Payload; var jsonPacket = new JsonPacket(sentryProjectId) { Level = logEvent.Level == LogLevel.Error ? ErrorLevel.Error : ErrorLevel.Fatal, Tags = logEvent.Properties, TimeStamp = logEvent.Timestamp.UtcDateTime, Exceptions = logEvent.Exceptions?.Select(ex => ex.ToSentry()).ToList(), Message = logEvent.Message, MessageObject = logEvent.Message }; JsonPacketPatcher.PatchPacket(jsonPacket); return(jsonPacket); }); Parallel.ForEach( packets, new ParallelOptions { MaxDegreeOfParallelism = processorSettings.MaxTasks }, packet => { try { packetSender.SendPacket(packet, processorMetrics.SendingErrorCounter); try { processorMetrics.EventProcessedCounter.Add(); } catch (Exception e) { log.Error(e); } } catch (Exception e) { processorMetrics.EventFailedCounter.Add(); log.Error(e); } }); }