public async Task SendEventsAsync(IReadOnlyCollection <EventData> events, long transmissionSequenceNumber,
CancellationToken cancellationToken)
{
if (events == null || events.Count == 0)
{
return;
}
try
{
var rows = events.Select(s => CreateRowData(s)).ToList();
await InsertDataAsync(rows, cancellationToken);
this.healthReporter.ReportHealthy();
}
catch (OperationCanceledException)
{
return;
}
catch (Exception ex)
{
ErrorHandlingPolicies.HandleOutputTaskError(ex, () =>
{
string errorMessage = nameof(BigQueryOutput) + ": Failed to write events to Bq." + Environment.NewLine + ex.ToString();
this.healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Output);
});
}
}
public async Task SendEventsAsync(IReadOnlyCollection <EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken)
{
if (this.connectionData == null || events == null || events.Count == 0)
{
return;
}
try
{
string currentIndexName = this.GetIndexName(this.connectionData);
if (!string.Equals(currentIndexName, this.connectionData.LastIndexName, StringComparison.Ordinal))
{
await this.EnsureIndexExists(currentIndexName, this.connectionData.Client).ConfigureAwait(false);
this.connectionData.LastIndexName = currentIndexName;
}
BulkRequest request = new BulkRequest();
List <IBulkOperation> operations = new List <IBulkOperation>();
string documentTypeName = this.connectionData.Configuration.EventDocumentTypeName;
foreach (EventData eventData in events)
{
operations.AddRange(GetCreateOperationsForEvent(eventData, currentIndexName, documentTypeName));
}
request.Operations = operations;
if (cancellationToken.IsCancellationRequested)
{
return;
}
// Note: the NEST client is documented to be thread-safe so it should be OK to just reuse the this.esClient instance
// between different SendEventsAsync callbacks.
// Reference: https://www.elastic.co/blog/nest-and-elasticsearch-net-1-3
IBulkResponse response = await this.connectionData.Client.BulkAsync(request).ConfigureAwait(false);
if (!response.IsValid)
{
this.ReportEsRequestError(response, "Bulk upload");
}
else
{
this.healthReporter.ReportHealthy();
}
}
catch (Exception e)
{
ErrorHandlingPolicies.HandleOutputTaskError(e, () =>
{
string errorMessage = nameof(ElasticSearchOutput) + ": diagnostics data upload has failed." + Environment.NewLine + e.ToString();
this.healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Output);
});
}
}
public async Task SendEventsAsync(IReadOnlyCollection <EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken)
{
if (this.connectionData == null || events == null || events.Count == 0)
{
return;
}
try
{
string jsonData = JsonConvert.SerializeObject(events);
string dateString = DateTime.UtcNow.ToString("r");
string signature = BuildSignature(jsonData, dateString);
HttpContent content = new StringContent(jsonData, Encoding.UTF8, JsonContentId);
content.Headers.ContentType = new MediaTypeHeaderValue(JsonContentId);
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, OmsDataUploadUrl);
request.Headers.Add("Authorization", signature);
request.Headers.Add(MsDateHeaderName, dateString);
request.Content = content;
// SendAsync is thread safe
HttpResponseMessage response = await this.connectionData.HttpClient.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (response.IsSuccessStatusCode)
{
this.healthReporter.ReportHealthy();
}
else
{
string responseContent = string.Empty;
try
{
responseContent = await response.Content.ReadAsStringAsync();
}
catch { }
string errorMessage = $"{nameof(OmsOutput)}: OMS REST API returned an error. Code: {response.StatusCode} Description: {response.ReasonPhrase} {responseContent}";
this.healthReporter.ReportProblem(errorMessage);
}
}
catch (Exception e)
{
ErrorHandlingPolicies.HandleOutputTaskError(e, () =>
{
string errorMessage = nameof(OmsOutput) + ": an error occurred while sending data to OMS: " + Environment.NewLine + e.ToString();
this.healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Output);
});
}
}
public async Task SendEventsAsync(IReadOnlyCollection <EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken)
{
try
{
await splunkHttpEventCollectorClient.SendEventsAsync(events, cancellationToken);
healthReporter.ReportHealthy();
}
catch (Exception e)
{
ErrorHandlingPolicies.HandleOutputTaskError(e, () =>
{
string errorMessage = $"{nameof(SplunkOutput)}: An error occurred while sending data to Splunk. Exception: {e}";
healthReporter.ReportProblem(errorMessage, EventFlowContextIdentifiers.Output);
});
}
}
public async Task SendDataAsync(EventData[] events)
{
if (events.Length > 0)
{
try
{
await this.output.SendEventsAsync(events, Interlocked.Increment(ref transmissionSequenceNumber), this.cancellationToken).ConfigureAwait(false);
}
catch (Exception e)
{
ErrorHandlingPolicies.HandleOutputTaskError(e, () =>
{
this.healthReporter.ReportWarning(
nameof(DiagnosticPipeline) + ": an output has thrown an exception while sending data" + Environment.NewLine + e.ToString(),
EventFlowContextIdentifiers.Output);
});
}
}
}
public async Task SendEventsAsync(IReadOnlyCollection <EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken)
{
if (events == null || events.Count == 0)
{
return;
}
try
{
var payload = new StringBuilder("");
switch (configuration.Format)
{
case HttpOutputFormat.Json:
payload.Append(JsonConvert.SerializeObject(events, SerializerSettings));
break;
case HttpOutputFormat.JsonLines:
foreach (EventData evt in events)
{
payload.AppendLine(JsonConvert.SerializeObject(evt, SerializerSettings));
}
break;
}
HttpContent contentPost = new StringContent(payload.ToString(), Encoding.UTF8, configuration.HttpContentType);
HttpResponseMessage response = await httpClient.PostAsync(new Uri(configuration.ServiceUri), contentPost);
response.EnsureSuccessStatusCode();
this.healthReporter.ReportHealthy();
}
catch (Exception e)
{
ErrorHandlingPolicies.HandleOutputTaskError(e, () =>
{
string errorMessage = nameof(HttpOutput) + ": diagnostic data upload failed: " + Environment.NewLine + e.ToString();
this.healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Output);
});
}
}
public async Task SendEventsAsync(IReadOnlyCollection <EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken)
{
if (this.cloudTable == null || events == null || events.Count == 0)
{
return;
}
DateTime now = DateTime.UtcNow;
string partitionKey = now.ToString("yyyyMMddhhmm") + KeySegmentSeparator + (transmissionSequenceNumber % MaxConcurrentPartitions).ToString("D2");
string rowKeyPrefix = now.ToString("ssfff");
try
{
TableBatchOperation batchOperation = new TableBatchOperation();
foreach (EventData eventData in events)
{
DynamicTableEntity entity = this.ToTableEntity(eventData, partitionKey, rowKeyPrefix);
TableOperation insertOperation = TableOperation.Insert(entity);
batchOperation.Add(insertOperation);
}
if (cancellationToken.IsCancellationRequested)
{
return;
}
// CONSIDER exposing TableRequestOptions and OperationContext for the batch operation
await this.cloudTable.ExecuteBatchAsync(batchOperation, null, null, cancellationToken).ConfigureAwait(false);
this.healthReporter.ReportHealthy();
}
catch (Exception e)
{
ErrorHandlingPolicies.HandleOutputTaskError(e, () =>
{
string errorMessage = nameof(TableStorageSender) + ": diagnostics data upload has failed." + Environment.NewLine + e.ToString();
this.healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Output);
});
}
}
public Task SendEventsAsync(IReadOnlyCollection<EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken)
{
if (this.telemetryClient == null || events == null || events.Count == 0)
{
return CompletedTask;
}
try
{
foreach (var e in events)
{
if (cancellationToken.IsCancellationRequested)
{
return CompletedTask;
}
IReadOnlyCollection<EventMetadata> metadata;
bool tracked = false;
if (e.TryGetMetadata(MetricData.MetricMetadataKind, out metadata))
{
tracked = TrackMetric(e, metadata);
}
else if (e.TryGetMetadata(RequestData.RequestMetadataKind, out metadata))
{
tracked = TrackRequest(e, metadata);
}
else if (e.TryGetMetadata(DependencyData.DependencyMetadataKind, out metadata))
{
tracked = TrackDependency(e, metadata);
}
else if (e.TryGetMetadata(ExceptionData.ExceptionMetadataKind, out metadata))
{
tracked = TrackException(e, metadata);
}
else if (e.TryGetMetadata(EventTelemetryData.EventMetadataKind, out metadata))
{
tracked = TrackAiEvent(e, metadata);
}
if (!tracked)
{
object message = null;
e.Payload.TryGetValue("Message", out message);
TraceTelemetry t = new TraceTelemetry(message as string ?? string.Empty);
t.SeverityLevel = ToSeverityLevel[(int)e.Level];
AddProperties(t, e);
telemetryClient.TrackTrace(t);
}
}
telemetryClient.Flush();
this.healthReporter.ReportHealthy();
}
catch (Exception e)
{
ErrorHandlingPolicies.HandleOutputTaskError(e, () =>
{
string errorMessage = nameof(ApplicationInsightsOutput) + ": diagnostics data upload has failed." + Environment.NewLine + e.ToString();
this.healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Output);
});
}
return CompletedTask;
}
public async Task SendEventsAsync(IReadOnlyCollection <EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken)
{
// Get a reference to the current connections array first, just in case there is another thread wanting to clean
// up the connections with CleanUpAsync(), we won't get a null reference exception here.
IEventHubClient[] currentClients = Interlocked.CompareExchange <IEventHubClient[]>(ref this.clients, this.clients, this.clients);
if (currentClients == null || events == null || events.Count == 0)
{
return;
}
try
{
var groupedEventData = events.GroupBy(e => string.IsNullOrEmpty(outputConfiguration.PartitionKeyProperty) == false && e.TryGetPropertyValue(outputConfiguration.PartitionKeyProperty, out var partionKeyData) ? partionKeyData.ToString() : string.Empty, e => e);
List <Task> tasks = new List <Task>();
foreach (var partitionedEventData in groupedEventData)
{
//assemble the full list of MessagingEventData items plus their messageSize
List <(MessagingEventData message, int messageSize)> batchRecords = partitionedEventData.Select(
e => new Tuple <MessagingEventData, int>(e.ToMessagingEventData(SerializerSettings, out var messageSize), messageSize).ToValueTuple()).ToList();
SendBatch(batchRecords);
void SendBatch(IReadOnlyCollection <(MessagingEventData message, int messageSize)> batch)
{
// Since event hub limits each message/batch to be a certain size, we need to
// keep checking the size in bytes of the batch and recursively keep splitting into two batches as needed
if (batch.Count >= 2 && batch.Sum(b => b.messageSize) > EventHubMessageSizeLimit)
{
//the batch total message size is too big to send to EventHub, but it still contains at least two items,
//so we split the batch up in half and recusively call the inline SendBatch() method with the two new smaller batches
var indexMiddle = batch.Count / 2;
SendBatch(batch.Take(indexMiddle).ToList());
SendBatch(batch.Skip(indexMiddle).ToList());
return;
}
IEventHubClient hubClient = currentClients[transmissionSequenceNumber % ConcurrentConnections];
if (string.IsNullOrEmpty(partitionedEventData.Key))
{
tasks.Add(hubClient.SendAsync(batch.Select(b => b.message)));
}
else
{
tasks.Add(hubClient.SendAsync(batch.Select(b => b.message), partitionedEventData.Key));
}
}
if (cancellationToken.IsCancellationRequested)
{
return;
}
}
await Task.WhenAll(tasks).ConfigureAwait(false);
this.healthReporter.ReportHealthy();
}
catch (Exception e)
{
ErrorHandlingPolicies.HandleOutputTaskError(e, () =>
{
string errorMessage = nameof(EventHubOutput) + ": diagnostics data upload has failed." + Environment.NewLine + e.ToString();
this.healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Output);
});
}
}
public async Task SendEventsAsync(IReadOnlyCollection <EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken)
{
// Get a reference to the current connections array first, just in case there is another thread wanting to clean
// up the connections with CleanUpAsync(), we won't get a null reference exception here.
IEventHubClient[] currentClients = Interlocked.CompareExchange <IEventHubClient[]>(ref this.clients, this.clients, this.clients);
if (currentClients == null || events == null || events.Count == 0)
{
return;
}
try
{
// Since event hub limits each message/batch to be a certain size, we need to
// keep checking the size for exceeds and split into a new batch as needed
List <List <MessagingEventData> > batches = new List <List <MessagingEventData> >();
int batchByteSize = 0;
foreach (EventData eventData in events)
{
int messageSize;
MessagingEventData messagingEventData = eventData.ToMessagingEventData(out messageSize);
// If we don't have a batch yet, or the addition of this message will exceed the limit for this batch, then
// start a new batch.
if (batches.Count == 0 ||
batchByteSize + messageSize > EventHubMessageSizeLimit)
{
batches.Add(new List <MessagingEventData>());
batchByteSize = 0;
}
batchByteSize += messageSize;
List <MessagingEventData> currentBatch = batches[batches.Count - 1];
currentBatch.Add(messagingEventData);
}
if (cancellationToken.IsCancellationRequested)
{
return;
}
IEventHubClient hubClient = currentClients[transmissionSequenceNumber % ConcurrentConnections];
List <Task> tasks = new List <Task>();
foreach (List <MessagingEventData> batch in batches)
{
tasks.Add(hubClient.SendAsync(batch));
}
await Task.WhenAll(tasks).ConfigureAwait(false);
this.healthReporter.ReportHealthy();
}
catch (Exception e)
{
ErrorHandlingPolicies.HandleOutputTaskError(e, () =>
{
string errorMessage = nameof(EventHubOutput) + ": diagnostics data upload has failed." + Environment.NewLine + e.ToString();
this.healthReporter.ReportWarning(errorMessage, EventFlowContextIdentifiers.Output);
});
}
}
