private async Task SendEventsAsync( IEnumerable <EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken) { if (events == null) { return; } try { var currentIndexName = this.GetIndexName(this.connectionData); if (!string.Equals(currentIndexName, this.connectionData.LastIndexName, StringComparison.Ordinal)) { await this.EnsureIndexExists(currentIndexName, this.connectionData.Client); this.connectionData.LastIndexName = currentIndexName; } var request = new BulkRequest(); request.Refresh = true; var operations = new List <IBulkOperation>(); foreach (var eventData in events) { var operation = new BulkCreateOperation <EventData>(eventData); operation.Index = currentIndexName; operation.Type = EventDocumentTypeName; operations.Add(operation); } request.Operations = operations; if (cancellationToken.IsCancellationRequested) { return; } // Note: the NEST client is documented to be thread-safe so it should be OK to just reuse the this.esClient instance // between different SendEventsAsync callbacks. // Reference: https://www.elastic.co/blog/nest-and-elasticsearch-net-1-3 var response = await this.connectionData.Client.BulkAsync(request); if (!response.IsValid) { this.ReportEsRequestError(response, "Bulk upload"); } this.ReportListenerHealthy(); } catch (Exception e) { this.ReportListenerProblem("Diagnostics data upload has failed." + Environment.NewLine + e); } }
private static BulkCreateOperation<IPageDataIndexModel> CreateBulkOperation(IPageDataIndexModel indexModel) { var bulkCreateOperation = new BulkCreateOperation<IPageDataIndexModel>(indexModel) { Type = TypeName.Create(indexModel.GetType()) }; return bulkCreateOperation; }
private async Task SendEventsAsync(IEnumerable<EventData> events, long transmissionSequenceNumber, CancellationToken cancellationToken) { if (events == null) { return; } try { string currentIndexName = this.GetIndexName(this.connectionData); if (!string.Equals(currentIndexName, this.connectionData.LastIndexName, StringComparison.Ordinal)) { await this.EnsureIndexExists(currentIndexName, this.connectionData.Client); this.connectionData.LastIndexName = currentIndexName; } BulkRequest request = new BulkRequest(); request.Refresh = true; List<IBulkOperation> operations = new List<IBulkOperation>(); foreach (EventData eventData in events) { BulkCreateOperation<EventData> operation = new BulkCreateOperation<EventData>(eventData); operation.Index = currentIndexName; operation.Type = EventDocumentTypeName; operations.Add(operation); } request.Operations = operations; if (cancellationToken.IsCancellationRequested) { return; } // Note: the NEST client is documented to be thread-safe so it should be OK to just reuse the this.esClient instance // between different SendEventsAsync callbacks. // Reference: https://www.elastic.co/blog/nest-and-elasticsearch-net-1-3 IBulkResponse response = await this.connectionData.Client.BulkAsync(request); if (!response.IsValid) { this.ReportEsRequestError(response, "Bulk upload"); } this.ReportListenerHealthy(); } catch (Exception e) { this.ReportListenerProblem("Diagnostics data upload has failed." + Environment.NewLine + e.ToString()); } }
private IEnumerable <IBulkOperation> GetCreateOperationsForEvent(EventData eventData, string currentIndexName, string documentTypeName) { bool reportedAsMetricOrRequest = false; BulkCreateOperation <EventData> operation; // Synthesize a separate record for each metric and request metadata associated with the event IReadOnlyCollection <EventMetadata> metadataSet; if (eventData.TryGetMetadata(MetricData.MetricMetadataKind, out metadataSet)) { foreach (var metricMetadata in metadataSet) { MetricData metricData; var result = MetricData.TryGetData(eventData, metricMetadata, out metricData); if (result.Status != DataRetrievalStatus.Success) { this.healthReporter.ReportProblem("ElasticSearchOutput: " + result.Message, EventFlowContextIdentifiers.Output); continue; } var metricEventData = eventData.DeepClone(); metricEventData.Payload[nameof(MetricData.MetricName)] = metricData.MetricName; metricEventData.Payload[nameof(MetricData.Value)] = metricData.Value; operation = new BulkCreateOperation <EventData>(metricEventData); operation.Index = currentIndexName; operation.Type = documentTypeName; reportedAsMetricOrRequest = true; yield return(operation); } } if (eventData.TryGetMetadata(RequestData.RequestMetadataKind, out metadataSet)) { foreach (var requestMetadata in metadataSet) { RequestData requestData; var result = RequestData.TryGetData(eventData, requestMetadata, out requestData); if (result.Status != DataRetrievalStatus.Success) { this.healthReporter.ReportProblem("ElasticSearchOutput: " + result.Message, EventFlowContextIdentifiers.Output); continue; } var requestEventData = eventData.DeepClone(); requestEventData.Payload[nameof(RequestData.RequestName)] = requestData.RequestName; if (requestData.Duration != null) { requestEventData.Payload[nameof(RequestData.Duration)] = requestData.Duration; } if (requestData.IsSuccess != null) { requestEventData.Payload[nameof(RequestData.IsSuccess)] = requestData.IsSuccess; } if (requestData.ResponseCode != null) { requestEventData.Payload[nameof(RequestData.ResponseCode)] = requestData.ResponseCode; } operation = new BulkCreateOperation <EventData>(requestEventData); operation.Index = currentIndexName; operation.Type = documentTypeName; reportedAsMetricOrRequest = true; yield return(operation); } } if (!reportedAsMetricOrRequest) { operation = new BulkCreateOperation <EventData>(eventData); operation.Index = currentIndexName; operation.Type = documentTypeName; yield return(operation); } }