public async Task ProcessEvents_InvalidDocument_AndUnknownDocumentType_AbandonsInvalidAndUnknownDocuments() { _client.SimulateElasticSearch(); var events = new[] { MockEventData.CreateInvalidEventData(@"{ ""message"": ""invalid type"" }"), MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 1}"), MockEventData.CreateSerilogEventData(@"invalid json"), MockEventData.CreateRoboCustosEventData(@"{ ""message"": ""log message"", ""data"": 2}"), MockEventData.CreateSerilogEventData(@""), MockEventData.CreateExternalTelemetryEventData(@"{ ""message"": ""log message"", ""data"": 3}"), MockEventData.CreateAzureResourcesEventData(@"{ ""message"": ""log message"", ""data"": 4}"), }; await _processor.OpenAsync(_partition); await _processor.ProcessEventsAsync(_partition, events); await _processor.CloseAsync(_partition, CloseReason.Shutdown); // We should try calling BulkAsync once for the initial request, MaxFailedDocumentRetries times to retry the failed documents, // once to abandon the invalid documents, and once to abandon the unknown document type. _client.VerifyBulkAsyncCalled(Times.Exactly(1 + MaxFailedDocumentRetries + 1 + 1)); Assert.Equal(4 + 2 + 3, _client.GetTotalItems()); // 4 successful, 2 ingestionstats, 3 abandoned Assert.Equal(1, _client.GetTotalItemsInIndex("logstash")); Assert.Equal(1, _client.GetTotalItemsInIndex("robointeractions")); Assert.Equal(1, _client.GetTotalItemsInIndex("externaltelemetry")); Assert.Equal(1, _client.GetTotalItemsInIndex("azure-resources")); Assert.Equal(2, _client.GetTotalItemsInIndex("ingestionstats")); Assert.Equal(3, _client.GetTotalItemsInIndex("abandoneddocs")); Assert.Equal(2, _processor.LastBatchFailedDocuments); Assert.Equal(3, _processor.LastBatchAbandonedDocuments); }
public async Task ProcessEvents_DocumentWithNewlines_AbandonsInvalidDocument() { _client.SimulateElasticSearch(); var events = new[] { MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 1}"), MockEventData.CreateSerilogEventData(@" { ""message"": ""documents formatted with newlines are invalid for Elasticsearch bulk upload."", ""data"": 2 }" ), MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 3}"), }; await _processor.OpenAsync(_partition); await _processor.ProcessEventsAsync(_partition, events); await _processor.CloseAsync(_partition, CloseReason.Shutdown); // We should try calling BulkAsync once for the initial request, then once to abandon the invalid document. _client.VerifyBulkAsyncCalled(Times.Exactly(1 + 1)); Assert.Equal(0, _processor.LastBatchFailedDocuments); Assert.Equal(1, _processor.LastBatchAbandonedDocuments); }
public async Task ProcessEvents_MultipleDocumentTypes_Succeeds() { _client.SimulateElasticSearch(); var events = new[] { MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 1}"), MockEventData.CreateRoboCustosEventData(@"{ ""message"": ""log message"", ""data"": 2}"), MockEventData.CreateExternalTelemetryEventData(@"{ ""message"": ""log message"", ""data"": 3}"), MockEventData.CreateAzureResourcesEventData(@"{ ""message"": ""log message"", ""data"": 4}"), }; await _processor.OpenAsync(_partition); await _processor.ProcessEventsAsync(_partition, events); await _processor.CloseAsync(_partition, CloseReason.Shutdown); // We should try calling BulkAsync once for the initial request. _client.VerifyBulkAsyncCalled(Times.Exactly(1)); Assert.Equal(4 + 2, _client.GetTotalItems()); // 4 for serilog/robocustos/externaltelemetry/azure-resources indexes, 2 for ingestionstats Assert.Equal(1, _client.GetTotalItemsInIndex("logstash")); Assert.Equal(1, _client.GetTotalItemsInIndex("robointeractions")); Assert.Equal(1, _client.GetTotalItemsInIndex("externaltelemetry")); Assert.Equal(1, _client.GetTotalItemsInIndex("azure-resources")); Assert.Equal(2, _client.GetTotalItemsInIndex("ingestionstats")); Assert.Equal(0, _processor.LastBatchFailedDocuments); Assert.Equal(0, _processor.LastBatchAbandonedDocuments); }
public void ToBulkString_MultilineContent_MultipleNewlines() { var data = MockEventData.CreateSerilogEventData(body: "{ \"message\": \"log message\n with newline\" }"); var item = BulkItem.FromEventData(data); var bulkBody = item.ToBulkString(); Assert.False(item.IsValid, "BulkItem with document body containing newlines should be invalid."); Assert.Equal(2, bulkBody.Count(c => c == '\n')); }
public void ToBulkString_SimpleContent_OneNewline() { var data = MockEventData.CreateSerilogEventData(body: "{ \"message\": \"log message\" }"); var item = BulkItem.FromEventData(data); var bulkBody = item.ToBulkString(); // BulkString should have exactly one newline when formatted correctly. Assert.Equal(1, bulkBody.Count(c => c == '\n')); }
public void ToBulkString_ComplexContent_OneNewline() { var data = MockEventData.CreateSerilogEventData(body: JsonConvert.SerializeObject(new { message = "log message\n with newline" })); var item = BulkItem.FromEventData(data); var bulkBody = item.ToBulkString(); // BulkString should have exactly one newline when formatted correctly. Assert.True(item.IsValid, "BulkItem with json encoded newlines should be valid."); Assert.Equal(1, bulkBody.Count(c => c == '\n')); }
public async Task ProcessEvents_OneEmptyDocument_AbandonsDocument() { await _processor.OpenAsync(_partition); await _processor.ProcessEventsAsync(_partition, new[] { MockEventData.CreateSerilogEventData("") }); await _processor.CloseAsync(_partition, CloseReason.Shutdown); Assert.Equal(1, _processor.LastBatchFailedDocuments); Assert.Equal(1, _processor.LastBatchAbandonedDocuments); }
public async Task ProcessEvents_OneDocument_Succeeds_OneElasticClientCall() { _client.OnBulkAsync().Returns(HttpStatusCode.OK); await _processor.OpenAsync(_partition); await _processor.ProcessEventsAsync(_partition, new [] { MockEventData.CreateSerilogEventData("{}") }); await _processor.CloseAsync(_partition, CloseReason.Shutdown); _client.VerifyBulkAsyncCalled(Times.Once()); Assert.Equal(0, _processor.LastBatchFailedDocuments); Assert.Equal(0, _processor.LastBatchAbandonedDocuments); }
public async Task ProcessEvents_OneEmptyDocument_AbandonsDocument() { _client.SimulateElasticSearch(); await _processor.OpenAsync(_partition); await _processor.ProcessEventsAsync(_partition, new[] { MockEventData.CreateSerilogEventData("") }); await _processor.CloseAsync(_partition, CloseReason.Shutdown); // We should try calling BulkAsync once for the initial request, MaxFailedDocumentRetries times to retry the failed document, then once to abandon the document. _client.VerifyBulkAsyncCalled(Times.Exactly(1 + MaxFailedDocumentRetries + 1)); Assert.Equal(1, _processor.LastBatchFailedDocuments); Assert.Equal(1, _processor.LastBatchAbandonedDocuments); }
public async Task ProcessEvents_OneUnknownDocumentType_AbandonsUnknownDocumentType() { var events = new[] { MockEventData.CreateInvalidEventData(@"{ ""message"": ""invalid type"" }"), }; await _processor.OpenAsync(_partition); await _processor.ProcessEventsAsync(_partition, events); await _processor.CloseAsync(_partition, CloseReason.Shutdown); Assert.Equal(0, _processor.LastBatchFailedDocuments); // The invalid document type should not be retried. Assert.Equal(1, _processor.LastBatchAbandonedDocuments); // The invalid document type should be abandoned immediately. }
public async Task ProcessEvents_OneDocument_Succeeds() { var events = new[] { MockEventData.CreateSerilogEventData($"{{ \"message\": \"ProcessEvents_OneDocument_Succeeds\", \"@timestamp\":\"{DateTime.UtcNow:o}\", \"data\": 1 }}"), }; await _processor.OpenAsync(_partition); await _processor.ProcessEventsAsync(_partition, events); await _processor.CloseAsync(_partition, CloseReason.Shutdown); Assert.Equal(0, _processor.LastBatchFailedDocuments); Assert.Equal(0, _processor.LastBatchAbandonedDocuments); }
public async Task ProcessEvents_MultipleInvalidDocuments_AbandonsInvalidDocuments() { var events = new[] { MockEventData.CreateSerilogEventData(@"invalid json"), MockEventData.CreateSerilogEventData(@""), MockEventData.CreateSerilogEventData(@"invalid json"), }; await _processor.OpenAsync(_partition); await _processor.ProcessEventsAsync(_partition, events); await _processor.CloseAsync(_partition, CloseReason.Shutdown); Assert.Equal(3, _processor.LastBatchFailedDocuments); Assert.Equal(3, _processor.LastBatchAbandonedDocuments); }
public async Task ProcessEvents_OneInvalidJsonDocument_TwoValidDocuments_AbandonsInvalidJsonDocument() { var events = new[] { MockEventData.CreateSerilogEventData($"{{ \"message\": \"ProcessEvents_OneInvalidJsonDocument_TwoValidDocuments_AbandonsInvalidJsonDocument\", \"@timestamp\":\"{DateTime.UtcNow:o}\", \"data\": 1 }}"), MockEventData.CreateSerilogEventData(@"invalid json"), MockEventData.CreateSerilogEventData($"{{ \"message\": \"ProcessEvents_OneInvalidJsonDocument_TwoValidDocuments_AbandonsInvalidJsonDocument\", \"@timestamp\":\"{DateTime.UtcNow:o}\", \"data\": 2 }}"), }; await _processor.OpenAsync(_partition); await _processor.ProcessEventsAsync(_partition, events); await _processor.CloseAsync(_partition, CloseReason.Shutdown); Assert.Equal(1, _processor.LastBatchFailedDocuments); Assert.Equal(1, _processor.LastBatchAbandonedDocuments); }
public async Task ProcessEvents_OneDocument_WithFailures_RetriesElasticClientCallUntilSuccess() { // Fail the BulkAsync call 100 times with BadGateway, then succeed with OK. const int failureCount = 100; int callCount = 0; _client.OnBulkAsync().Returns(() => ElasticsearchResponseExtensions.GetBulkResponse(++callCount <= failureCount ? HttpStatusCode.BadGateway : HttpStatusCode.OK)); await _processor.OpenAsync(_partition); await _processor.ProcessEventsAsync(_partition, new[] { MockEventData.CreateSerilogEventData("{}") }); await _processor.CloseAsync(_partition, CloseReason.Shutdown); _client.VerifyBulkAsyncCalled(Times.Exactly(failureCount + 1)); Assert.Equal(0, _processor.LastBatchFailedDocuments); Assert.Equal(0, _processor.LastBatchAbandonedDocuments); }
public async Task ProcessEvents_MultipleDocumentTypes_Succeeds() { var events = new[] { MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 1}"), MockEventData.CreateRoboCustosEventData(@"{ ""message"": ""log message"", ""data"": 2}"), MockEventData.CreateExternalTelemetryEventData(@"{ ""message"": ""log message"", ""data"": 3}"), MockEventData.CreateAzureResourcesEventData(@"{ ""message"": ""log message"", ""data"": 4}"), }; await _processor.OpenAsync(_partition); await _processor.ProcessEventsAsync(_partition, events); await _processor.CloseAsync(_partition, CloseReason.Shutdown); Assert.Equal(0, _processor.LastBatchFailedDocuments); Assert.Equal(0, _processor.LastBatchAbandonedDocuments); }
public async Task ProcessEvents_OneUnknownDocumentType_AbandonsUnknownDocumentType() { _client.SimulateElasticSearch(); var events = new[] { MockEventData.CreateInvalidEventData(@"{ ""message"": ""invalid type"" }"), }; await _processor.OpenAsync(_partition); await _processor.ProcessEventsAsync(_partition, events); await _processor.CloseAsync(_partition, CloseReason.Shutdown); // We should try calling BulkAsync once to abandon the invalid document. _client.VerifyBulkAsyncCalled(Times.Exactly(1)); Assert.Equal(0, _processor.LastBatchFailedDocuments); // The invalid document type should not be retried. Assert.Equal(1, _processor.LastBatchAbandonedDocuments); // The invalid document type should be abandoned immediately. }
public async Task ProcessEvents_DocumentWithNewlines_AbandonsInvalidDocument() { var events = new[] { MockEventData.CreateSerilogEventData($"{{ \"message\": \"ProcessEvents_DocumentWithNewlines_AbandonsInvalidDocument\", \"@timestamp\":\"{DateTime.UtcNow:o}\", \"data\": 1 }}"), MockEventData.CreateSerilogEventData(@" { ""message"": ""documents formatted with newlines are invalid for Elasticsearch bulk upload."", ""data"": 2 }" ), MockEventData.CreateSerilogEventData($"{{ \"message\": \"ProcessEvents_DocumentWithNewlines_AbandonsInvalidDocument\", \"@timestamp\":\"{DateTime.UtcNow:o}\", \"data\": 3 }}"), }; await _processor.OpenAsync(_partition); await _processor.ProcessEventsAsync(_partition, events); await _processor.CloseAsync(_partition, CloseReason.Shutdown); Assert.Equal(0, _processor.LastBatchFailedDocuments); Assert.Equal(1, _processor.LastBatchAbandonedDocuments); }
public async Task ProcessEvents_InvalidDocument_AndUnknownDocumentType_AbandonsInvalidAndUnknownDocuments() { var events = new[] { MockEventData.CreateInvalidEventData(@"{ ""message"": ""invalid type"" }"), MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 1}"), MockEventData.CreateSerilogEventData(@"invalid json"), MockEventData.CreateRoboCustosEventData(@"{ ""message"": ""log message"", ""data"": 2}"), MockEventData.CreateSerilogEventData(@""), MockEventData.CreateExternalTelemetryEventData(@"{ ""message"": ""log message"", ""data"": 3}"), MockEventData.CreateAzureResourcesEventData(@"{ ""message"": ""log message"", ""data"": 4}"), }; await _processor.OpenAsync(_partition); await _processor.ProcessEventsAsync(_partition, events); await _processor.CloseAsync(_partition, CloseReason.Shutdown); Assert.Equal(2, _processor.LastBatchFailedDocuments); Assert.Equal(3, _processor.LastBatchAbandonedDocuments); }
public async Task ProcessEvents_OneInvalidJsonDocument_TwoValidDocuments_AbandonsInvalidJsonDocument() { _client.SimulateElasticSearch(); var events = new[] { MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 1}"), MockEventData.CreateSerilogEventData(@"invalid json"), MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 2}"), }; await _processor.OpenAsync(_partition); await _processor.ProcessEventsAsync(_partition, events); await _processor.CloseAsync(_partition, CloseReason.Shutdown); // We should try calling BulkAsync once for the initial request, MaxFailedDocumentRetries times to retry the failed documents, then once to abandon the document. _client.VerifyBulkAsyncCalled(Times.Exactly(1 + MaxFailedDocumentRetries + 1)); Assert.Equal(1, _processor.LastBatchFailedDocuments); Assert.Equal(1, _processor.LastBatchAbandonedDocuments); }