public async Task ProcessEvents_InvalidDocument_AndUnknownDocumentType_AbandonsInvalidAndUnknownDocuments()
{
_client.SimulateElasticSearch();
var events = new[]
{
MockEventData.CreateInvalidEventData(@"{ ""message"": ""invalid type"" }"),
MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 1}"),
MockEventData.CreateSerilogEventData(@"invalid json"),
MockEventData.CreateRoboCustosEventData(@"{ ""message"": ""log message"", ""data"": 2}"),
MockEventData.CreateSerilogEventData(@""),
MockEventData.CreateExternalTelemetryEventData(@"{ ""message"": ""log message"", ""data"": 3}"),
MockEventData.CreateAzureResourcesEventData(@"{ ""message"": ""log message"", ""data"": 4}"),
};
await _processor.OpenAsync(_partition);
await _processor.ProcessEventsAsync(_partition, events);
await _processor.CloseAsync(_partition, CloseReason.Shutdown);
// We should try calling BulkAsync once for the initial request, MaxFailedDocumentRetries times to retry the failed documents,
// once to abandon the invalid documents, and once to abandon the unknown document type.
_client.VerifyBulkAsyncCalled(Times.Exactly(1 + MaxFailedDocumentRetries + 1 + 1));
Assert.Equal(4 + 2 + 3, _client.GetTotalItems()); // 4 successful, 2 ingestionstats, 3 abandoned
Assert.Equal(1, _client.GetTotalItemsInIndex("logstash"));
Assert.Equal(1, _client.GetTotalItemsInIndex("robointeractions"));
Assert.Equal(1, _client.GetTotalItemsInIndex("externaltelemetry"));
Assert.Equal(1, _client.GetTotalItemsInIndex("azure-resources"));
Assert.Equal(2, _client.GetTotalItemsInIndex("ingestionstats"));
Assert.Equal(3, _client.GetTotalItemsInIndex("abandoneddocs"));
Assert.Equal(2, _processor.LastBatchFailedDocuments);
Assert.Equal(3, _processor.LastBatchAbandonedDocuments);
}
public async Task ProcessEvents_OneUnknownDocumentType_AbandonsUnknownDocumentType()
{
var events = new[]
{
MockEventData.CreateInvalidEventData(@"{ ""message"": ""invalid type"" }"),
};
await _processor.OpenAsync(_partition);
await _processor.ProcessEventsAsync(_partition, events);
await _processor.CloseAsync(_partition, CloseReason.Shutdown);
Assert.Equal(0, _processor.LastBatchFailedDocuments); // The invalid document type should not be retried.
Assert.Equal(1, _processor.LastBatchAbandonedDocuments); // The invalid document type should be abandoned immediately.
}
public async Task ProcessEvents_OneUnknownDocumentType_AbandonsUnknownDocumentType()
{
_client.SimulateElasticSearch();
var events = new[]
{
MockEventData.CreateInvalidEventData(@"{ ""message"": ""invalid type"" }"),
};
await _processor.OpenAsync(_partition);
await _processor.ProcessEventsAsync(_partition, events);
await _processor.CloseAsync(_partition, CloseReason.Shutdown);
// We should try calling BulkAsync once to abandon the invalid document.
_client.VerifyBulkAsyncCalled(Times.Exactly(1));
Assert.Equal(0, _processor.LastBatchFailedDocuments); // The invalid document type should not be retried.
Assert.Equal(1, _processor.LastBatchAbandonedDocuments); // The invalid document type should be abandoned immediately.
}
public async Task ProcessEvents_InvalidDocument_AndUnknownDocumentType_AbandonsInvalidAndUnknownDocuments()
{
var events = new[]
{
MockEventData.CreateInvalidEventData(@"{ ""message"": ""invalid type"" }"),
MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 1}"),
MockEventData.CreateSerilogEventData(@"invalid json"),
MockEventData.CreateRoboCustosEventData(@"{ ""message"": ""log message"", ""data"": 2}"),
MockEventData.CreateSerilogEventData(@""),
MockEventData.CreateExternalTelemetryEventData(@"{ ""message"": ""log message"", ""data"": 3}"),
MockEventData.CreateAzureResourcesEventData(@"{ ""message"": ""log message"", ""data"": 4}"),
};
await _processor.OpenAsync(_partition);
await _processor.ProcessEventsAsync(_partition, events);
await _processor.CloseAsync(_partition, CloseReason.Shutdown);
Assert.Equal(2, _processor.LastBatchFailedDocuments);
Assert.Equal(3, _processor.LastBatchAbandonedDocuments);
}